├── .gitignore ├── LICENSE ├── README.md ├── RokuHandRemote.spec ├── RokuRemote.spec ├── gestureDetection.py ├── images ├── icon.ico ├── readme-images │ ├── HandiRokuRemoteGif-large.gif │ ├── HandiRokuRemoteGif.gif │ ├── HandiRokuRemoteGif2-large.gif │ ├── HandiRokuRemoteGif2.gif │ ├── handiRokuRemote-GUI.jpg │ ├── handiRokuRemote-cheatsheet-small.jpg │ ├── handiRokuRemote-cheatsheet.jpg │ └── tooltip-demo.jpg ├── splash.png └── tooltip-images │ ├── fingers-crossed.jpg │ ├── fingers-deactivate.jpg │ ├── fingers-mute.jpg │ ├── fingersSplayed.jpg │ ├── fist.jpg │ ├── home.jpg │ ├── horns.jpg │ ├── i-down.jpg │ ├── i-left.jpg │ ├── i-m-left.jpg │ ├── i-m-right.jpg │ ├── i-m-up.jpg │ ├── i-right.jpg │ ├── i-up.jpg │ ├── palmup.jpg │ ├── thumbs-down.jpg │ ├── thumbs-in.jpg │ ├── thumbs-left.jpg │ ├── thumbs-out.jpg │ ├── thumbs-right.jpg │ └── thumbs-up.jpg ├── requirements.txt ├── rokuRemote.py └── tooltip.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # PyInstaller build directories and files 7 | /build/ 8 | /dist/ 9 | # *.spec 10 | 11 | # Logs and debug files 12 | *.log 13 | 14 | # Config files 15 | roku_config.json 16 | 17 | # Exclude virtual environments 18 | venv/ 19 | env/ 20 | 21 | # OS-specific files 22 | .DS_Store 23 | Thumbs.db"*.exe" 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Handi Roku Remote 2 | [Link to Windows Download](https://github.com/BBelk/HandiRokuRemote/releases/tag/v1) 3 | ## Table of contents 4 | 1. [Description](#description) 5 | 2. [How It Works](#how-it-works) 6 | 3. [Installation and Interface Overview](#installation-and-interface-overview) 7 | 4. [Gestures and Commands](#gestures-and-commands) 8 | 5. [Limitations](#limitations) 9 | 6. [Potential Future Developent](#potential-future-development) 10 | 11 | ![Alt text](./images/readme-images/HandiRokuRemoteGif2.gif "Handi Roku Remote Gif Demonstration") 12 | 13 | ## Description 14 | Handi Roku Remote allows you to control your Roku device using hand gestures. The application automatically connects to devices on your local network, gathers a list of available cameras connected to your computer, and tracks a hand to translate gestures into commands sent to Roku's External Control Protocol (ECP). 15 | 16 | ## How It Works 17 | This project uses [Google's Mediapipe](https://github.com/google-ai-edge/mediapipe) [Hand Landmark Detector](https://ai.google.dev/edge/mediapipe/solutions/vision/hand_landmarker) for real-time hand tracking. It identifies keypoints on the palm and fingers, evaluates their positions and orientations, and determines the user's gesture. The recognized gesture is then translated into a command sent to the Roku device through its ECP interface. 18 | 19 | The application is built using Python and leverages several libraries: 20 | 21 | • Tkinter for the graphical user interface. 22 | 23 | • OpenCV for video capture and display. 24 | 25 | • Requests for HTTP communication to discover devices and send commands. 26 | 27 | ## Installation and Interface Overview 28 | If you've already got the program running, skip ahead to the [Gestures and Commands](#gestures-and-commands) section. 29 | 30 | ### System Requirements 31 | • Operating System: Windows 32 | 33 | • Camera: Any built-in camera or USB camera should work 34 | 35 | • Roku Device: Connected to the same local network as the computer running this program 36 | 37 | ### Installation 38 | • [Download](https://github.com/BBelk/HandiRokuRemote/releases/tag/v1) the latest version of the .exe 39 | 40 | • Place the .exe file anywhere on your computer 41 | 42 | • Double-click HandiRokuRemote.exe to launch the applicaiton 43 | 44 | ### Interface Overview 45 | 46 | Upon launching the applicaiton, you will see the main window: 47 | 48 | ![Alt text](./images/readme-images/handiRokuRemote-GUI.jpg "HandiRokuRemote GUI") 49 | 50 | ### Video Feed Controls 51 | • Start Video Feed: Begins capturing video from the selected camera. The feed appears on the right side of the window. 52 | 53 | • Stop Video Feed: Halts the video capture. 54 | 55 | ### Settings 56 | • Debug Mode: Displays additional information such as finger extension status and base distance. Useful for troubleshooting. 57 | 58 | • Auto Start: Automatically starts the video feed with the selected options when the application launches. 59 | 60 | • Skeleton View: Overlays landmarks and connections on the hand image for visual feedback. 61 | 62 | ### Device Selection 63 | • Automatic Discovery: The application attempts to discover Roku devices on your local network. Select your device from the dropdown list. 64 | 65 | • Manual IP Entry: If your device is not found, you can enter its IP address manually. To find your Roku's IP address, navigate on your Roku device to Settings > Network > About. 66 | 67 | ### Camera Selection 68 | • Select Camera: Choose the camera you wish to use for gesture detection from the dropdown list. 69 | 70 | • Refresh Cameras: Updates the list of available cameras if you connect a new one. 71 | ### Settings Directory 72 | Displays the location of configuration and log files used by the application: 73 | 74 | • roku_config.json: Stores settings and discovered devices for easier access. 75 | 76 | • roku_remote.log: Contains logs that can be used for troubleshooting. 77 | 78 | ### Navigation Buttons 79 | These buttons simulate a Roku remote and can be clicked to send commands directly to your Roku device. Hover over each button to see a tooltip that shows the corresponding gesture. 80 | 81 | ![Alt text](./images/readme-images/tooltip-demo.jpg "Tooltip Demonstration") 82 | 83 | ## Gestures and Commands 84 | Once you've got the Video Feed started, on your right you will see your camera's view. In the top right of the camera view, you should see text saying either "Idle" or "Active". Idle mode is triggered when a hand is not currently being tracked for gesture recognition. Note that only one hand is tracked at a time. 85 | 86 | To activate gesture recognition, cross your index finger over your middle finger, keeping the tips relatively close. This is the letter 'R' in American sign language (get it, R for Roku?). Now the program is 'Active' and detecting gestures on the hand. Between each gesture, you must make a fist. Once a fist is made, you can then perform another gesture, and the program sends the command. To stop tracking, you can either perform the 'R' gesture again, or simply hide the hand from the camera's view. 87 | 88 | Here is a cheatsheet of gestures mapped to Roku commands: 89 | 90 | ![Alt text](./images/readme-images/handiRokuRemote-cheatsheet-small.jpg "Gesture to Command Cheatsheet") 91 | 92 | Note: The direction in which you point your fingers determines the command for certain gestures, especially for navigation and media control. 93 | 94 | ### Pro-Tips: 95 | 96 | • Lighting: Ensure you hand is well lit and avoid presenting the hand with a similar skin tone behind it 97 | 98 | • Distance: Keep within 2 meters of the camera for optimal tracking 99 | 100 | • Steady Gestures: The application waits for a gesture to be held for 1/3rd of a second before recognition occours 101 | 102 | ![Alt text](./images/readme-images/HandiRokuRemoteGif.gif "Handi Roku Remote Gif Demonstration") 103 | 104 | ## Limitations 105 | Overall I am pretty pleased with the project. Mediapipe is incredibly powerful but unfortunately, it's not magic. It struggles detecting far away hands; I can get about 2 meters away before the illusion fails. This is just fine for playing with on a regular computer directly in front of you. But in a hypothetical scenario where you'd want to shove this on a raspberry pi and mount it on top of your TV, a method for detecting visually smaller hands would be necessary. 106 | 107 | ## Potential Future Development 108 | There are multiple avenues for continued development. Creating a further-away hand-landmark detector or gesture detector is a relatively straight-forward process (provided you have a few thousand annotated images of hands). I originally wanted to slap this on a raspberry pi but the distance limits on hand detection threatened to turn this side-side project into a real side-project. 109 | 110 | This project's Roku specific code could be modified or further extended to work with specific Smart TVs or any TV that supports HDMI-CEC (Consumer Electronics Control). Really, the sky is the limit. If TV manufacturers aren't already experimenting with this kind of tech then they're missing out, it's a lot of fun! 111 | 112 | 113 | 114 | ## Thanks To 115 | [OpenMoji](https://openmoji.org/) for the great free emojis! -------------------------------------------------------------------------------- /RokuHandRemote.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python ; coding: utf-8 -*- 2 | 3 | block_cipher = None 4 | 5 | a = Analysis( 6 | ['rokuRemote.py'], 7 | pathex=[], 8 | binaries=[], 9 | datas=[ 10 | # Include only necessary mediapipe runtime files 11 | # For example, if you only need the 'hands' module: 12 | (r'C:\ProgramData\anaconda3\envs\rokuRemoteEnv\Lib\site-packages\mediapipe\modules', 'mediapipe/modules'), 13 | # Include additional Python files 14 | ('gestureDetection.py', '.'), 15 | ('tooltip.py', '.'), # Include if used 16 | # Include images 17 | ('images/icon.ico', 'images'), 18 | ('images/tooltip-images', 'images/tooltip-images') 19 | ], 20 | 21 | hiddenimports=[ 22 | 'mediapipe.python._framework_bindings', # For mediapipe bindings 23 | 'mediapipe.python.solutions', # To include hands solution 24 | 'PIL._imagingtk', # For tkinter/Pillow 25 | 'PIL.Image', # Pillow 26 | 'cv2', # OpenCV 27 | 'numpy', # NumPy 28 | 'secrets' 29 | # Include any other hidden imports required 30 | ], 31 | 32 | hookspath=[], 33 | hooksconfig={}, 34 | runtime_hooks=[], 35 | excludes=[ 36 | ], 37 | noarchive=False, 38 | optimize=0, # Optimize bytecode 39 | ) 40 | 41 | pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) 42 | 43 | exe = EXE( 44 | pyz, 45 | a.scripts, 46 | a.binaries, 47 | a.datas, 48 | [], 49 | name='HandiRokuRemote', 50 | debug=False, # Disable debug logging in final builds 51 | bootloader_ignore_signals=False, 52 | strip=True, # Strip symbols to reduce size 53 | upx=True, # Enable UPX compression 54 | upx_exclude=[], 55 | runtime_tmpdir=None, 56 | console=False, # Hide console window 57 | disable_windowed_traceback=False, 58 | argv_emulation=False, 59 | target_arch=None, 60 | codesign_identity=None, 61 | entitlements_file=None, 62 | icon='images/icon.ico' 63 | ) 64 | 65 | # Explicitly set UPX directory if needed 66 | import os 67 | os.environ['UPX_DIR'] = r'C:\Tools\upx' 68 | -------------------------------------------------------------------------------- /RokuRemote.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python ; coding: utf-8 -*- 2 | 3 | 4 | a = Analysis( 5 | ['RokuRemote.py'], 6 | pathex=[], 7 | binaries=[], 8 | datas=[], 9 | hiddenimports=[], 10 | hookspath=[], 11 | hooksconfig={}, 12 | runtime_hooks=[], 13 | excludes=[], 14 | noarchive=False, 15 | optimize=0, 16 | ) 17 | pyz = PYZ(a.pure) 18 | 19 | exe = EXE( 20 | pyz, 21 | a.scripts, 22 | a.binaries, 23 | a.datas, 24 | [], 25 | name='RokuRemote', 26 | debug=False, 27 | bootloader_ignore_signals=False, 28 | strip=False, 29 | upx=True, 30 | upx_exclude=[], 31 | runtime_tmpdir=None, 32 | console=True, 33 | disable_windowed_traceback=False, 34 | argv_emulation=False, 35 | target_arch=None, 36 | codesign_identity=None, 37 | entitlements_file=None, 38 | icon=['images\\icon.ico'], 39 | ) 40 | -------------------------------------------------------------------------------- /gestureDetection.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import mediapipe as mp 3 | import time 4 | import math 5 | import logging 6 | 7 | class HandClassifier(threading.Thread): 8 | def __init__(self, hand_id, initial_landmarks, command_callback, hand_label): 9 | super().__init__(daemon=True) 10 | self.hand_id = hand_id 11 | self.landmarks = initial_landmarks 12 | self.landmarks_lock = threading.Lock() 13 | self.running = True 14 | self.command_callback = command_callback 15 | self.hand_label = hand_label # 'Left' or 'Right' 16 | 17 | # Gesture detection variables 18 | self.last_gesture = None 19 | self.state = 'Idle' # Initial state is 'Idle' 20 | self.last_fist_time = None # Time when fist was entered 21 | self.delay_after_fist = 0.3 # Delay in seconds 22 | 23 | self.r_gesture_detected = False # Flag to prevent multiple transitions on continuous 'R' gesture 24 | self.r_gesture_start_time = None # Start time when 'R' gesture is first detected 25 | self.min_r_duration = 0.3 # Minimum duration for 'R' gesture 26 | 27 | self.fist_made_in_active = False # Flag to ensure fist is made after entering 'Active' 28 | 29 | # Variables to store current gesture, hand angle, and extended fingers 30 | self.current_gesture = None 31 | self.current_direction = None # To store the current direction 32 | self.extended_fingers = None # This will be a tuple of booleans (t, i, m, r, p) 33 | self.base_distance = None # Base distance will be set upon detecting R gesture 34 | self.debug_string = "" 35 | self.thumb_distance = None 36 | self.angle = 0 37 | self.angle2 = 0 38 | 39 | # Variables to store points for drawing 40 | self.line_points = None # Tuple of (thumb_point_index, selected_other_point_index) 41 | 42 | # Variables for gesture delay after fist 43 | self.gesture_start_time = None # Start time when fingers are extended after fist 44 | self.min_gesture_duration = 0.3 # Minimum duration to hold the gesture before accepting it 45 | self.debug_r_string = "" 46 | self.end_tracking_time = time.time() 47 | 48 | def update_landmarks(self, new_landmarks, hand_label): 49 | with self.landmarks_lock: 50 | self.landmarks = new_landmarks 51 | self.hand_label = hand_label 52 | 53 | def stop(self): 54 | self.running = False 55 | 56 | def run(self): 57 | try: 58 | while self.running: 59 | with self.landmarks_lock: 60 | landmarks = self.landmarks 61 | 62 | if landmarks: 63 | # Process landmarks to detect gestures 64 | gesture = self.detect_gesture(landmarks, self.hand_label) 65 | if not self.running: 66 | break # Exit immediately if stopped 67 | if gesture and gesture != self.last_gesture: 68 | self.last_gesture = gesture 69 | if self.current_direction is None: 70 | self.current_direction = "None" 71 | if gesture is not None: 72 | # Build the debug string 73 | self.debug_string = ( 74 | f"T:{self.extended_fingers[0]} I:{self.extended_fingers[1]} " 75 | f"M:{self.extended_fingers[2]} R:{self.extended_fingers[3]} " 76 | f"P:{self.extended_fingers[4]}\nThumb distance: {self.thumb_distance}\nAngle:{self.angle}" 77 | f"Base Distance: {self.base_distance}" 78 | # \nGesture: {self.current_gesture or 'None'}\n" 79 | # f"Direction: {self.current_direction or 'None'}\nState: {self.state}" 80 | ) 81 | # Call the callback function with the detected gesture 82 | self.command_callback(gesture, self.debug_string, self.current_direction, self.state) 83 | elif gesture is None: 84 | self.last_gesture = None 85 | if self.extended_fingers is not None: 86 | self.debug_string = ( 87 | f"T:{self.extended_fingers[0]} I:{self.extended_fingers[1]} " 88 | f"M:{self.extended_fingers[2]} R:{self.extended_fingers[3]} " 89 | f"P:{self.extended_fingers[4]}\nThumb distance: {self.thumb_distance}\nAngle:{self.angle}" 90 | f"Base Distance: {self.base_distance}" 91 | # \nGesture: {self.current_gesture or 'None'}\n" 92 | # f"Direction: {self.current_direction or 'None'}\nState: {self.state}" 93 | ) 94 | self.command_callback(None, self.debug_string, self.current_direction, self.state) 95 | else: 96 | self.last_gesture = None 97 | # Do not reset the state immediately; handled in main code 98 | self.r_gesture_detected = False # Reset the R gesture flag when hand is lost 99 | self.fist_made_in_active = False # Reset fist flag 100 | self.base_distance = None # Reset base_distance when hand is lost 101 | self.current_direction = None # Reset direction 102 | self.r_gesture_start_time = None # Reset R gesture start time 103 | self.gesture_start_time = None # Reset gesture start time 104 | 105 | time.sleep(0.01) 106 | except Exception as e: 107 | print(f"Exception in HandClassifier thread: {e}") 108 | logging.debug(f"Exception in HandClassifier thread: {e}") 109 | 110 | def get_extended_fingers(self, landmarks): 111 | """ 112 | Determines which fingers are extended based on angles and distances. 113 | Returns booleans for thumb, index, middle, ring, pinky fingers. 114 | Also stores the thumb point and the selected other point for drawing. 115 | """ 116 | try: 117 | if self.base_distance is None: 118 | # Cannot determine finger extensions without base_distance 119 | self.line_points = None # Reset line points 120 | return False, False, False, False, False 121 | 122 | import math # Ensure math module is imported 123 | 124 | mp_hands = mp.solutions.hands 125 | 126 | wrist = landmarks.landmark[mp_hands.HandLandmark.WRIST] 127 | 128 | # List of landmark IDs for all fingers (index, middle, ring, pinky) 129 | finger_tip_ids = [ 130 | mp_hands.HandLandmark.INDEX_FINGER_TIP, 131 | mp_hands.HandLandmark.MIDDLE_FINGER_TIP, 132 | mp_hands.HandLandmark.RING_FINGER_TIP, 133 | mp_hands.HandLandmark.PINKY_TIP 134 | ] 135 | finger_dip_ids = [ 136 | mp_hands.HandLandmark.INDEX_FINGER_DIP, 137 | mp_hands.HandLandmark.MIDDLE_FINGER_DIP, 138 | mp_hands.HandLandmark.RING_FINGER_DIP, 139 | mp_hands.HandLandmark.PINKY_DIP 140 | ] 141 | finger_pip_ids = [ 142 | mp_hands.HandLandmark.INDEX_FINGER_PIP, 143 | mp_hands.HandLandmark.MIDDLE_FINGER_PIP, 144 | mp_hands.HandLandmark.RING_FINGER_PIP, 145 | mp_hands.HandLandmark.PINKY_PIP 146 | ] 147 | finger_mcp_ids = [ 148 | mp_hands.HandLandmark.INDEX_FINGER_MCP, 149 | mp_hands.HandLandmark.MIDDLE_FINGER_MCP, 150 | mp_hands.HandLandmark.RING_FINGER_MCP, 151 | mp_hands.HandLandmark.PINKY_MCP 152 | ] 153 | 154 | # Helper function to calculate the angle between three points 155 | def calculate_angle(a, b, c): 156 | """ 157 | Calculates the angle at point 'b' given three points a, b, and c. 158 | """ 159 | ba = [a.x - b.x, a.y - b.y, a.z - b.z] 160 | bc = [c.x - b.x, c.y - b.y, c.z - b.z] 161 | 162 | # Calculate the dot product and magnitudes 163 | dot_product = ba[0]*bc[0] + ba[1]*bc[1] + ba[2]*bc[2] 164 | magnitude_ba = math.sqrt(ba[0]**2 + ba[1]**2 + ba[2]**2) 165 | magnitude_bc = math.sqrt(bc[0]**2 + bc[1]**2 + bc[2]**2) 166 | 167 | if magnitude_ba * magnitude_bc == 0: 168 | return 0.0 169 | 170 | # Calculate the angle in radians and then convert to degrees 171 | angle_rad = math.acos(dot_product / (magnitude_ba * magnitude_bc)) 172 | angle_deg = math.degrees(angle_rad) 173 | return angle_deg 174 | 175 | # Check extension for each finger 176 | extended = [] 177 | for tip_id, dip_id, pip_id, mcp_id in zip(finger_tip_ids, finger_dip_ids, finger_pip_ids, finger_mcp_ids): 178 | # Retrieve landmark points 179 | mcp = landmarks.landmark[mcp_id] 180 | pip = landmarks.landmark[pip_id] 181 | dip = landmarks.landmark[dip_id] 182 | tip = landmarks.landmark[tip_id] 183 | 184 | # Calculate angles at PIP and DIP joints 185 | angle_pip = calculate_angle(mcp, pip, dip) 186 | angle_dip = calculate_angle(pip, dip, tip) 187 | 188 | # Determine finger extension based on angles 189 | if angle_pip > 140 and angle_dip > 140: 190 | angle_extended = True 191 | else: 192 | angle_extended = False 193 | 194 | # Distance-based method (kept for robustness) 195 | distance = self.calculate_distance(tip, mcp) 196 | wrist_distance = self.calculate_distance(tip, wrist) 197 | knuckle_distance = self.calculate_distance(mcp, wrist) 198 | dip_distance = self.calculate_distance(dip, pip) 199 | 200 | if (wrist_distance < knuckle_distance) or (dip_distance < 0.35 * self.base_distance): 201 | distance_extended = False 202 | elif distance > 0.5 * self.base_distance: 203 | distance_extended = True 204 | else: 205 | distance_extended = False 206 | 207 | # Combine angle and distance methods, angle method overrules 208 | if angle_extended and not (wrist_distance < knuckle_distance): 209 | extended.append(True) 210 | else: 211 | extended.append(distance_extended) 212 | 213 | # Thumb detection (unchanged) 214 | thumb_points = [ 215 | (mp_hands.HandLandmark.THUMB_TIP, landmarks.landmark[mp_hands.HandLandmark.THUMB_TIP]), 216 | (mp_hands.HandLandmark.THUMB_IP, landmarks.landmark[mp_hands.HandLandmark.THUMB_IP]) 217 | ] 218 | 219 | candidate_points = [ 220 | (mp_hands.HandLandmark.INDEX_FINGER_MCP, landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_MCP]), 221 | (mp_hands.HandLandmark.INDEX_FINGER_DIP, landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_DIP]), 222 | (mp_hands.HandLandmark.INDEX_FINGER_PIP, landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_PIP]), 223 | (mp_hands.HandLandmark.MIDDLE_FINGER_DIP, landmarks.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_DIP]), 224 | (mp_hands.HandLandmark.MIDDLE_FINGER_MCP, landmarks.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_MCP]), 225 | (mp_hands.HandLandmark.RING_FINGER_MCP, landmarks.landmark[mp_hands.HandLandmark.RING_FINGER_MCP]) 226 | ] 227 | 228 | min_distance = None 229 | selected_thumb_point = None 230 | selected_other_point = None 231 | 232 | for thumb_id, thumb_point in thumb_points: 233 | for other_id, other_point in candidate_points: 234 | distance = self.calculate_distance(thumb_point, other_point) 235 | if min_distance is None or distance < min_distance: 236 | min_distance = distance 237 | selected_thumb_point = thumb_id 238 | selected_other_point = other_id 239 | 240 | self.thumb_distance = min_distance 241 | thumb_extended = self.thumb_distance > 0.3 * self.base_distance 242 | self.line_points = (selected_thumb_point, selected_other_point) 243 | 244 | thumb_extended, index_extended, middle_extended, ring_extended, pinky_extended = [thumb_extended] + extended 245 | return thumb_extended, index_extended, middle_extended, ring_extended, pinky_extended 246 | 247 | except Exception as e: 248 | print(f"Error in get_extended_fingers: {e}") 249 | logging.debug(f"Error in get_extended_fingers: {e}") 250 | return False, False, False, False, False 251 | 252 | def calculate_distance(self, point1, point2): 253 | """ 254 | Calculates the Euclidean distance between two landmarks. 255 | """ 256 | dx = point1.x - point2.x 257 | dy = point1.y - point2.y 258 | dz = point1.z - point2.z 259 | return math.sqrt(dx*dx + dy*dy + dz*dz) 260 | 261 | def detect_r_gesture(self, landmarks, hand_label): 262 | """ 263 | Detects the 'R' gesture (sign language 'R') where index and middle fingers are crossed, 264 | only when the palm is facing the camera and the fingers are extended. 265 | Works for both left and right hands. 266 | """ 267 | if self.end_tracking_time > time.time() - 1: 268 | return 269 | mp_hands = mp.solutions.hands 270 | self.debug_r_string = "" 271 | 272 | # Helper function to calculate the angle between three points 273 | def calculate_angle(a, b, c): 274 | """ 275 | Calculates the angle at point 'b' given three points a, b, and c. 276 | """ 277 | ba = [a.x - b.x, a.y - b.y, a.z - b.z] 278 | bc = [c.x - b.x, c.y - b.y, c.z - b.z] 279 | 280 | # Calculate the dot product and magnitudes 281 | dot_product = ba[0]*bc[0] + ba[1]*bc[1] + ba[2]*bc[2] 282 | magnitude_ba = math.sqrt(ba[0]**2 + ba[1]**2 + ba[2]**2) 283 | magnitude_bc = math.sqrt(bc[0]**2 + bc[1]**2 + bc[2]**2) 284 | 285 | if magnitude_ba * magnitude_bc == 0: 286 | return 0.0 287 | 288 | # Calculate the angle in radians and then convert to degrees 289 | angle_rad = math.acos(dot_product / (magnitude_ba * magnitude_bc)) 290 | angle_deg = math.degrees(angle_rad) 291 | return angle_deg 292 | 293 | # Helper function to check if two line segments in 2D intersect 294 | def lines_intersect(p1, p2, p3, p4): 295 | """ 296 | Checks if line segment p1-p2 intersects with line segment p3-p4. 297 | Points are expected to be in 2D. 298 | """ 299 | def ccw(a, b, c): 300 | return (c.y - a.y) * (b.x - a.x) > (b.y - a.y) * (c.x - a.x) 301 | 302 | return (ccw(p1, p3, p4) != ccw(p2, p3, p4)) and (ccw(p1, p2, p3) != ccw(p1, p2, p4)) 303 | 304 | wrist = landmarks.landmark[mp_hands.HandLandmark.WRIST] 305 | index_mcp = landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_MCP] 306 | index_tip = landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP] 307 | pinky_mcp = landmarks.landmark[mp_hands.HandLandmark.PINKY_MCP] 308 | 309 | # Vectors along the palm 310 | v1 = [ 311 | index_mcp.x - wrist.x, 312 | index_mcp.y - wrist.y, 313 | index_mcp.z - wrist.z 314 | ] 315 | v2 = [ 316 | pinky_mcp.x - wrist.x, 317 | pinky_mcp.y - wrist.y, 318 | pinky_mcp.z - wrist.z 319 | ] 320 | 321 | # Normal vector (cross product) 322 | normal = [ 323 | v1[1]*v2[2] - v1[2]*v2[1], 324 | v1[2]*v2[0] - v1[0]*v2[2], 325 | v1[0]*v2[1] - v1[1]*v2[0] 326 | ] 327 | # Adjust for left hand 328 | if hand_label == 'Left': 329 | normal = [-n for n in normal] 330 | 331 | # Determine if palm is facing the camera 332 | palm_facing_camera = normal[2] > 0 333 | angle_wrist = calculate_angle(wrist, index_mcp, index_tip) 334 | 335 | self.debug_r_string += f"n:{normal[2]} pfc:{palm_facing_camera}\n wrist_angle:{angle_wrist}" 336 | 337 | if not palm_facing_camera or angle_wrist < 150: 338 | return False 339 | 340 | # Check if index and middle fingers are extended using angles 341 | finger_ids = { 342 | 'index': [ 343 | mp_hands.HandLandmark.INDEX_FINGER_MCP, 344 | mp_hands.HandLandmark.INDEX_FINGER_PIP, 345 | mp_hands.HandLandmark.INDEX_FINGER_DIP, 346 | mp_hands.HandLandmark.INDEX_FINGER_TIP 347 | ], 348 | 'middle': [ 349 | mp_hands.HandLandmark.MIDDLE_FINGER_MCP, 350 | mp_hands.HandLandmark.MIDDLE_FINGER_PIP, 351 | mp_hands.HandLandmark.MIDDLE_FINGER_DIP, 352 | mp_hands.HandLandmark.MIDDLE_FINGER_TIP 353 | ] 354 | } 355 | 356 | fingers_extended = {} 357 | for finger_name, ids in finger_ids.items(): 358 | mcp = landmarks.landmark[ids[0]] 359 | pip = landmarks.landmark[ids[1]] 360 | dip = landmarks.landmark[ids[2]] 361 | tip = landmarks.landmark[ids[3]] 362 | 363 | angle_pip = calculate_angle(mcp, pip, dip) 364 | angle_dip = calculate_angle(pip, dip, tip) 365 | self.debug_r_string += f"\n{finger_name} pip: {angle_pip} dip: {angle_dip}" 366 | 367 | if angle_pip > 160 and angle_dip > 160: 368 | fingers_extended[finger_name] = True 369 | else: 370 | fingers_extended[finger_name] = False 371 | 372 | if not (fingers_extended['index'] and fingers_extended['middle']): 373 | return False 374 | 375 | # Now that fingers are extended, calculate temp_base_distance 376 | temp_base_distance = self.calculate_distance(index_tip, wrist) 377 | middle_mcp = landmarks.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_MCP] 378 | middle_to_wrist_distance = self.calculate_distance(middle_mcp, wrist) 379 | 380 | self.debug_r_string += f"\ntbd: {temp_base_distance} mtwd: {middle_to_wrist_distance}" 381 | if temp_base_distance < middle_to_wrist_distance: 382 | return False 383 | 384 | middle_tip = landmarks.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_TIP] 385 | 386 | tip_distance = self.calculate_distance(index_tip, middle_tip) 387 | 388 | fingers_close = tip_distance < 0.08 * temp_base_distance 389 | self.debug_r_string += f"\ntip_distance: {tip_distance}" 390 | 391 | if fingers_close: 392 | # Additional check: Do the lines from index PIP to TIP and middle PIP to TIP intersect? 393 | index_pip = landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_PIP] 394 | middle_pip = landmarks.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_PIP] 395 | 396 | # Convert 3D landmarks to 2D by ignoring the Z coordinate 397 | index_line_start = index_pip 398 | index_line_end = index_tip 399 | middle_line_start = middle_pip 400 | middle_line_end = middle_tip 401 | 402 | # Check if the lines intersect 403 | intersect = lines_intersect( 404 | index_line_start, index_line_end, 405 | middle_line_start, middle_line_end 406 | ) 407 | 408 | if intersect: 409 | self.base_distance = self.calculate_distance(index_tip, middle_mcp) 410 | return True 411 | else: 412 | return False 413 | else: 414 | return False 415 | 416 | 417 | def detect_gesture(self, landmarks, hand_label): 418 | current_time = time.time() 419 | 420 | # Check for 'R' gesture in Idle state 421 | if self.state == "Idle": 422 | is_r_gesture = self.detect_r_gesture(landmarks, hand_label) 423 | if is_r_gesture: 424 | if self.r_gesture_start_time is None: 425 | self.r_gesture_start_time = current_time 426 | print(f"{self.hand_id}: 'R' gesture detected, starting timer...") 427 | logging.debug(f"{self.hand_id}: 'R' gesture detected, starting timer...") 428 | elif current_time - self.r_gesture_start_time >= self.min_r_duration: 429 | self.state = "Active" 430 | self.r_gesture_detected = True 431 | self.last_fist_time = None 432 | self.fist_made_in_active = False 433 | print(f"{self.hand_id}: Transition to 'Active' state (Gesture detection ON)") 434 | logging.debug(f"{self.hand_id}: Transition to 'Active' state (Gesture detection ON)") 435 | else: 436 | self.r_gesture_start_time = None 437 | 438 | return None 439 | 440 | if self.state == "Active": 441 | self.r_gesture_start_time = None 442 | 443 | thumb_extended, index_extended, middle_extended, ring_extended, pinky_extended = self.get_extended_fingers(landmarks) 444 | 445 | self.extended_fingers = (thumb_extended, index_extended, middle_extended, ring_extended, pinky_extended) 446 | 447 | if not self.fist_made_in_active: 448 | if self.is_fist(): 449 | self.last_fist_time = current_time 450 | self.fist_made_in_active = True 451 | print(f"{self.hand_id}: Fist detected in 'Active' state") 452 | logging.debug(f"{self.hand_id}: Fist detected in 'Active' state") 453 | return 'Fist' 454 | return None 455 | 456 | if self.last_fist_time and (current_time - self.last_fist_time < self.delay_after_fist): 457 | return None 458 | 459 | if not self.is_fist(): 460 | if self.gesture_start_time is None: 461 | self.gesture_start_time = current_time 462 | elif current_time - self.gesture_start_time >= self.min_gesture_duration: 463 | gesture = self.recognize_gesture(landmarks, hand_label) 464 | if gesture: 465 | print(f"{self.hand_id}: Detected gesture '{gesture}'") 466 | logging.debug(f"{self.hand_id}: Detected gesture '{gesture}'") 467 | self.last_fist_time = None 468 | self.fist_made_in_active = False 469 | self.gesture_start_time = None 470 | return gesture 471 | # if self.extended_fingers == [False, True, True, False, False]: 472 | # detect_r = self.detect_r_gesture(landmarks, hand_label) 473 | # else: 474 | # gesture = self.recognize_gesture(landmarks, hand_label) 475 | # if gesture and not detect_r: 476 | # print(f"{self.hand_id}: Detected gesture '{gesture}'") 477 | # logging.debug(f"{self.hand_id}: Detected gesture '{gesture}'") 478 | # self.last_fist_time = None 479 | # self.fist_made_in_active = False 480 | # self.gesture_start_time = None 481 | # return gesture 482 | # if detect_r: 483 | # self.return_to_idle() 484 | else: 485 | self.gesture_start_time = None 486 | 487 | return None 488 | 489 | def is_fist(self): 490 | """ 491 | Determines if the hand is in a fist position. 492 | """ 493 | # Ensure base_distance is set before checking finger extensions 494 | if self.base_distance is None or self.extended_fingers is None: 495 | return False 496 | 497 | thumb_extended, index_extended, middle_extended, ring_extended, pinky_extended = self.extended_fingers 498 | return not (thumb_extended or index_extended or middle_extended or ring_extended or pinky_extended) 499 | 500 | def recognize_gesture(self, landmarks, hand_label): 501 | """ 502 | Recognizes gestures after the 'Fist' state. 503 | Uses the hand direction and finger extensions. 504 | """ 505 | # Ensure base_distance is set before recognizing gestures 506 | if self.base_distance is None: 507 | return None 508 | 509 | # Get booleans for each finger 510 | thumb_extended, index_extended, middle_extended, ring_extended, pinky_extended = self.extended_fingers 511 | mp_hands = mp.solutions.hands 512 | 513 | gesture = None 514 | direction = None 515 | 516 | # Create a list of the extended fingers 517 | extended_fingers = [thumb_extended, index_extended, middle_extended, ring_extended, pinky_extended] 518 | 519 | if extended_fingers == [True, False, False, False, False]: # Only thumb extended 520 | print("thumb") 521 | logging.debug("Only thumb extended") 522 | from_angle = landmarks.landmark[mp_hands.HandLandmark.WRIST] 523 | tip_point = landmarks.landmark[mp_hands.HandLandmark.THUMB_TIP] 524 | direction = self.get_direction(landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_MCP], tip_point, is_thumb=True) 525 | if direction == "Left": 526 | gesture = "Select" 527 | if hand_label == 'Left': 528 | gesture = "Back" 529 | elif direction == "Right": 530 | gesture = "Back" 531 | if hand_label == 'Left': 532 | gesture = "Select" 533 | elif direction == "Up": 534 | gesture = "VolumeUp" 535 | elif direction == "Down": 536 | gesture = "VolumeDown" 537 | 538 | elif extended_fingers[1:] == [True, False, False, False]: # Index finger only, ignoring thumb 539 | tip_point = landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP] 540 | direction = self.get_direction(landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_MCP], tip_point) 541 | gesture = direction 542 | 543 | elif extended_fingers[1:] == [True, True, False, False]: 544 | tip_point = landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP] 545 | direction = self.get_direction(landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_MCP], tip_point) 546 | if direction == "Left": 547 | gesture = "Rev" 548 | if direction == "Right": 549 | gesture = "Fwd" 550 | if direction == "Up": 551 | if self.detect_r_gesture(landmarks, hand_label): 552 | self.return_to_idle() 553 | gesture = "End Tracking" 554 | return gesture 555 | else: 556 | gesture = "Play" 557 | 558 | elif extended_fingers[1:] == [True, True, True, False]: 559 | tip_point = landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP] 560 | mcp_point = landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_MCP] 561 | direction = self.get_direction(mcp_point, tip_point) 562 | 563 | # if direction == "Up": 564 | # self.state = "Idle" 565 | # print(f"{self.hand_id}: Transition to 'Idle' state (Gesture detection OFF)") 566 | # logging.debug(f"{self.hand_id}: Transition to 'Idle' state (Gesture detection OFF)") 567 | # self.last_fist_time = None 568 | # self.fist_made_in_active = False 569 | # self.gesture_start_time = None 570 | # return None 571 | if direction == "Down": 572 | # Mute volume 573 | gesture = "VolumeMute" 574 | return gesture 575 | else: 576 | return None 577 | 578 | elif extended_fingers == [False, True, False, False, True]: # Index and pinky extended 579 | gesture = "Power" 580 | direction = None 581 | elif extended_fingers == [True, False, False, False, True]: # Thumb and pinky 582 | gesture = "Home" 583 | direction = None 584 | 585 | else: 586 | gesture = None 587 | self.current_direction = None 588 | return None 589 | 590 | if direction: 591 | self.current_direction = direction # Store direction for debug string 592 | else: 593 | self.current_direction = None 594 | 595 | return gesture 596 | 597 | def return_to_idle(self): 598 | self.state = "Idle" 599 | print(f"{self.hand_id}: Transition to 'Idle' state (Gesture detection OFF)") 600 | logging.debug(f"{self.hand_id}: Transition to 'Idle' state (Gesture detection OFF)") 601 | self.last_fist_time = None 602 | self.fist_made_in_active = False 603 | self.gesture_start_time = None 604 | # time.sleep(1) 605 | self.end_tracking_time = time.time() 606 | return None 607 | 608 | def get_direction(self, wrist_point, tip_point, is_thumb=False): 609 | """ 610 | Determines direction (Up, Down, Left, Right) based on tip point relative to wrist. 611 | Uses normal 90-degree quadrants for thumb gestures and adjusted thresholds for others. 612 | """ 613 | dx = tip_point.x - wrist_point.x 614 | dy = tip_point.y - wrist_point.y 615 | 616 | # Calculate angle in degrees and normalize it to [0, 360) 617 | angle = math.degrees(math.atan2(dy, dx)) # Calculate raw angle 618 | angle = (angle + 360) % 360 # Normalize angle to range [0, 360) 619 | 620 | print(f"Normalized angle: {angle}") 621 | # logging.debug(f"Normalized angle: {angle}") 622 | 623 | # Adjust angle based on hand label (if needed) 624 | if self.hand_label == 'Left': 625 | dx = -dx # Flip the x-axis for left hand 626 | angle = (math.degrees(math.atan2(dy, dx)) + 360) % 360 # Recalculate angle after flipping dx 627 | 628 | # Determine direction based on angle 629 | if is_thumb: 630 | # Use normal 90-degree quadrants for thumb 631 | if 45 <= angle < 135: 632 | return 'Down' 633 | elif 135 <= angle < 225: 634 | return 'Left' 635 | elif 225 <= angle < 315: 636 | return 'Up' 637 | else: 638 | # Covers angles from 315 to 360 and 0 to 45 639 | return 'Right' 640 | else: 641 | # Adjusted thresholds: 642 | # Up/Down reduced to 45 degrees each 643 | # Left/Right increased to 135 degrees each 644 | if 0 <= angle < 67.5 or 292.5 <= angle < 360: 645 | return 'Right' 646 | elif 67.5 <= angle < 112.5: 647 | return 'Down' 648 | elif 112.5 <= angle < 247.5: 649 | return 'Left' 650 | elif 247.5 <= angle < 292.5: 651 | return 'Up' 652 | else: 653 | return 'Unknown' 654 | 655 | 656 | -------------------------------------------------------------------------------- /images/icon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/icon.ico -------------------------------------------------------------------------------- /images/readme-images/HandiRokuRemoteGif-large.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/readme-images/HandiRokuRemoteGif-large.gif -------------------------------------------------------------------------------- /images/readme-images/HandiRokuRemoteGif.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/readme-images/HandiRokuRemoteGif.gif -------------------------------------------------------------------------------- /images/readme-images/HandiRokuRemoteGif2-large.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/readme-images/HandiRokuRemoteGif2-large.gif -------------------------------------------------------------------------------- /images/readme-images/HandiRokuRemoteGif2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/readme-images/HandiRokuRemoteGif2.gif -------------------------------------------------------------------------------- /images/readme-images/handiRokuRemote-GUI.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/readme-images/handiRokuRemote-GUI.jpg -------------------------------------------------------------------------------- /images/readme-images/handiRokuRemote-cheatsheet-small.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/readme-images/handiRokuRemote-cheatsheet-small.jpg -------------------------------------------------------------------------------- /images/readme-images/handiRokuRemote-cheatsheet.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/readme-images/handiRokuRemote-cheatsheet.jpg -------------------------------------------------------------------------------- /images/readme-images/tooltip-demo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/readme-images/tooltip-demo.jpg -------------------------------------------------------------------------------- /images/splash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/splash.png -------------------------------------------------------------------------------- /images/tooltip-images/fingers-crossed.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/fingers-crossed.jpg -------------------------------------------------------------------------------- /images/tooltip-images/fingers-deactivate.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/fingers-deactivate.jpg -------------------------------------------------------------------------------- /images/tooltip-images/fingers-mute.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/fingers-mute.jpg -------------------------------------------------------------------------------- /images/tooltip-images/fingersSplayed.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/fingersSplayed.jpg -------------------------------------------------------------------------------- /images/tooltip-images/fist.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/fist.jpg -------------------------------------------------------------------------------- /images/tooltip-images/home.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/home.jpg -------------------------------------------------------------------------------- /images/tooltip-images/horns.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/horns.jpg -------------------------------------------------------------------------------- /images/tooltip-images/i-down.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/i-down.jpg -------------------------------------------------------------------------------- /images/tooltip-images/i-left.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/i-left.jpg -------------------------------------------------------------------------------- /images/tooltip-images/i-m-left.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/i-m-left.jpg -------------------------------------------------------------------------------- /images/tooltip-images/i-m-right.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/i-m-right.jpg -------------------------------------------------------------------------------- /images/tooltip-images/i-m-up.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/i-m-up.jpg -------------------------------------------------------------------------------- /images/tooltip-images/i-right.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/i-right.jpg -------------------------------------------------------------------------------- /images/tooltip-images/i-up.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/i-up.jpg -------------------------------------------------------------------------------- /images/tooltip-images/palmup.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/palmup.jpg -------------------------------------------------------------------------------- /images/tooltip-images/thumbs-down.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/thumbs-down.jpg -------------------------------------------------------------------------------- /images/tooltip-images/thumbs-in.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/thumbs-in.jpg -------------------------------------------------------------------------------- /images/tooltip-images/thumbs-left.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/thumbs-left.jpg -------------------------------------------------------------------------------- /images/tooltip-images/thumbs-out.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/thumbs-out.jpg -------------------------------------------------------------------------------- /images/tooltip-images/thumbs-right.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/thumbs-right.jpg -------------------------------------------------------------------------------- /images/tooltip-images/thumbs-up.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BBelk/HandiRokuRemote/9c43d1b554d7cd149b6ec218da77ce9800f56d82/images/tooltip-images/thumbs-up.jpg -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==2.1.0 2 | altgraph==0.17.4 3 | attrs==24.2.0 4 | certifi==2024.8.30 5 | cffi==1.17.1 6 | charset-normalizer==3.4.0 7 | contourpy==1.1.1 8 | cv2_enumerate_cameras==1.1.17 9 | cycler==0.12.1 10 | flatbuffers==24.3.25 11 | fonttools==4.55.0 12 | getmac==0.9.5 13 | idna==3.10 14 | importlib_metadata==8.5.0 15 | importlib_resources==6.4.5 16 | jax==0.4.13 17 | kiwisolver==1.4.7 18 | matplotlib==3.7.5 19 | mediapipe==0.10.11 20 | ml-dtypes==0.2.0 21 | numpy==1.24.4 22 | opencv-contrib-python==4.10.0.84 23 | opencv-python==4.10.0.84 24 | opt_einsum==3.4.0 25 | packaging==24.2 26 | pefile==2023.2.7 27 | pillow==10.4.0 28 | protobuf==3.20.3 29 | pycparser==2.22 30 | pyinstaller==6.11.1 31 | pyinstaller-hooks-contrib==2024.10 32 | pyparsing==3.1.4 33 | python-dateutil==2.9.0.post0 34 | pywin32-ctypes==0.2.3 35 | requests==2.32.3 36 | scipy==1.10.1 37 | six==1.16.0 38 | sounddevice==0.5.1 39 | urllib3==2.2.3 40 | zipp==3.20.2 41 | -------------------------------------------------------------------------------- /rokuRemote.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import mediapipe as mp 3 | import tkinter as tk 4 | from tkinter import ttk 5 | import requests 6 | import threading 7 | import socket 8 | import time 9 | import gestureDetection 10 | import xml.etree.ElementTree as ET 11 | import json 12 | import os 13 | import sys 14 | from PIL import Image, ImageTk 15 | import concurrent.futures 16 | import webbrowser 17 | import logging 18 | from tkinter import filedialog 19 | from tooltip import Tooltip 20 | from cv2_enumerate_cameras import enumerate_cameras 21 | 22 | class VideoDisplayPanel(tk.Label): 23 | def __init__(self, parent): 24 | super().__init__(parent) 25 | self.image = None 26 | 27 | def update_image(self, frame): 28 | self.config(text='') 29 | display_width = 640 30 | display_height = 360 31 | frame_resized = cv2.resize(frame, (display_width, display_height)) 32 | 33 | frame_rgb = cv2.cvtColor(frame_resized, cv2.COLOR_BGR2RGB) 34 | img = ImageTk.PhotoImage(image=Image.fromarray(frame_rgb)) 35 | self.config(image=img) 36 | self.image = img 37 | 38 | def clear_image(self): 39 | self.config(image='', text='') 40 | self.image = None 41 | 42 | def show_load(self): 43 | self.config(image='', text="Loading...", font=("Helvetica", 24), anchor="center") 44 | self.image = None 45 | 46 | def show_idle(self): 47 | self.config(image='', text="No video feed active.", font=("Helvetica", 24), anchor="center") 48 | self.image = None 49 | 50 | class VideoFeed: 51 | def __init__(self, camera_index=0, frame_callback=None): 52 | self.camera_index = camera_index 53 | self.cap = None 54 | self.running = False 55 | self.frame_callback = frame_callback 56 | self.thread = None 57 | self.cap_lock = threading.Lock() 58 | 59 | def start(self): 60 | self.cap = cv2.VideoCapture(self.camera_index, cv2.CAP_MSMF) 61 | if not self.cap.isOpened(): 62 | print(f"Failed to open camera {self.camera_index}.") 63 | return False 64 | 65 | # Set camera resolution to 1920x1080 66 | self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1920) 67 | self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 1080) 68 | 69 | self.running = True 70 | self.thread = threading.Thread(target=self._update, daemon=True) 71 | self.thread.start() 72 | return True 73 | 74 | def stop(self): 75 | self.running = False 76 | if self.thread is not None: 77 | self.thread.join() # Wait for the thread to finish 78 | self.thread = None 79 | with self.cap_lock: 80 | if self.cap: 81 | self.cap.release() 82 | self.cap = None 83 | print("VideoFeed stopped and camera released.") 84 | if self.frame_callback: 85 | self.frame_callback = None 86 | 87 | def _update(self): 88 | while self.running: 89 | with self.cap_lock: 90 | if not self.cap or not self.cap.isOpened(): 91 | break 92 | try: 93 | ret, frame = self.cap.read() 94 | except Exception as e: 95 | print(f"Exception during cap.read(): {e}") 96 | break # Exit the loop if an exception occurs 97 | if not ret: 98 | print("Failed to capture a frame.") 99 | continue 100 | 101 | flipped_frame = cv2.flip(frame, 1) 102 | 103 | if self.frame_callback and self.running: 104 | self.frame_callback(flipped_frame) 105 | 106 | class RokuRemote: 107 | def __init__(self): 108 | self.roku_devices = [] 109 | self.selected_roku_device = None 110 | self.roku_ip = None 111 | self.roku_mac = None 112 | self.hand_threads = {} 113 | self.hand_threads_lock = threading.Lock() 114 | self.gesture_commands_lock = threading.Lock() 115 | self.debug_mode = True 116 | self.skeleton_view = True 117 | self.auto_start = False 118 | self.selected_camera_index = 0 119 | self.video_feed = None 120 | self.video_active = False 121 | self.gesture_state = None 122 | self.last_action = None 123 | 124 | # Initialize Mediapipe Hand Detector 125 | self.hands = mp.solutions.hands.Hands( 126 | min_detection_confidence=0.6, 127 | min_tracking_confidence=0.2, 128 | max_num_hands=1 129 | ) 130 | self.mp_hands_lock = threading.Lock() 131 | 132 | self.root = None 133 | self.left_frame = None 134 | self.right_frame = None 135 | self.video_display_panel = None 136 | self.control_buttons = [] 137 | 138 | # Settings and logging directories 139 | self.path_file_path = self.get_path_file_path() 140 | self.settings_directory = self.load_settings_directory() 141 | self.log_file_path = None 142 | self.config_file_path = None 143 | 144 | self.load_config() 145 | self.setup_logging() 146 | 147 | self.hand_absent_times = {} 148 | self.hand_absent_timeout = 0.5 149 | 150 | def get_app_data_dir(self): 151 | """Returns the path to the application's default data directory, platform-specific.""" 152 | app_name = 'HandiRokuRemote' 153 | 154 | if sys.platform.startswith('win'): 155 | # On Windows, use %APPDATA% 156 | appdata = os.getenv('APPDATA') 157 | if appdata: 158 | return os.path.join(appdata, app_name) 159 | else: 160 | # Fallback to user's home directory 161 | return os.path.join(os.path.expanduser('~'), app_name) 162 | elif sys.platform == 'darwin': 163 | # On macOS, use ~/Library/Application Support/ 164 | return os.path.join(os.path.expanduser('~/Library/Application Support'), app_name) 165 | else: 166 | # On Linux and other Unix-like systems, use ~/.local/share/ 167 | return os.path.join(os.path.expanduser('~/.local/share'), app_name) 168 | 169 | def get_path_file_path(self): 170 | """Returns the path to the 'roku_remote_path.txt' file.""" 171 | app_data_dir = self.get_app_data_dir() 172 | if not os.path.exists(app_data_dir): 173 | os.makedirs(app_data_dir) 174 | return os.path.join(app_data_dir, 'roku_remote_path.txt') 175 | 176 | def load_settings_directory(self): 177 | """Load the settings directory from 'roku_remote_path.txt' or set default.""" 178 | path_file = self.get_path_file_path() 179 | if os.path.exists(path_file): 180 | with open(path_file, 'r') as f: 181 | settings_dir = f.read().strip() 182 | if os.path.exists(settings_dir): 183 | return settings_dir 184 | # If the path file doesn't exist or the directory doesn't exist, use default 185 | default_settings_dir = self.get_app_data_dir() 186 | if not os.path.exists(default_settings_dir): 187 | os.makedirs(default_settings_dir) 188 | # Write the default path to the path file 189 | with open(path_file, 'w') as f: 190 | f.write(default_settings_dir) 191 | return default_settings_dir 192 | 193 | def save_settings_directory(self): 194 | """Save the settings directory to 'roku_remote_path.txt'.""" 195 | path_file = self.get_path_file_path() 196 | with open(path_file, 'w') as f: 197 | f.write(self.settings_directory) 198 | 199 | def setup_logging(self): 200 | """Set up logging based on the settings directory.""" 201 | if not os.path.exists(self.settings_directory): 202 | os.makedirs(self.settings_directory) 203 | 204 | self.log_file_path = os.path.join(self.settings_directory, 'roku_remote.log') 205 | 206 | # Remove all handlers associated with the root logger object 207 | for handler in logging.root.handlers[:]: 208 | logging.root.removeHandler(handler) 209 | 210 | logging.basicConfig( 211 | filename=self.log_file_path, 212 | level=logging.DEBUG if self.debug_mode else logging.INFO, 213 | format='%(asctime)s - %(levelname)s - %(message)s', 214 | filemode='a' 215 | ) 216 | logging.debug("Logging initialized.") 217 | 218 | def load_config(self): 219 | """Load the settings from a configuration file.""" 220 | # First, check if settings directory exists 221 | self.config_file_path = os.path.join(self.settings_directory, 'roku_config.json') 222 | 223 | if os.path.exists(self.config_file_path): 224 | with open(self.config_file_path, 'r') as f: 225 | data = json.load(f) 226 | self.roku_ip = data.get('roku_ip') 227 | self.roku_mac = data.get('roku_mac') 228 | self.selected_camera_index = data.get('selected_camera_index', 0) 229 | self.auto_start = data.get('auto_start', False) 230 | self.debug_mode = data.get('debug_mode', False) 231 | self.skeleton_view = data.get('skeleton_view', False) 232 | self.known_devices = data.get('known_devices', []) 233 | self.last_selected_device_ip = data.get('last_selected_device_ip') 234 | print(f"Loaded configuration from file: IP={self.roku_ip}, MAC={self.roku_mac}, " 235 | f"Camera Index={self.selected_camera_index}, Auto Start={self.auto_start}, " 236 | f"Settings Directory={self.settings_directory}") 237 | else: 238 | self.known_devices = [] 239 | self.last_selected_device_ip = None 240 | 241 | def save_config(self): 242 | """Save the settings to a configuration file.""" 243 | data = { 244 | 'roku_ip': self.roku_ip, 245 | 'roku_mac': self.roku_mac, 246 | 'selected_camera_index': self.selected_camera_index, 247 | 'auto_start': self.auto_start, 248 | 'debug_mode': self.debug_mode, 249 | 'skeleton_view': self.skeleton_view, 250 | 'known_devices': self.known_devices, 251 | 'last_selected_device_ip': self.selected_roku_device['ip'] if self.selected_roku_device else None 252 | } 253 | 254 | if not os.path.exists(self.settings_directory): 255 | os.makedirs(self.settings_directory) 256 | 257 | with open(self.config_file_path, 'w') as f: 258 | json.dump(data, f) 259 | print(f"Saved configuration to file: IP={self.roku_ip}, MAC={self.roku_mac}, " 260 | f"Camera Index={self.selected_camera_index}, Auto Start={self.auto_start}, " 261 | f"Settings Directory={self.settings_directory}") 262 | logging.debug(f"Configuration saved to {self.config_file_path}") 263 | 264 | def change_settings_directory(self): 265 | # Not used for now 266 | """Allows user to select a new settings directory.""" 267 | new_directory = filedialog.askdirectory(initialdir=self.settings_directory, title="Select Settings Directory") 268 | if new_directory: 269 | old_settings_directory = self.settings_directory 270 | self.settings_directory = new_directory 271 | 272 | # Move existing config and log files to new directory 273 | old_config_file = os.path.join(old_settings_directory, 'roku_config.json') 274 | old_log_file = os.path.join(old_settings_directory, 'roku_remote.log') 275 | 276 | new_config_file = os.path.join(self.settings_directory, 'roku_config.json') 277 | new_log_file = os.path.join(self.settings_directory, 'roku_remote.log') 278 | 279 | if not os.path.exists(self.settings_directory): 280 | os.makedirs(self.settings_directory) 281 | 282 | if os.path.exists(old_config_file): 283 | os.rename(old_config_file, new_config_file) 284 | logging.debug(f"Moved config file from {old_config_file} to {new_config_file}") 285 | if os.path.exists(old_log_file): 286 | os.rename(old_log_file, new_log_file) 287 | logging.debug(f"Moved log file from {old_log_file} to {new_log_file}") 288 | 289 | self.config_file_path = new_config_file 290 | self.save_settings_directory() # Update the path file 291 | self.setup_logging() # Re-setup logging with new directory 292 | self.settings_directory_entry.delete(0, tk.END) 293 | self.settings_directory_entry.insert(0, self.settings_directory) 294 | print(f"Settings directory changed to: {self.settings_directory}") 295 | logging.debug(f"Settings directory changed to: {self.settings_directory}") 296 | self.save_config() # Save the new settings 297 | 298 | def attempt_connect_known_devices(self): 299 | """Attempt to connect to the last selected device or known devices from the config.""" 300 | self.roku_devices = [] 301 | if hasattr(self, 'last_selected_device_ip') and self.last_selected_device_ip: 302 | print(f"Attempting to connect to the last selected device at {self.last_selected_device_ip}...") 303 | logging.debug(f"Attempting to connect to last selected device at {self.last_selected_device_ip}") 304 | device_info = self.get_roku_device_info(self.last_selected_device_ip) 305 | if device_info: 306 | self.selected_roku_device = device_info 307 | self.roku_ip = device_info['ip'] 308 | self.roku_mac = device_info.get('wifi_mac') or device_info.get('ethernet_mac') 309 | self.add_to_known_devices(device_info) 310 | self.roku_devices.append(device_info) 311 | print(f"Connected to last selected Roku device at {self.roku_ip}") 312 | logging.debug(f"Connected to last selected Roku device at {self.roku_ip}") 313 | self.save_config() 314 | else: 315 | print(f"Last selected device at {self.last_selected_device_ip} is not available.") 316 | logging.debug(f"Last selected device at {self.last_selected_device_ip} is not available.") 317 | 318 | # Attempt to connect to known devices 319 | if self.known_devices: 320 | print("Attempting to connect to known devices...") 321 | logging.debug("Attempting to connect to known devices...") 322 | for device_info in self.known_devices: 323 | ip = device_info['ip'] 324 | device_info = self.get_roku_device_info(ip) 325 | if device_info: 326 | if device_info not in self.roku_devices: 327 | self.roku_devices.append(device_info) 328 | print(f"Found known Roku device at {ip}") 329 | logging.debug(f"Found known Roku device at {ip}") 330 | if self.roku_devices: 331 | if not self.selected_roku_device: 332 | self.selected_roku_device = self.roku_devices[0] 333 | self.roku_ip = self.selected_roku_device['ip'] 334 | self.roku_mac = self.selected_roku_device.get('wifi_mac') or self.selected_roku_device.get('ethernet_mac') 335 | self.save_config() 336 | else: 337 | print("Known devices not found, starting network scan.") 338 | logging.debug("Known devices not found, starting network scan.") 339 | self.refresh_device_list() 340 | else: 341 | self.refresh_device_list() 342 | 343 | self.update_device_combo() 344 | 345 | def discover_roku_devices(self, subnet='192.168.1', start=1, end=254): 346 | """Scan the local network for all Roku devices.""" 347 | print(f"Scanning network {subnet}.0/{end - start + 1} for Roku devices...") 348 | logging.debug(f"Scanning network {subnet}.0/{end - start + 1} for Roku devices...") 349 | devices = [] 350 | ips = [f'{subnet}.{i}' for i in range(start, end + 1)] 351 | 352 | def scan_ip(ip): 353 | url = f'http://{ip}:8060/query/device-info' 354 | try: 355 | response = requests.get(url, timeout=0.25) 356 | if response.status_code == 200 and 'roku' in response.text.lower(): 357 | device_info = self.get_roku_device_info(ip) 358 | if device_info: 359 | device_name = device_info['user_device_name'] or device_info['friendly_device_name'] or device_info['default_device_name'] 360 | print(f"Found Roku device at {ip} with name '{device_name}'") 361 | logging.debug(f"Found Roku device at {ip} with name '{device_name}'") 362 | return device_info 363 | except requests.exceptions.RequestException as e: 364 | pass 365 | return None 366 | 367 | with concurrent.futures.ThreadPoolExecutor(max_workers=50) as executor: 368 | futures = [executor.submit(scan_ip, ip) for ip in ips] 369 | for future in concurrent.futures.as_completed(futures): 370 | device_info = future.result() 371 | if device_info: 372 | devices.append(device_info) 373 | if not devices: 374 | print("Failed to find Roku devices via network scan.") 375 | logging.debug("Failed to find Roku devices via network scan.") 376 | return devices 377 | 378 | def get_roku_device_info(self, ip_address): 379 | """Retrieve device info from the Roku device.""" 380 | url = f'http://{ip_address}:8060/query/device-info' 381 | try: 382 | response = requests.get(url, timeout=0.5) 383 | if response.status_code == 200: 384 | root = ET.fromstring(response.content) 385 | device_info = { 386 | 'ip': ip_address, 387 | 'serial_number': root.findtext('serial-number'), 388 | 'device_id': root.findtext('device-id'), 389 | 'vendor_name': root.findtext('vendor-name'), 390 | 'model_name': root.findtext('model-name'), 391 | 'friendly_device_name': root.findtext('friendly-device-name'), 392 | 'friendly_model_name': root.findtext('friendly-model-name'), 393 | 'default_device_name': root.findtext('default-device-name'), 394 | 'user_device_name': root.findtext('user-device-name'), 395 | 'user_device_location': root.findtext('user-device-location'), 396 | 'wifi_mac': root.findtext('wifi-mac'), 397 | 'ethernet_mac': root.findtext('ethernet-mac'), 398 | } 399 | return device_info 400 | except requests.exceptions.RequestException: 401 | pass 402 | return None 403 | 404 | def wake_on_lan(self): 405 | """Send a Wake-on-LAN magic packet to the Roku device.""" 406 | if not self.roku_mac: 407 | print("MAC address not known. Cannot send Wake-on-LAN packet.") 408 | logging.debug("MAC address not known. Cannot send Wake-on-LAN packet.") 409 | return 410 | mac_address = self.roku_mac.replace(':', '').replace('-', '').replace('.', '') 411 | if len(mac_address) != 12: 412 | print("Invalid MAC address format.") 413 | logging.debug("Invalid MAC address format.") 414 | return 415 | magic_packet = bytes.fromhex('FF' * 6 + mac_address * 16) 416 | try: 417 | sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 418 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) 419 | sock.sendto(magic_packet, ('', 9)) 420 | sock.close() 421 | print("Sent Wake-on-LAN packet.") 422 | logging.debug("Sent Wake-on-LAN packet.") 423 | except Exception as e: 424 | print(f"Failed to send Wake-on-LAN packet: {e}") 425 | logging.debug(f"Failed to send Wake-on-LAN packet: {e}") 426 | 427 | def send_command(self, command): 428 | """ 429 | Send a command to the Roku device via HTTP POST request. 430 | """ 431 | if not self.roku_ip: 432 | print("No Roku device IP address known.") 433 | logging.debug("No Roku device IP address known.") 434 | if self.roku_mac: 435 | print("Attempting to wake up Roku device via Wake-on-LAN...") 436 | logging.debug("Attempting to wake up Roku device via Wake-on-LAN...") 437 | self.wake_on_lan() 438 | time.sleep(1) 439 | self.roku_devices = self.discover_roku_devices() 440 | for device in self.roku_devices: 441 | device_mac = device.get('wifi_mac') or device.get('ethernet_mac') 442 | if device_mac and device_mac.lower() == self.roku_mac.lower(): 443 | self.selected_roku_device = device 444 | self.roku_ip = device['ip'] 445 | print(f"Reconnected to Roku device at {self.roku_ip}") 446 | logging.debug(f"Reconnected to Roku device at {self.roku_ip}") 447 | self.save_config() 448 | break 449 | if not self.roku_ip: 450 | print("Unable to discover Roku device after Wake-on-LAN.") 451 | logging.debug("Unable to discover Roku device after Wake-on-LAN.") 452 | return 453 | else: 454 | print("Cannot wake up Roku device because MAC address is unknown.") 455 | logging.debug("Cannot wake up Roku device because MAC address is unknown.") 456 | return 457 | 458 | url = f"http://{self.roku_ip}:8060/keypress/{command}" 459 | try: 460 | response = requests.post(url, timeout=0.25) 461 | if response.status_code == 200: 462 | print(f"Sent command '{command}' to Roku at {self.roku_ip}") 463 | logging.debug(f"Sent command '{command}' to Roku at {self.roku_ip}") 464 | elif response.status_code == 202: 465 | print(f"Command '{command}' accepted by Roku at {self.roku_ip}. Waiting for transition...") 466 | logging.debug(f"Command '{command}' accepted by Roku at {self.roku_ip}. Waiting for transition...") 467 | time.sleep(1) # Allow Roku to complete power transition 468 | elif response.status_code == 403: 469 | print(f"Command '{command}' is not supported on this device.") 470 | logging.debug(f"Command '{command}' is not supported on this device.") 471 | else: 472 | print(f"Failed to send command '{command}' to Roku at {self.roku_ip}, status code: {response.status_code}") 473 | logging.debug(f"Failed to send command '{command}' to Roku at {self.roku_ip}, status code: {response.status_code}") 474 | except requests.exceptions.RequestException as e: 475 | print(f"Error sending command '{command}' to Roku at {self.roku_ip}: {e}") 476 | logging.debug(f"Error sending command '{command}' to Roku at {self.roku_ip}: {e}") 477 | 478 | def start_video_feed(self): 479 | """Starts the video feed using the selected camera index.""" 480 | if self.video_feed: 481 | self.stop_video_feed() 482 | 483 | self.video_display_panel.show_load() 484 | self.root.update_idletasks() 485 | threading.Thread(target=self.initialize_video_feed, daemon=True).start() 486 | 487 | def initialize_video_feed(self): 488 | """Initializes the video feed after the GUI has updated.""" 489 | try: 490 | camera_selection = self.camera_combo.get() 491 | camera_index = int(camera_selection.split("(")[-1].rstrip(")")) 492 | print(f"Attempting to start video feed with camera index {camera_index}") 493 | logging.debug(f"Attempting to start video feed with camera index {camera_index}") 494 | self.video_feed = VideoFeed(camera_index=camera_index, frame_callback=self.process_frame) 495 | success = self.video_feed.start() 496 | if not success: 497 | raise Exception("Failed to start video feed.") 498 | print("Video feed started.") 499 | logging.debug("Video feed started.") 500 | self.video_active = True 501 | self.root.after(0, lambda: self.start_button.config(text="Stop Video Feed", command=self.stop_video_feed)) 502 | self.root.after(0, lambda: self.camera_combo.config(state='disabled')) 503 | self.root.after(0, self.update_start_button_state) 504 | self.root.after(0, self.root.geometry, "") 505 | except Exception as e: 506 | print(f"Error starting video feed: {e}") 507 | logging.debug(f"Error starting video feed: {e}") 508 | self.video_active = False 509 | self.video_feed = None 510 | self.root.after(0, self.video_display_panel.show_idle) 511 | self.root.after(0, lambda: self.start_button.config(text="Start Video Feed", command=self.start_video_feed)) 512 | self.root.after(0, lambda: self.camera_combo.config(state='normal')) 513 | self.root.after(0, self.update_start_button_state()) 514 | 515 | def stop_video_feed(self): 516 | """Stops the video feed.""" 517 | if self.video_active: 518 | self.gesture_state = None 519 | self.last_action = None 520 | self.video_active = False 521 | if self.video_feed: 522 | try: 523 | self.video_feed.stop() 524 | self.video_feed = None 525 | print("Camera feed stopped.") 526 | logging.debug("Camera feed stopped.") 527 | except Exception as e: 528 | print(f"Error stopping video feed: {e}") 529 | logging.debug(f"Error stopping video feed: {e}") 530 | 531 | # Stop all HandClassifier threads 532 | threads_to_join = [] 533 | with self.hand_threads_lock: 534 | for hand_thread in self.hand_threads.values(): 535 | hand_thread.stop() 536 | threads_to_join.append(hand_thread) 537 | self.hand_threads.clear() 538 | self.hand_absent_times.clear() 539 | # Now, outside the lock, join the threads 540 | for hand_thread in threads_to_join: 541 | hand_thread.join(timeout=1) # Wait for the thread to finish 542 | 543 | print("All HandClassifier threads stopped.") 544 | logging.debug("All HandClassifier threads stopped.") 545 | 546 | self.video_display_panel.show_idle() 547 | self.start_button.config(text="Start Video Feed", command=self.start_video_feed) 548 | self.camera_combo.config(state='normal') 549 | self.update_start_button_state() 550 | self.root.after(0, self.root.geometry, "") 551 | 552 | def process_frame(self, frame): 553 | """Handles incoming frames and schedules GUI updates in the main thread.""" 554 | if not self.video_active: 555 | return 556 | if frame is None: 557 | print("Received empty frame.") 558 | logging.debug("Received empty frame.") 559 | return 560 | self.root.after_idle(self._process_frame_in_main_thread, frame) 561 | 562 | def _process_frame_in_main_thread(self, frame): 563 | """Processes the frame and updates the GUI in the main thread.""" 564 | if not self.video_active: 565 | return 566 | 567 | frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) 568 | image_height, image_width, _ = frame.shape 569 | with self.mp_hands_lock: 570 | result = self.hands.process(frame_rgb) 571 | hand_landmarks_list = result.multi_hand_landmarks 572 | handedness_list = result.multi_handedness 573 | 574 | current_hand_ids = set() 575 | debug_strings = [] 576 | 577 | if hand_landmarks_list and handedness_list: 578 | for idx, (hand_landmarks, handedness) in enumerate(zip(hand_landmarks_list, handedness_list)): 579 | hand_label = handedness.classification[0].label # 'Left' or 'Right' 580 | current_hand_ids.add(hand_label) 581 | 582 | # Draw hand landmarks on the frame 583 | if self.skeleton_view: 584 | mp.solutions.drawing_utils.draw_landmarks( 585 | frame, hand_landmarks, mp.solutions.hands.HAND_CONNECTIONS) 586 | 587 | # Update or create a thread for this hand 588 | with self.hand_threads_lock: 589 | if hand_label in self.hand_threads: 590 | # Update landmarks 591 | self.hand_threads[hand_label].update_landmarks(hand_landmarks, hand_label) 592 | # Reset absent time since hand is detected 593 | self.hand_absent_times[hand_label] = None 594 | else: 595 | # Start new HandClassifier thread 596 | hand_thread = gestureDetection.HandClassifier( 597 | hand_id=hand_label, 598 | initial_landmarks=hand_landmarks, 599 | command_callback=self.gesture_command_callback, 600 | hand_label=hand_label 601 | ) 602 | hand_thread.start() 603 | self.hand_threads[hand_label] = hand_thread 604 | self.hand_absent_times[hand_label] = None # Initialize absent time 605 | 606 | # If debug mode is on, collect debug information 607 | if self.debug_mode: 608 | with self.hand_threads_lock: 609 | hand_thread = self.hand_threads.get(hand_label) 610 | if hand_thread: 611 | debug_string = hand_thread.debug_string 612 | debug_strings.append(debug_string) 613 | 614 | # Overlay the debug information on the frame 615 | x, y = 10, 30 + idx * 80 616 | for i, line in enumerate(debug_string.split('\n')): 617 | cv2.putText(frame, line, (x, y + i * 20), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 0, 0), 2) 618 | 619 | # Draw the red line between thumb point and selected other point 620 | if hand_thread.line_points: 621 | thumb_id, other_id = hand_thread.line_points 622 | thumb_landmark = hand_landmarks.landmark[thumb_id] 623 | other_landmark = hand_landmarks.landmark[other_id] 624 | 625 | thumb_x = int(thumb_landmark.x * image_width) 626 | thumb_y = int(thumb_landmark.y * image_height) 627 | other_x = int(other_landmark.x * image_width) 628 | other_y = int(other_landmark.y * image_height) 629 | 630 | # Draw red line 631 | cv2.line(frame, (thumb_x, thumb_y), (other_x, other_y), (0, 0, 255), 2) 632 | 633 | else: 634 | # If no hands are detected, set gesture state to None 635 | self.gesture_state = None 636 | self.last_action = None 637 | 638 | # Display gesture state in the top-right corner 639 | gesture_text = self.gesture_state if self.gesture_state else "Idle" 640 | font_scale = 1.8 # Scale for gesture state text 641 | thickness = 4 # Thickness for better visibility 642 | 643 | # Get text size for gesture state 644 | text_size = cv2.getTextSize(gesture_text, cv2.FONT_HERSHEY_SIMPLEX, font_scale, thickness)[0] 645 | text_x = max(10, image_width - text_size[0] - 10) # Align gesture state to the right 646 | text_y = min(image_height - 10, text_size[1] + 30) # Top padding 647 | 648 | # Render the gesture state text 649 | cv2.putText(frame, gesture_text, (text_x, text_y), cv2.FONT_HERSHEY_SIMPLEX, font_scale, (0, 255, 0), thickness) 650 | 651 | # Display last action below the gesture state 652 | if self.last_action: 653 | last_action_text = f"{self.last_action}" 654 | last_action_font_scale = 1.2 # Twice as large as before 655 | last_action_thickness = 3 # Thicker for better visibility 656 | 657 | # Get text size for last action 658 | last_action_text_size = cv2.getTextSize(last_action_text, cv2.FONT_HERSHEY_SIMPLEX, last_action_font_scale, last_action_thickness)[0] 659 | # Align last action text to the right, below the gesture state text 660 | last_action_text_x = max(10, image_width - last_action_text_size[0] - 10) 661 | last_action_text_y = text_y + text_size[1] + 20 # Add padding below the gesture text 662 | 663 | # Render the last action text in green, aligned to the right 664 | cv2.putText(frame, last_action_text, (last_action_text_x, last_action_text_y), cv2.FONT_HERSHEY_SIMPLEX, last_action_font_scale, (0, 255, 0), last_action_thickness) 665 | 666 | 667 | # Remove threads for hands that are no longer detected, after a grace period 668 | with self.hand_threads_lock: 669 | all_hand_ids = set(self.hand_threads.keys()) 670 | lost_hands = all_hand_ids - current_hand_ids 671 | for hand_id in lost_hands: 672 | if self.hand_absent_times.get(hand_id) is None: 673 | self.hand_absent_times[hand_id] = time.time() 674 | else: 675 | elapsed_time = time.time() - self.hand_absent_times[hand_id] 676 | if elapsed_time > self.hand_absent_timeout: 677 | self.gesture_state = None 678 | self.last_action = None 679 | self.hand_threads[hand_id].stop() 680 | del self.hand_threads[hand_id] 681 | del self.hand_absent_times[hand_id] 682 | 683 | for hand_id in current_hand_ids: 684 | if hand_id in self.hand_absent_times: 685 | self.hand_absent_times[hand_id] = None 686 | 687 | if self.video_display_panel: 688 | self.video_display_panel.update_image(frame) 689 | 690 | def gesture_command_callback(self, gesture, debug_string, direction, gesture_state): 691 | if not self.video_active: 692 | return 693 | self.root.after(0, self.handle_gesture, gesture, gesture_state) 694 | 695 | def handle_gesture(self, gesture, gesture_state): 696 | if not self.video_active: 697 | return # Do not handle gestures if video feed is stopped 698 | try: 699 | with self.mp_hands_lock: 700 | self.gesture_state = gesture_state 701 | 702 | # Map gestures to Roku commands 703 | gesture_to_command = { 704 | 'Play': 'Play', 705 | 'Select': 'Select', 706 | 'Up': 'Up', 707 | 'Down': 'Down', 708 | 'Left': 'Left', 709 | 'Right': 'Right', 710 | "Power": "Power", 711 | "VolumeUp": "VolumeUp", 712 | "VolumeDown": "VolumeDown", 713 | "Back": "Back", 714 | "Home": "Home", 715 | "VolumeMute": "VolumeMute", 716 | "Rev": 'Rev', 717 | "Fwd": 'Fwd', 718 | } 719 | if gesture in gesture_to_command: 720 | command = gesture_to_command[gesture] 721 | # Start a new thread to send the command 722 | threading.Thread(target=self.send_command, args=(command,), daemon=True).start() 723 | self.last_action = command 724 | logging.debug(f"Gesture '{gesture}' recognized, command '{command}' sent.") 725 | if gesture == 'Fist': 726 | if self.last_action != 'Fist': 727 | self.last_action = 'Fist' 728 | if gesture == 'End Tracking': 729 | if self.last_action != 'End Tracking': 730 | self.last_action = 'End Tracking' 731 | if self.gesture_state == "Active" and self.last_action == "End Tracking": 732 | self.last_action = None 733 | 734 | except Exception as e: 735 | print(f"Error in handle_gesture: {e}") 736 | logging.debug(f"Error in handle_gesture: {e}") 737 | 738 | def toggle_debug_mode(self): 739 | """Toggle the debug mode.""" 740 | self.debug_mode = self.debug_var.get() 741 | self.setup_logging() # Re-setup logging with new debug mode 742 | self.save_config() 743 | print(f"Debug mode set to {self.debug_mode}") 744 | logging.debug(f"Debug mode set to {self.debug_mode}") 745 | 746 | def toggle_skeleton_view(self): 747 | """Toggle the debug mode.""" 748 | self.skeleton_view = self.skeleton_var.get() 749 | self.setup_logging() # Re-setup logging with new debug mode 750 | self.save_config() 751 | print(f"Skeleton view set to {self.skeleton_view}") 752 | logging.debug(f"Skeleton view set to {self.skeleton_view}") 753 | 754 | def toggle_auto_start(self): 755 | """Toggle the auto start setting.""" 756 | self.auto_start = self.auto_start_var.get() 757 | self.save_config() 758 | print(f"Auto start set to {self.auto_start}") 759 | logging.debug(f"Auto start set to {self.auto_start}") 760 | 761 | def on_camera_selected(self, event): 762 | """Update selected camera index when camera selection changes.""" 763 | selected_camera = self.camera_combo.get() 764 | if "(" in selected_camera and selected_camera.endswith(")"): 765 | self.selected_camera_index = int(selected_camera.split("(")[-1].rstrip(")")) 766 | self.save_config() 767 | print(f"Selected camera index set to {self.selected_camera_index}") 768 | logging.debug(f"Selected camera index set to {self.selected_camera_index}") 769 | self.update_start_button_state() 770 | 771 | def on_device_selected(self, event): 772 | """Handle device selection from dropdown.""" 773 | selected_device_name = self.device_combo.get() 774 | for device in self.roku_devices: 775 | device_name = device['user_device_name'] or device['friendly_device_name'] or device['default_device_name'] 776 | device_location = device.get('user_device_location', '') 777 | if device_location: 778 | device_name += f" - {device_location}" 779 | display_name = f"{device_name} ({device['ip']})" 780 | if display_name == selected_device_name: 781 | self.selected_roku_device = device 782 | self.roku_ip = device['ip'] 783 | self.roku_mac = device.get('wifi_mac') or device.get('ethernet_mac') 784 | self.add_to_known_devices(device) 785 | print(f"Selected Roku device: {device_name} at {self.roku_ip}") 786 | logging.debug(f"Selected Roku device: {device_name} at {self.roku_ip}") 787 | self.save_config() 788 | break 789 | self.update_control_buttons_state() 790 | self.update_start_button_state() 791 | 792 | def add_to_known_devices(self, device): 793 | """Add or update a device in the known devices list.""" 794 | for known_device in self.known_devices: 795 | if known_device['ip'] == device['ip']: 796 | known_device.update(device) 797 | break 798 | else: 799 | self.known_devices.append(device) 800 | self.save_config() 801 | 802 | def on_connect_manual_ip(self): 803 | """Handle manual IP input and connect button.""" 804 | manual_ip = self.manual_ip_entry.get().strip() 805 | if manual_ip: 806 | device_info = self.get_roku_device_info(manual_ip) 807 | if device_info: 808 | self.selected_roku_device = device_info 809 | self.roku_ip = device_info['ip'] 810 | self.roku_mac = device_info.get('wifi_mac') or device_info.get('ethernet_mac') 811 | self.add_to_known_devices(device_info) 812 | self.roku_devices.append(device_info) 813 | print(f"Connected to Roku device: {device_info['user_device_name']} at {self.roku_ip}") 814 | logging.debug(f"Connected to Roku device: {device_info['user_device_name']} at {self.roku_ip}") 815 | self.update_device_combo() 816 | self.manual_ip_entry.delete(0, tk.END) 817 | else: 818 | print(f"Failed to connect to Roku at {manual_ip}.") 819 | logging.debug(f"Failed to connect to Roku at {manual_ip}.") 820 | else: 821 | print("Please enter a valid IP address.") 822 | logging.debug("No IP address entered in manual IP field.") 823 | 824 | def refresh_device_list(self): 825 | """Refresh the list of Roku devices.""" 826 | def _discover_and_update(): 827 | self.roku_devices = self.discover_roku_devices() 828 | for device in self.roku_devices: 829 | self.add_to_known_devices(device) 830 | self.root.after(0, self.update_device_combo) 831 | threading.Thread(target=_discover_and_update).start() 832 | self.device_combo['values'] = ["Loading devices..."] 833 | self.device_combo.set("Loading devices...") 834 | logging.debug("Started thread to refresh device list.") 835 | 836 | def update_device_combo(self): 837 | device_names = [] 838 | for device in self.roku_devices: 839 | device_name = device['user_device_name'] or device['friendly_device_name'] or device['default_device_name'] 840 | device_location = device.get('user_device_location', '') 841 | if device_location: 842 | device_name += f" - {device_location}" 843 | display_name = f"{device_name} ({device['ip']})" 844 | device_names.append(display_name) 845 | if device_names: 846 | self.device_combo['values'] = device_names 847 | self.device_combo.config(width=max(len(name) for name in device_names)) 848 | if self.selected_roku_device: 849 | selected_name = self.selected_roku_device['user_device_name'] or self.selected_roku_device['friendly_device_name'] or self.selected_roku_device['default_device_name'] 850 | selected_location = self.selected_roku_device.get('user_device_location', '') 851 | if selected_location: 852 | selected_name += f" - {selected_location}" 853 | selected_display_name = f"{selected_name} ({self.selected_roku_device['ip']})" 854 | if selected_display_name in device_names: 855 | self.device_combo.set(selected_display_name) 856 | else: 857 | self.device_combo.current(0) 858 | self.on_device_selected(None) 859 | else: 860 | self.device_combo.current(0) 861 | self.on_device_selected(None) 862 | else: 863 | self.device_combo['values'] = [] 864 | self.device_combo.set("No devices found") 865 | logging.debug("No devices found after updating device combo.") 866 | self.update_control_buttons_state() 867 | self.update_start_button_state() 868 | 869 | def refresh_camera_list(self): 870 | """Refreshes the camera list by detecting available cameras using enumerate_cameras.""" 871 | print("Refreshing camera list") 872 | logging.debug("Refreshing camera list") 873 | available_cameras = [] 874 | try: 875 | for camera_info in enumerate_cameras(cv2.CAP_MSMF): 876 | available_cameras.append(f"{camera_info.name} ({camera_info.index})") 877 | 878 | if available_cameras: 879 | self.camera_combo['values'] = available_cameras 880 | for idx, cam in enumerate(available_cameras): 881 | if f"({self.selected_camera_index})" in cam: 882 | self.camera_combo.current(idx) 883 | break 884 | else: 885 | self.camera_combo.current(0) 886 | self.on_camera_selected(None) 887 | else: 888 | self.camera_combo['values'] = ["No cameras found"] 889 | self.camera_combo.set("No cameras found") 890 | logging.debug("No cameras found during camera refresh.") 891 | except Exception as e: 892 | print(f"Error while refreshing camera list: {e}") 893 | logging.debug(f"Error while refreshing camera list: {e}") 894 | self.camera_combo['values'] = ["Error detecting cameras"] 895 | self.camera_combo.set("Error detecting cameras") 896 | self.update_start_button_state() 897 | 898 | def update_start_button_state(self): 899 | """Enable or disable the start video feed button based on selections.""" 900 | if hasattr(self, 'start_button'): # Ensure the button is created 901 | if self.video_active: 902 | self.start_button.config(state='normal') 903 | elif self.device_combo.get() and self.camera_combo.get(): 904 | self.start_button.config(state='normal') 905 | else: 906 | self.start_button.config(state='disabled') 907 | 908 | def update_control_buttons_state(self): 909 | """Enable or disable control buttons based on device selection.""" 910 | if hasattr(self, 'control_buttons'): # Ensure control buttons are created 911 | if self.selected_roku_device: 912 | for btn in self.control_buttons: 913 | btn.config(state='normal') 914 | else: 915 | for btn in self.control_buttons: 916 | btn.config(state='disabled') 917 | 918 | def setup_gui(self): 919 | """Setup the GUI for starting and stopping the video feed and controlling Roku.""" 920 | self.root = tk.Tk() 921 | self.root.title("HandiRokuRemote") 922 | self.root.geometry("800x720") 923 | self.root.resizable(True, True) 924 | 925 | base_dir = os.path.dirname(os.path.abspath(__file__)) 926 | icon_path = os.path.join(base_dir, "images", "icon.ico") 927 | if os.path.exists(icon_path): 928 | self.root.iconbitmap(icon_path) 929 | 930 | # Main frame to hold left and right frames 931 | main_frame = tk.Frame(self.root) 932 | main_frame.pack(fill='both', expand=True) 933 | 934 | # Left frame for controls 935 | self.left_frame = tk.Frame(main_frame, width=200) 936 | self.left_frame.pack(side='left', fill='y', padx=10, pady=10) 937 | 938 | # Right frame for video feed 939 | self.right_frame = tk.Frame(main_frame) 940 | self.right_frame.pack(side='right', fill='both', expand=True) 941 | 942 | # Start/Stop Video Button 943 | self.start_button = tk.Button(self.left_frame, text="Start Video Feed", command=self.start_video_feed, width=20, height=2) 944 | self.start_button.pack(pady=10) 945 | self.start_button.config(state='disabled') # Initially disabled 946 | 947 | # Auto Start and Debug Mode Checkboxes on the same row 948 | checkbox_frame = tk.Frame(self.left_frame) 949 | checkbox_frame.pack(pady=5) 950 | 951 | # Instructions Button 952 | instructions_button = tk.Button( 953 | checkbox_frame, text="Instructions", command=lambda: webbrowser.open("https://github.com/BBelk/HandiRokuRemote?tab=readme-ov-file#installation-and-interface-overview") 954 | ) 955 | instructions_button.grid(row=0, column=0, padx=10) 956 | 957 | # Auto Start Checkbox 958 | self.auto_start_var = tk.BooleanVar(value=self.auto_start) 959 | auto_start_checkbox = tk.Checkbutton( 960 | checkbox_frame, text="Auto Start", variable=self.auto_start_var, command=self.toggle_auto_start) 961 | auto_start_checkbox.grid(row=1, column=0, padx=10) 962 | 963 | # Debug Mode Checkbox 964 | self.debug_var = tk.BooleanVar(value=self.debug_mode) 965 | debug_checkbox = tk.Checkbutton( 966 | checkbox_frame, text="Debug Mode", variable=self.debug_var, command=self.toggle_debug_mode) 967 | debug_checkbox.grid(row=0, column=1, padx=10) 968 | 969 | # Skeleton View Checkbox 970 | self.skeleton_var = tk.BooleanVar(value=self.skeleton_view) 971 | skeleton_checkbox = tk.Checkbutton( 972 | checkbox_frame, text="Skeleton View", variable=self.skeleton_var, command=self.toggle_skeleton_view) 973 | skeleton_checkbox.grid(row=1, column=1, padx=10) 974 | 975 | 976 | # Device Selection Frame 977 | device_frame = tk.LabelFrame(self.left_frame, text="Device Selection") 978 | device_frame.pack(fill='x', pady=2) 979 | 980 | # Device dropdown and manual IP entry 981 | self.device_combo = ttk.Combobox(device_frame, state="readonly", width=40) # Set width for device dropdown 982 | self.device_combo.pack(pady=5) 983 | self.device_combo.bind("<>", self.on_device_selected) 984 | 985 | # Manual IP Entry 986 | manual_frame = tk.Frame(device_frame) 987 | manual_frame.pack(fill='x', pady=2) 988 | manual_ip_label = tk.Label(manual_frame, text="Manual IP:") 989 | manual_ip_label.pack(side='left', padx=5) 990 | self.manual_ip_entry = tk.Entry(manual_frame) 991 | self.manual_ip_entry.pack(side='left', fill='x', expand=True, padx=5) 992 | connect_button = tk.Button(manual_frame, text="Connect", command=self.on_connect_manual_ip) 993 | connect_button.pack(side='left', padx=5) 994 | 995 | # Refresh Devices Button 996 | refresh_devices_button = tk.Button(device_frame, text="Refresh Devices", command=self.refresh_device_list) 997 | refresh_devices_button.pack(pady=2) 998 | 999 | # Camera Selection Frame 1000 | camera_frame = tk.LabelFrame(self.left_frame, text="Camera Selection") 1001 | camera_frame.pack(fill='x', pady=2) 1002 | 1003 | # Camera selection Combobox 1004 | self.camera_combo = ttk.Combobox(camera_frame, state="readonly", width=40) # Match width to device dropdown 1005 | self.camera_combo.pack(pady=5) 1006 | self.refresh_camera_list() # Populate the camera list 1007 | self.camera_combo.bind("<>", self.on_camera_selected) 1008 | 1009 | # Button to refresh camera list 1010 | refresh_cameras_button = tk.Button(camera_frame, text="Refresh Cameras", command=self.refresh_camera_list) 1011 | refresh_cameras_button.pack(pady=2) 1012 | 1013 | settings_frame = tk.LabelFrame(self.left_frame, text="Settings Directory") 1014 | settings_frame.pack(fill='x', pady=2) 1015 | 1016 | # Read-only Text widget to display the directory with wrapping and highlighting support 1017 | self.settings_directory_display = tk.Text( 1018 | settings_frame, 1019 | height=2, # Adjust height as needed 1020 | wrap='word', # Wrap text within the widget 1021 | state='normal', # Allow changes to content 1022 | bg=self.root.cget('bg'), # Match the background color of the root 1023 | relief='flat', 1024 | width=15, # Set a fixed width to prevent expansion 1025 | padx=5, 1026 | pady=5 1027 | ) 1028 | self.settings_directory_display.insert('1.0', self.settings_directory) # Insert the directory text 1029 | self.settings_directory_display.config(state='disabled') # Make it read-only 1030 | self.settings_directory_display.pack(fill='x', padx=5, pady=5) 1031 | 1032 | 1033 | 1034 | def resource_path(relative_path): 1035 | """ Get absolute path to resource, works for dev and for PyInstaller """ 1036 | base_path = getattr(sys, '_MEIPASS', os.path.abspath(".")) 1037 | return os.path.join(base_path, relative_path) 1038 | 1039 | # Define the base path for the tooltip images 1040 | base_image_path = resource_path(os.path.join('images', 'tooltip-images')) 1041 | 1042 | # Roku Control Buttons 1043 | control_frame = tk.Frame(self.left_frame) 1044 | control_frame.pack(pady=10, fill='x') 1045 | 1046 | # Add spacing columns to align content equidistantly 1047 | control_frame.grid_columnconfigure(0, weight=1) # Spacer column on the left 1048 | control_frame.grid_columnconfigure(3, weight=1) # Spacer column on the right 1049 | 1050 | # Create a frame specifically for centering the buttons 1051 | center_frame = tk.Frame(control_frame) 1052 | center_frame.grid(row=0, column=0, columnspan=3, pady=5) # Span across 3 columns to center 1053 | 1054 | # Activate/Deactivate button 1055 | activate_button = tk.Button( 1056 | center_frame, text="🤞 Activate/Deactivate", state="disabled", width=17, height=1 1057 | ) 1058 | activate_button.grid(row=0, column=0, padx=10) # Add padding for spacing 1059 | Tooltip(activate_button, image_path=os.path.join(base_image_path, "fingers-crossed.jpg")) 1060 | 1061 | # Middle button (currently commented out) 1062 | # power_button = tk.Button( 1063 | # center_frame, text="⚡ Power", command=lambda: self.send_command('Power'), width=10, height=1, state='disabled' 1064 | # ) 1065 | # power_button.grid(row=0, column=1, padx=10) # Add padding for spacing 1066 | # Tooltip(power_button, image_path=os.path.join(base_image_path, "horns.jpg")) 1067 | 1068 | # Fist button 1069 | fist_button = tk.Button( 1070 | center_frame, text="✊ Between Gestures", state="disabled", width=17, height=1 1071 | ) 1072 | fist_button.grid(row=0, column=1, padx=10) # Add padding for spacing 1073 | Tooltip(fist_button, image_path=os.path.join(base_image_path, "fist.jpg")) 1074 | 1075 | 1076 | # deactivate_button = tk.Button( 1077 | # control_frame, text="Deactivate", state="disabled", width=10, height=1 1078 | # ) 1079 | # deactivate_button.grid(row=0, column=2, pady=5) # Right of the Power button 1080 | # Tooltip(deactivate_button, image_path=os.path.join(base_image_path, "fingers-deactivate.jpg")) 1081 | 1082 | # Back and Home Row 1083 | back_button = tk.Button(control_frame, text="⟲ Back", command=lambda: self.send_command('Back'), width=10, height=1, state='disabled') 1084 | Tooltip(back_button, image_path=os.path.join(base_image_path, "thumbs-out.jpg"), wide=True) 1085 | 1086 | power_button = tk.Button( 1087 | control_frame, text="⚡ Power", command=lambda: self.send_command('Power'), width=10, height=1, state='disabled' 1088 | ) 1089 | Tooltip(power_button, image_path=os.path.join(base_image_path, "horns.jpg")) 1090 | 1091 | home_button = tk.Button(control_frame, text="⌂ Home", command=lambda: self.send_command('Home'), width=10, height=1, state='disabled') 1092 | Tooltip(home_button, image_path=os.path.join(base_image_path, "home.jpg")) 1093 | 1094 | back_button.grid(row=1, column=0, pady=5, padx=5) 1095 | power_button.grid(row=1, column=1, pady=5, padx=5) 1096 | home_button.grid(row=1, column=2, pady=5, padx=5) 1097 | self.control_buttons.extend([back_button, power_button, home_button]) 1098 | 1099 | # Navigation and Volume Controls 1100 | nav_frame = tk.Frame(control_frame) 1101 | nav_frame.grid(row=2, column=1, pady=5, padx=10) # Navigation on the left 1102 | 1103 | up_button = tk.Button(nav_frame, text="↑", command=lambda: self.send_command('Up'), width=5, height=1, state='disabled') 1104 | Tooltip(up_button, image_path=os.path.join(base_image_path, "i-up.jpg")) 1105 | 1106 | left_button = tk.Button(nav_frame, text="←", command=lambda: self.send_command('Left'), width=5, height=1, state='disabled') 1107 | Tooltip(left_button, image_path=os.path.join(base_image_path, "i-left.jpg")) 1108 | 1109 | ok_button = tk.Button(nav_frame, text="OK", command=lambda: self.send_command('Select'), width=5, height=1, state='disabled') 1110 | Tooltip(ok_button, image_path=os.path.join(base_image_path, "thumbs-in.jpg"), wide=True) 1111 | 1112 | right_button = tk.Button(nav_frame, text="→", command=lambda: self.send_command('Right'), width=5, height=1, state='disabled') 1113 | Tooltip(right_button, image_path=os.path.join(base_image_path, "i-right.jpg")) 1114 | 1115 | down_button = tk.Button(nav_frame, text="↓", command=lambda: self.send_command('Down'), width=5, height=1, state='disabled') 1116 | Tooltip(down_button, image_path=os.path.join(base_image_path, "i-down.jpg")) 1117 | 1118 | # Arrange Navigation Buttons in a Grid 1119 | up_button.grid(row=0, column=1, pady=5) 1120 | left_button.grid(row=1, column=0, pady=5) 1121 | ok_button.grid(row=1, column=1, pady=5) 1122 | right_button.grid(row=1, column=2, pady=5) 1123 | down_button.grid(row=2, column=1, pady=5) 1124 | 1125 | self.control_buttons.extend([up_button, left_button, ok_button, right_button, down_button]) 1126 | 1127 | volume_frame = tk.Frame(control_frame) 1128 | volume_frame.grid(row=2, column=2, pady=5, padx=10) # Volume controls on the right 1129 | 1130 | volume_down_button = tk.Button(volume_frame, text="🔉 Vol-", command=lambda: self.send_command('VolumeDown'), width=8, height=1, state='disabled') 1131 | Tooltip(volume_down_button, image_path=os.path.join(base_image_path, "thumbs-down.jpg")) 1132 | 1133 | mute_button = tk.Button(volume_frame, text="🔇 Mute", command=lambda: self.send_command('VolumeMute'), width=8, height=1, state='disabled') 1134 | Tooltip(mute_button, image_path=os.path.join(base_image_path, "fingers-mute.jpg")) 1135 | 1136 | volume_up_button = tk.Button(volume_frame, text="🔊 Vol+", command=lambda: self.send_command('VolumeUp'), width=8, height=1, state='disabled') 1137 | Tooltip(volume_up_button, image_path=os.path.join(base_image_path, "thumbs-up.jpg")) 1138 | 1139 | # Arrange Volume Controls in a Grid 1140 | volume_up_button.grid(row=0, column=0, pady=5) 1141 | mute_button.grid(row=2, column=0, pady=5) 1142 | volume_down_button.grid(row=1, column=0, pady=5) 1143 | 1144 | self.control_buttons.extend([volume_down_button, mute_button, volume_up_button]) 1145 | 1146 | # Media Controls Row 1147 | rw_button = tk.Button(control_frame, text="⏪ RW", command=lambda: self.send_command('Rev'), width=8, height=1, state='disabled') 1148 | Tooltip(rw_button, image_path=os.path.join(base_image_path, "i-m-left.jpg")) 1149 | 1150 | play_pause_button = tk.Button(control_frame, text="⏯ Play/Pause", command=lambda: self.send_command('Play'), width=16, height=1, state='disabled') 1151 | Tooltip(play_pause_button, image_path=os.path.join(base_image_path, "i-m-up.jpg")) 1152 | 1153 | ff_button = tk.Button(control_frame, text="⏩ FF", command=lambda: self.send_command('Fwd'), width=8, height=1, state='disabled') 1154 | Tooltip(ff_button, image_path=os.path.join(base_image_path, "i-m-right.jpg")) 1155 | 1156 | rw_button.grid(row=3, column=0, pady=5) 1157 | play_pause_button.grid(row=3, column=1, pady=5) 1158 | ff_button.grid(row=3, column=2, pady=5) 1159 | self.control_buttons.extend([rw_button, play_pause_button, ff_button]) 1160 | 1161 | 1162 | # Video Display Panel 1163 | self.video_display_panel = VideoDisplayPanel(self.right_frame) 1164 | self.video_display_panel.pack(fill='both', expand=True) 1165 | self.video_display_panel.show_idle() # Show idle message initially 1166 | 1167 | # Attempt to connect to known devices after GUI is set up 1168 | self.attempt_connect_known_devices() 1169 | 1170 | self.root.after(0, self.root.geometry, "") 1171 | 1172 | # Check if auto_start is enabled and start video feed 1173 | if self.auto_start: 1174 | if self.roku_ip: 1175 | print(f"Auto-starting video feed with camera index {self.selected_camera_index}") 1176 | logging.debug(f"Auto-starting video feed with camera index {self.selected_camera_index}") 1177 | self.start_video_feed() 1178 | else: 1179 | print("Auto-start is enabled, but no Roku device found. Waiting to connect.") 1180 | logging.debug("Auto-start is enabled, but no Roku device found. Waiting to connect.") 1181 | 1182 | self.root.mainloop() 1183 | 1184 | if __name__ == '__main__': 1185 | roku_remote = RokuRemote() 1186 | if roku_remote.roku_ip: 1187 | print(f"Connected to Roku at {roku_remote.roku_ip}") 1188 | logging.debug(f"Connected to Roku at {roku_remote.roku_ip}") 1189 | else: 1190 | print("No Roku device found on the network.") 1191 | logging.debug("No Roku device found on the network.") 1192 | roku_remote.setup_gui() 1193 | -------------------------------------------------------------------------------- /tooltip.py: -------------------------------------------------------------------------------- 1 | import tkinter as tk 2 | from PIL import Image, ImageTk 3 | 4 | 5 | class Tooltip: 6 | def __init__(self, widget, image_path=None, text=None, wide=None): 7 | self.widget = widget 8 | self.wide = wide 9 | self.image_path = image_path 10 | self.text = text 11 | self.tooltip_window = None 12 | self.widget.bind("", self.show_tooltip) 13 | self.widget.bind("", self.hide_tooltip) 14 | 15 | def show_tooltip(self, event): 16 | if self.tooltip_window: 17 | return # Tooltip already displayed 18 | 19 | # Create a top-level window for the tooltip 20 | self.tooltip_window = tk.Toplevel(self.widget) 21 | self.tooltip_window.wm_overrideredirect(True) # Remove window decorations 22 | 23 | # Get the position of the widget to place the tooltip near it 24 | x, y, _, _ = self.widget.bbox("insert") 25 | x += self.widget.winfo_rootx() + 20 # Offset for better positioning 26 | y += self.widget.winfo_rooty() + 20 27 | self.tooltip_window.geometry(f"+{x}+{y}") 28 | 29 | # Add image or text to the tooltip 30 | if self.image_path: 31 | img = Image.open(self.image_path) 32 | width = 100 33 | height = 100 34 | if self.wide: 35 | width *= 2 # Double the width if `wide` is True 36 | img = img.resize((width, height)) # Resize for consistency 37 | self.tooltip_image = ImageTk.PhotoImage(img) # Store reference to avoid garbage collection 38 | label = tk.Label(self.tooltip_window, image=self.tooltip_image, borderwidth=1, relief="solid") 39 | else: 40 | label = tk.Label(self.tooltip_window, text=self.text, borderwidth=1, relief="solid") 41 | 42 | label.pack() 43 | 44 | def hide_tooltip(self, event): 45 | if self.tooltip_window: 46 | self.tooltip_window.destroy() 47 | self.tooltip_window = None 48 | --------------------------------------------------------------------------------