├── .gitignore
├── __pycache__
    └── directinput.cpython-38.pyc
├── requirements.txt
├── README.md
├── directinput.py
└── steeringwheel.py


/.gitignore:
--------------------------------------------------------------------------------
1 | venv
2 | .idea
3 | 


--------------------------------------------------------------------------------
/__pycache__/directinput.cpython-38.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HaiderZaidiDev/virtual-ml-steering-wheel/HEAD/__pycache__/directinput.cpython-38.pyc


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
 1 | absl-py==0.13.0
 2 | attrs==21.2.0
 3 | cycler==0.10.0
 4 | kiwisolver==1.3.1
 5 | matplotlib==3.4.2
 6 | mediapipe==0.8.6
 7 | MouseInfo==0.1.3
 8 | numpy==1.21.1
 9 | opencv-contrib-python==4.5.3.56
10 | opencv-python==4.5.3.56
11 | Pillow==8.3.1
12 | protobuf==3.17.3
13 | PyAutoGUI==0.9.53
14 | PyDirectInput==1.0.4
15 | PyGetWindow==0.0.9
16 | PyMsgBox==1.0.9
17 | pyparsing==2.4.7
18 | pyperclip==1.8.2
19 | PyRect==0.1.4
20 | PyScreeze==0.1.27
21 | python-dateutil==2.8.2
22 | PyTweening==1.0.3
23 | six==1.16.0
24 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | <br />
 2 | <p align="center">
 3 |   <h3 align="center">Virtual Steering Wheel</h3>
 4 |   <p align="center">
 5 |     A virtual steering wheel, made with Python and pre-trained machine learning models. 
 6 |   </p>
 7 | </p>
 8 | 
 9 | 
10 | <!-- ABOUT THE PROJECT -->
11 | ## About The Project
12 | 
13 | Recently got a webcam, and I've always wanted to explore using it to play around with computer vision (e.g., hand and gesture tracking); so I decided to create a virtual steering wheel with Python.
14 | 
15 | ## How it Works
16 | Uses Python (OpenCV) and the pre-trained models from [Mediapipe (Google)](http://google.github.io/mediapipe/ "Mediapipe (Google)") to capture video, recognize hands in the frame and then calculate the slope between the two hands to determine which direction to turn in. 
17 | 
18 | Want to see it in action? Click the thumbnail for a demo.
19 | [![demo-thubmnail](https://i.imgur.com/XeL4zIn.png)](https://www.youtube.com/watch?v=q0O3pqBi1xs)
20 | 
21 | <!-- GETTING STARTED -->
22 | ## Getting Started
23 | 
24 | Looking to demo the app yourself? Check this part out.
25 | 
26 | ### Installation
27 | 1. Clone the repo
28 |    ```sh
29 |    git clone https://github.com/HaiderZaidiDev/virtual-ml-steering-wheel
30 |    ```
31 | 2. Install PIP packages
32 |    ```sh
33 |     pip install -r requirements.txt
34 |    ```
35 | 3. Run the Python script
36 |    ```sh
37 |     python3 steeringwheel.py
38 |    ```
39 | 
40 | <!-- CONTRIBUTING -->
41 | ## Contributing
42 | 
43 | Contributions are welcome, feel free to make a PR!
44 | 
45 | 1. Fork the Project
46 | 2. Create your Feature Branch (`git checkout -b feature/AmazingFeature`)
47 | 3. Commit your Changes (`git commit -m 'Add some AmazingFeature'`)
48 | 4. Push to the Branch (`git push origin feature/AmazingFeature`)
49 | 5. Open a Pull Request
50 | 


--------------------------------------------------------------------------------
/directinput.py:
--------------------------------------------------------------------------------
 1 | # Sending keys through direct input (PyAutoGui does not send directly, thus, not working in games).
 2 | # Implemented from (with modifications):
 3 | # https://pythonprogramming.net/direct-input-game-python-plays-gta-v/?completed=/open-cv-basics-python-plays-gta-v/
 4 | 
 5 | import ctypes
 6 | 
 7 | keys = {
 8 |     "w":0x11,
 9 |     "a":0x1E,
10 |     "s":0x1F,
11 |     "d":0x20,
12 | }
13 | PUL = ctypes.POINTER(ctypes.c_ulong)
14 | class KeyBdInput(ctypes.Structure):
15 |     _fields_ = [("wVk", ctypes.c_ushort),
16 |                 ("wScan", ctypes.c_ushort),
17 |                 ("dwFlags", ctypes.c_ulong),
18 |                 ("time", ctypes.c_ulong),
19 |                 ("dwExtraInfo", PUL)]
20 | 
21 | class HardwareInput(ctypes.Structure):
22 |     _fields_ = [("uMsg", ctypes.c_ulong),
23 |                 ("wParamL", ctypes.c_short),
24 |                 ("wParamH", ctypes.c_ushort)]
25 | 
26 | class MouseInput(ctypes.Structure):
27 |     _fields_ = [("dx", ctypes.c_long),
28 |                 ("dy", ctypes.c_long),
29 |                 ("mouseData", ctypes.c_ulong),
30 |                 ("dwFlags", ctypes.c_ulong),
31 |                 ("time",ctypes.c_ulong),
32 |                 ("dwExtraInfo", PUL)]
33 | 
34 | class Input_I(ctypes.Union):
35 |     _fields_ = [("ki", KeyBdInput),
36 |                  ("mi", MouseInput),
37 |                  ("hi", HardwareInput)]
38 | 
39 | class Input(ctypes.Structure):
40 |     _fields_ = [("type", ctypes.c_ulong),
41 |                 ("ii", Input_I)]
42 | 
43 | def press_key(key):
44 |     extra = ctypes.c_ulong(0)
45 |     ii_ = Input_I()
46 |     ii_.ki = KeyBdInput( 0, keys[key], 0x0008, 0, ctypes.pointer(extra) )
47 |     x = Input( ctypes.c_ulong(1), ii_ )
48 |     ctypes.windll.user32.SendInput(1, ctypes.pointer(x), ctypes.sizeof(x))
49 | 
50 | def release_key(key):
51 |     extra = ctypes.c_ulong(0)
52 |     ii_ = Input_I()
53 |     ii_.ki = KeyBdInput( 0, keys[key], 0x0008 | 0x0002, 0, ctypes.pointer(extra) )
54 |     x = Input( ctypes.c_ulong(1), ii_ )
55 |     ctypes.windll.user32.SendInput(1, ctypes.pointer(x), ctypes.sizeof(x))
56 | 


--------------------------------------------------------------------------------
/steeringwheel.py:
--------------------------------------------------------------------------------
  1 | import cv2
  2 | import mediapipe as mp
  3 | import pydirectinput
  4 | import directinput
  5 | import time
  6 | 
  7 | 
  8 | cap = cv2.VideoCapture(1) # Fetching video camera.
  9 | # Setting video capture resolution for the webcam to 1280x720 (px)
 10 | cap.set(3, 1280)
 11 | cap.set(4, 720)
 12 | 
 13 | mp_drawing = mp.solutions.drawing_utils
 14 | 
 15 | # Fetching hand models.
 16 | mp_hands = mp.solutions.hands
 17 | hands = mp_hands.Hands()
 18 | 
 19 | def write_text(img, text, x, y):
 20 |     """ Writing (overlaying) text on the OpenCV camera view.
 21 | 
 22 |     Parameters
 23 |     __________
 24 |     img:
 25 |         Frame of the current image.
 26 |     text: str
 27 |         Text being written to the camera view.
 28 |     x: int
 29 |         X coordinate for plotting the text.
 30 |     y: int
 31 |         Y coordinate for plotting the text.
 32 |     """
 33 |     font = cv2.FONT_HERSHEY_SIMPLEX
 34 |     pos = (x, y)
 35 |     fontScale = 1
 36 |     fontColor = (255, 255, 255) # White.
 37 |     lineType = 2
 38 |     cv2.putText(img,
 39 |                 text,
 40 |                 pos,
 41 |                 font,
 42 |                 fontScale,
 43 |                 fontColor,
 44 |                 lineType)
 45 | 
 46 | # Used for FPS calculations.
 47 | def steering_wheel():
 48 |     """ Uses the slope of the distance between middle fingers to create a virtual steering wheel for gaming.
 49 |     """
 50 |     prev_frame_time = 0
 51 |     new_frame_time = 0
 52 |     while cap.isOpened():
 53 |         success, img = cap.read()
 54 |         cv2.waitKey(1) # Continuously refreshes the webcam frame every 1ms.
 55 |         img = cv2.flip(img, 1)
 56 |         img.flags.writeable = False # Making the images not writeable for optimization.
 57 |         results = hands.process(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) # Processing video.
 58 |         # Note: Converting to RGB results in a significant increase in hand recognition accuracy.
 59 | 
 60 |         # Checking if a hand exists in the frame.
 61 |         landmarks = results.multi_hand_landmarks # Fetches all the landmarks (points) on the hand.
 62 |         if landmarks:
 63 |             # When a hand exists in the frame.
 64 | 
 65 |             # FPS Calculations
 66 |             new_frame_time = time.time()
 67 |             fps = str(int(1/(new_frame_time - prev_frame_time)))
 68 |             write_text(img, fps, 150, 500)
 69 |             prev_frame_time = new_frame_time
 70 | 
 71 |             if(len(landmarks) == 2): # If 2 hands are in view.
 72 |                 left_hand_landmarks = landmarks[1].landmark
 73 |                 right_hand_landmarks = landmarks[0].landmark
 74 | 
 75 |                 # Fetching the height and width of the camera view.
 76 |                 shape = img.shape
 77 |                 width = shape[1]
 78 |                 height = shape[0]
 79 | 
 80 |                 # Isolating the tip of middle fingers from both hands, and normalizing their coordinates based on height/width
 81 |                 # of the camera view.
 82 |                 left_mFingerX, left_mFingerY = (left_hand_landmarks[11].x * width), (left_hand_landmarks[11].y * height)
 83 |                 right_mFingerX, right_mFingerY = (right_hand_landmarks[11].x * width), (right_hand_landmarks[11].y * height)
 84 | 
 85 |                 # Calculating slope between middle fingers of both hands (we use this to determine whether we're turning
 86 |                 # left or right.
 87 |                 slope = ((right_mFingerY - left_mFingerY)/(right_mFingerX-left_mFingerX))
 88 | 
 89 |                 # Outputs for testing.
 90 |                 #print(f"Left hand: ({left_mFingerX}, {left_mFingerY})")
 91 |                 #print(f"Right hand: ({right_mFingerX}, {right_mFingerY})")
 92 |                 #print(f"Slope: {slope}")
 93 | 
 94 |                 sensitivity = 0.3 # Adjusts sentivity for turing; the higher this is, the more you have to turn your hands.
 95 |                 if abs(slope) > sensitivity:
 96 |                     if slope < 0:
 97 |                         # When the slope is negative, we turn left.
 98 |                         print("Turn left.")
 99 |                         write_text(img, "Left.", 360, 360)
100 |                         directinput.release_key("w")
101 |                         directinput.release_key('a')
102 |                         directinput.press_key('a')
103 |                     if slope > 0:
104 |                         # When the slope is positive, we turn right.
105 |                         print("Turn right.")
106 |                         write_text(img, "Right.", 360, 360)
107 |                         directinput.release_key('w')
108 |                         directinput.release_key('a')
109 |                         directinput.press_key('d')
110 |                 if abs(slope) < sensitivity:
111 |                     # When our hands are straight, we stay still (and also throttle).
112 |                     print("Keeping straight.")
113 |                     write_text(img, "Straight.", 360, 360)
114 |                     directinput.release_key('a')
115 |                     directinput.release_key('d')
116 |                     directinput.press_key('w') # Remove this if you have pedals to control the speed.
117 | 
118 |              # Iterating through landmarks (i.e., co-ordinates for finger joints) and drawing the connections.
119 |             for hand_landmarks in landmarks:
120 |                 mp_drawing.draw_landmarks(img, hand_landmarks, mp_hands.HAND_CONNECTIONS)
121 |         cv2.imshow("Hand Recognition", img)
122 |     cap.release()
123 | steering_wheel()
124 | 


--------------------------------------------------------------------------------