├── README.md ├── image.png ├── main.py ├── requirements.txt └── thumb.png /README.md: -------------------------------------------------------------------------------- 1 | # Car controller for games using OpenCV 2 | 3 | # Info 4 | - Use OpenCV to detect contours and filters out the blue color on the steering wheel (See [Sample steering wheel](image.png)) 5 | - It then gets the slope of the line formed using the center of the wheel and the top of the wheel and uses that to determine the steering input 6 | - Also, the position center of the wheel is responsible for braking and accelerating 7 | - [Watch the video](https://youtu.be/TXieQvzbTD8) 8 | [![Watch the video](thumb.png)](https://youtu.be/TXieQvzbTD8) 9 | 10 | # Usage (Requires python3): 11 | 1. `pip install -r requirements.txt` 12 | 2. `python main.py` 13 | 14 | -------------------------------------------------------------------------------- /image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adityamhatre/GameCarControl-CV/acb458591dcdfb2320b17f4da17f3fb6fdcc33db/image.png -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import cv2.cv2 as cv2 4 | import imutils 5 | import numpy as np 6 | from imutils.video import VideoStream 7 | from pynput.keyboard import Controller, Key 8 | 9 | keyboard = Controller() 10 | 11 | video = VideoStream(src=0).start() 12 | frame = None 13 | 14 | lb = [40, 100, 0] 15 | rb = [255, 255, 255] 16 | 17 | 18 | def set_lb(i, v): 19 | lb[i] = v 20 | 21 | 22 | def set_rb(i, v): 23 | rb[i] = v 24 | 25 | 26 | actions = ["", ""] 27 | 28 | cv2.namedWindow('mask') 29 | 30 | # use this for getting your own threshold values for whatever color 31 | # cv2.createTrackbar('lower_b_0', 'mask', lb[0], 255, (lambda a: set_lb(0, a))) 32 | # cv2.createTrackbar('lower_b_1', 'mask', lb[1], 255, (lambda a: set_lb(1, a))) 33 | # cv2.createTrackbar('lower_b_2', 'mask', lb[2], 255, (lambda a: set_lb(2, a))) 34 | # cv2.createTrackbar('upper_b_0', 'mask', rb[0], 255, (lambda a: set_rb(0, a))) 35 | # cv2.createTrackbar('upper_b_1', 'mask', rb[1], 255, (lambda a: set_rb(1, a))) 36 | # cv2.createTrackbar('upper_b_2', 'mask', rb[2], 255, (lambda a: set_rb(2, a))) 37 | 38 | 39 | def press_key(key): 40 | keyboard.press(key) 41 | 42 | 43 | def straighten(): 44 | keyboard.release(Key.left) 45 | keyboard.release(Key.right) 46 | 47 | 48 | def neutral(): 49 | keyboard.release(Key.up) 50 | keyboard.release(Key.down) 51 | 52 | 53 | def steer(slope): 54 | if 70 <= slope <= 105: 55 | actions[1] = "straight" 56 | straighten() 57 | elif slope > 105: 58 | actions[1] = "left" 59 | press_key(Key.left) 60 | elif slope < 70: 61 | actions[1] = "right" 62 | press_key(Key.right) 63 | 64 | 65 | def gas(y): 66 | if 200 <= y <= 250: 67 | actions[0] = "Coasting" 68 | neutral() 69 | elif y < 200: 70 | actions[0] = "Accelerating" 71 | press_key(Key.up) 72 | elif y > 250: 73 | actions[0] = "Braking" 74 | press_key(Key.down) 75 | 76 | 77 | def get_action(): 78 | return "{} {}".format(actions[0], actions[1]) 79 | 80 | 81 | def process_wheel(): 82 | global frame 83 | wheel_frame = frame.copy() 84 | 85 | hsv = cv2.cvtColor(wheel_frame, cv2.COLOR_BGR2HSV) 86 | 87 | lower_blue = np.array(lb.copy()) # [35, 100, 0] 88 | upper_blue = np.array(rb.copy()) # [255, 255, 255]) 89 | 90 | mask = cv2.inRange(hsv, lower_blue, upper_blue) 91 | 92 | anded_res = cv2.bitwise_and(wheel_frame, wheel_frame, mask=mask) 93 | contours, _ = cv2.findContours(cv2.Canny(anded_res, 255 / 3, 255), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) 94 | 95 | area_threshold = 400 96 | inds = [] 97 | for i, c in enumerate(contours): 98 | a = cv2.contourArea(c) 99 | if a > area_threshold and len(inds) < 2: 100 | inds.append(i) 101 | 102 | if not inds or len(inds) != 2: 103 | cv2.imshow('wheel', wheel_frame) # [165:200, 326:500]) 104 | # cv2.imshow('mask', mask) # [165:200, 326:500]) 105 | return 106 | 107 | if cv2.contourArea(contours[inds[0]]) < cv2.contourArea(contours[inds[1]]): 108 | inds[0], inds[1] = inds[1], inds[0] 109 | 110 | moments1 = cv2.moments(contours[inds[0]]) 111 | moments2 = cv2.moments(contours[inds[1]]) 112 | 113 | p1 = [int(moments1["m10"] / moments1["m00"]), int(moments1["m01"] / moments1["m00"])] 114 | p2 = [int(moments2["m10"] / moments2["m00"]), int(moments2["m01"] / moments2["m00"])] 115 | 116 | cv2.circle(wheel_frame, (p1[0], p1[1]), 3, (255, 255, 255), -1) 117 | cv2.circle(wheel_frame, (p2[0], p2[1]), 3, (255, 255, 255), -1) 118 | 119 | cv2.line(wheel_frame, (p1[0], p1[1]), (p2[0], p2[1]), (0, 255, 0), 2) 120 | if p2[0] - p1[0] == 0: 121 | slope = 90 122 | else: 123 | slope = -np.rad2deg(np.arctan2((p2[1] - p1[1]), (p2[0] - p1[0]))) % 360 124 | 125 | cv2.drawContours(wheel_frame, contours, inds[0], (0, 0, 255), 2) 126 | cv2.drawContours(wheel_frame, contours, inds[1], (0, 255, 0), 2) 127 | 128 | cv2.putText(wheel_frame, "Steering angle: {}".format(np.round(slope)), (10, 100), cv2.FONT_HERSHEY_PLAIN, 2, 129 | (255, 255, 0), 2) 130 | cv2.putText(wheel_frame, "{}".format(get_action()), (10, 140), cv2.FONT_HERSHEY_PLAIN, 2, 131 | (255, 255, 0), 2) 132 | 133 | cv2.line(wheel_frame, (0, 200), (600, 200), (255, 255, 255), 1) 134 | cv2.line(wheel_frame, (0, 250), (600, 250), (255, 255, 255), 1) 135 | steer(slope) 136 | gas(p1[1]) 137 | cv2.imshow('wheel', wheel_frame) # [165:200, 326:500]) 138 | # cv2.imshow('mask', mask) # [165:200, 326:500]) 139 | 140 | 141 | while True: 142 | # Capture frame-by-frame 143 | frame = video.read() 144 | 145 | frame = cv2.flip(frame, 1) 146 | frame = cv2.medianBlur(frame, 5) 147 | frame = imutils.resize(frame, width=600, height=400) 148 | 149 | process_wheel() 150 | 151 | if cv2.waitKey(1) & 0xFF == ord('q'): 152 | break 153 | 154 | # When everything done, release the capture 155 | video.stop() 156 | cv2.destroyAllWindows() 157 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | opencv-python==4.3.0.36 2 | pynput==1.6.8 3 | numpy==1.16.4 4 | imutils==0.5.3 -------------------------------------------------------------------------------- /thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adityamhatre/GameCarControl-CV/acb458591dcdfb2320b17f4da17f3fb6fdcc33db/thumb.png --------------------------------------------------------------------------------