├── AIVirtualMouse.py ├── HandTrackingModule.py └── README.md /AIVirtualMouse.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import HandTrackingModule as htm 4 | import time 5 | import autopy 6 | 7 | ###################### 8 | wCam, hCam = 640, 480 9 | frameR = 100 #Frame Reduction 10 | smoothening = 7 #random value 11 | ###################### 12 | 13 | pTime = 0 14 | plocX, plocY = 0, 0 15 | clocX, clocY = 0, 0 16 | cap = cv2.VideoCapture(0) 17 | cap.set(3, wCam) 18 | cap.set(4, hCam) 19 | 20 | detector = htm.handDetector(maxHands=1) 21 | wScr, hScr = autopy.screen.size() 22 | 23 | # print(wScr, hScr) 24 | 25 | while True: 26 | # Step1: Find the landmarks 27 | success, img = cap.read() 28 | img = detector.findHands(img) 29 | lmList, bbox = detector.findPosition(img) 30 | 31 | # Step2: Get the tip of the index and middle finger 32 | if len(lmList) != 0: 33 | x1, y1 = lmList[8][1:] 34 | x2, y2 = lmList[12][1:] 35 | 36 | # Step3: Check which fingers are up 37 | fingers = detector.fingersUp() 38 | cv2.rectangle(img, (frameR, frameR), (wCam - frameR, hCam - frameR), 39 | (255, 0, 255), 2) 40 | 41 | # Step4: Only Index Finger: Moving Mode 42 | if fingers[1] == 1 and fingers[2] == 0: 43 | 44 | # Step5: Convert the coordinates 45 | x3 = np.interp(x1, (frameR, wCam-frameR), (0, wScr)) 46 | y3 = np.interp(y1, (frameR, hCam-frameR), (0, hScr)) 47 | 48 | # Step6: Smooth Values 49 | clocX = plocX + (x3 - plocX) / smoothening 50 | clocY = plocY + (y3 - plocY) / smoothening 51 | 52 | # Step7: Move Mouse 53 | autopy.mouse.move(wScr - clocX, clocY) 54 | cv2.circle(img, (x1, y1), 15, (255, 0, 255), cv2.FILLED) 55 | plocX, plocY = clocX, clocY 56 | 57 | # Step8: Both Index and middle are up: Clicking Mode 58 | if fingers[1] == 1 and fingers[2] == 1: 59 | 60 | # Step9: Find distance between fingers 61 | length, img, lineInfo = detector.findDistance(8, 12, img) 62 | 63 | # Step10: Click mouse if distance short 64 | if length < 40: 65 | cv2.circle(img, (lineInfo[4], lineInfo[5]), 15, (0, 255, 0), cv2.FILLED) 66 | autopy.mouse.click() 67 | 68 | # Step11: Frame rate 69 | cTime = time.time() 70 | fps = 1/(cTime-pTime) 71 | pTime = cTime 72 | cv2.putText(img, str(int(fps)), (28, 58), cv2.FONT_HERSHEY_PLAIN, 3, (255, 8, 8), 3) 73 | 74 | # Step12: Display 75 | cv2.imshow("Image", img) 76 | cv2.waitKey(1) -------------------------------------------------------------------------------- /HandTrackingModule.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import mediapipe as mp 3 | import time 4 | import math 5 | import numpy as np 6 | 7 | 8 | class handDetector(): 9 | def __init__(self, mode=False, maxHands=2, detectionCon=0.5, trackCon=0.5): 10 | self.mode = mode 11 | self.maxHands = maxHands 12 | self.detectionCon = detectionCon 13 | self.trackCon = trackCon 14 | 15 | self.mpHands = mp.solutions.hands 16 | self.hands = self.mpHands.Hands(self.mode, self.maxHands, self.detectionCon, self.trackCon) 17 | self.mpDraw = mp.solutions.drawing_utils 18 | self.tipIds = [4, 8, 12, 16, 20] 19 | 20 | def findHands(self, img, draw=True): 21 | imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 22 | self.results = self.hands.process(imgRGB) 23 | # print(results.multi_hand_landmarks) 24 | 25 | if self.results.multi_hand_landmarks: 26 | for handLms in self.results.multi_hand_landmarks: 27 | if draw: 28 | self.mpDraw.draw_landmarks(img, handLms, self.mpHands.HAND_CONNECTIONS) 29 | return img 30 | 31 | def findPosition(self, img, handNo=0, draw=True): 32 | xList = [] 33 | yList = [] 34 | bbox = [] 35 | self.lmList = [] 36 | if self.results.multi_hand_landmarks: 37 | myHand = self.results.multi_hand_landmarks[handNo] 38 | for id, lm in enumerate(myHand.landmark): 39 | # print(id, lm) 40 | h, w, c = img.shape 41 | cx, cy = int(lm.x * w), int(lm.y * h) 42 | xList.append(cx) 43 | yList.append(cy) 44 | # print(id, cx, cy) 45 | self.lmList.append([id, cx, cy]) 46 | if draw: 47 | cv2.circle(img, (cx, cy), 5, (255, 0, 255), cv2.FILLED) 48 | 49 | xmin, xmax = min(xList), max(xList) 50 | ymin, ymax = min(yList), max(yList) 51 | bbox = xmin, ymin, xmax, ymax 52 | 53 | if draw: 54 | cv2.rectangle(img, (xmin - 20, ymin - 20), (xmax + 20, ymax + 20), (0, 255, 0), 2) 55 | 56 | return self.lmList, bbox 57 | 58 | def fingersUp(self): 59 | fingers = [] 60 | # Thumb 61 | if self.lmList[self.tipIds[0]][1] > self.lmList[self.tipIds[0] -1][1]: 62 | fingers.append(1) 63 | else: 64 | fingers.append(0) 65 | # Fingers 66 | for id in range(1, 5): 67 | if self.lmList[self.tipIds[id]][2] < self.lmList[self.tipIds[id] -2][2]: 68 | fingers.append(1) 69 | else: 70 | fingers.append(0) 71 | # totalFingers = fingers.count(1) 72 | return fingers 73 | 74 | def findDistance(self, p1, p2, img, draw=True, r=15, t=3): 75 | x1, y1 = self.lmList[p1][1:] 76 | x2, y2 = self.lmList[p2][1:] 77 | cx, cy = (x1 + x2) // 2, (y1 + y2) // 2 78 | 79 | if draw: 80 | cv2.line(img, (x1, y1), (x2, y2), (255, 0, 255), t) 81 | cv2.circle(img, (x1, y1), r, (255, 0, 255), cv2.FILLED) 82 | cv2.circle(img, (x2, y2), r, (255, 0, 255), cv2.FILLED) 83 | cv2.circle(img, (cx, cy), r, (0, 0, 255), cv2.FILLED) 84 | length = math.hypot(x2 - x1, y2 - y1) 85 | 86 | return length, img, [x1, y1, x2, y2, cx, cy] 87 | 88 | def main(): 89 | pTime = 0 90 | cTime = 0 91 | cap = cv2.VideoCapture(1) 92 | detector = handDetector() 93 | while True: 94 | success, img = cap.read() 95 | img = detector.findHands(img) 96 | lmList, bbox = detector.findPosition(img) 97 | if len(lmList) != 0: 98 | print(lmList[4]) 99 | cTime = time.time() 100 | fps = 1 / (cTime - pTime) 101 | pTime = cTime 102 | cv2.putText(img, str(int(fps)), (10, 70), cv2.FONT_HERSHEY_PLAIN, 3, 103 | (255, 0, 255), 3) 104 | cv2.imshow("Image", img) 105 | cv2.waitKey(1) 106 | 107 | if __name__ == "__main__": 108 | main() 109 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## AI Virtual Mouse 2 | 3 | > Developed an AI-based approach for controlling the mouse movement using Python and openCV with real-time camera that detects hand landmarks, tracks gesture patterns instead of a physical mouse. 4 | 5 | ## Dependencies - 6 | > Please install all the required dependencies. 7 | * openCV - (For image processing and drawing) 8 | * mediapipe - (For Hand Tracking) 9 | * autopy - (For controlling the mouse movement and click) 10 | * numpy 11 | 12 | ## Installation 13 | 14 | To setup the system for development on your local machine, please follow the instructions below: 15 | 16 | 1. Clone the repository to your machine 17 | 18 | ```bash 19 | git clone https://github.com/ravigithub19/ai-virtual-mouse.git 20 | ``` 21 | 22 | 2. Run the ```AIVirtualMouse.py``` python file. 23 | 24 | 25 | --------------------------------------------------------------------------------