├── LEDSerialArduino.py ├── README.md ├── handtracking.py └── mqtt ├── mqttESP8266 └── mqttESP8266.ino └── mqttPython.py /LEDSerialArduino.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import pyfirmata 3 | from cvzone.HandTrackingModule import HandDetector 4 | from cvzone.FPS import FPS 5 | 6 | cap = cv2.VideoCapture(0) 7 | #Set size screen 8 | x_max, y_max = 1280, 720 9 | cap.set(3, x_max) 10 | cap.set(4, y_max) 11 | 12 | if not cap.isOpened(): 13 | print("Camera couldn't Access") 14 | exit() 15 | 16 | fpsReader = FPS() 17 | fps = fpsReader.update() 18 | 19 | detector = HandDetector(detectionCon=0.7) 20 | pinR, pinY, pinG = 2, 3, 4 21 | port = 'COM7' #Select your COM 22 | board = pyfirmata.Arduino(port) 23 | 24 | counter_R, counter_Y, counter_G = 0, 0, 0 25 | R_on, Y_on, G_on = False, False, False 26 | 27 | while True: 28 | success, img = cap.read() 29 | img = cv2.flip(img, 1) 30 | img = detector.findHands(img) 31 | fps, img = fpsReader.update(img) 32 | lmList, bboxInfo = detector.findPosition(img) 33 | 34 | if lmList : 35 | x, y = 100, 100 36 | w, h = 225, 225 37 | X, Y = 120, 190 38 | 39 | fx, fy = lmList[8][0], lmList[8][1] #index fingertip 40 | posFinger = [fx, fy] 41 | cv2.circle(img, (fx, fy), 15, (255, 0, 255), cv2.FILLED) #draw circle on index fingertip 42 | cv2.putText(img, str(posFinger), (fx+10, fy-10), cv2.FONT_HERSHEY_PLAIN, 2, (255, 255, 0), 3) 43 | # cv2.line(img, (0, fy), (x_max, fy), (255,255,0), 2) # x line 44 | # cv2.line(img, (fx, y_max), (fx, 0), (255, 255, 0), 2)# y line 45 | 46 | 47 | if x < fx < x + w - 95 and y < fy < y + h - 95: 48 | counter_R += 1 49 | cv2.rectangle(img, (x, y), (w, h), (255, 255, 0), cv2.FILLED) 50 | if counter_R == 1: 51 | R_on = not R_on 52 | else : 53 | counter_R = 0 54 | if R_on: 55 | R_val = 1 56 | cv2.rectangle(img, (x, y), (w, h), (0, 0, 255), cv2.FILLED) 57 | cv2.putText(img, "ON", (X, Y), cv2.FONT_HERSHEY_PLAIN, 58 | 4, (255, 255, 255), 5) 59 | else: 60 | R_val = 0 61 | cv2.rectangle(img, (x, y), (w, h), (150, 150, 150), cv2.FILLED) 62 | cv2.putText(img, "OFF", (X-15, Y), cv2.FONT_HERSHEY_PLAIN, 63 | 4, (0, 0, 255), 5) 64 | 65 | if x + 250 < fx < x + 155 + w and y < fy < y + h - 95: #155 = 250 - 95 66 | counter_Y += 1 67 | cv2.rectangle(img, (x + 250, y), (w + 250, h), (255, 255, 0), cv2.FILLED) 68 | if counter_Y == 1: 69 | Y_on = not Y_on 70 | else: 71 | counter_Y = 0 72 | if Y_on: 73 | Y_val = 1 74 | cv2.rectangle(img, (x+250, y), (w+250, h), (0, 255, 255), cv2.FILLED) 75 | cv2.putText(img, "ON", (X+250, Y), cv2.FONT_HERSHEY_PLAIN, 76 | 4, (255, 255, 255), 5) 77 | else: 78 | Y_val = 0 79 | cv2.rectangle(img, (x + 250, y), (w + 250, h), (150, 150, 150), cv2.FILLED) 80 | cv2.putText(img, "OFF", (X-15 + 250, Y), cv2.FONT_HERSHEY_PLAIN, 81 | 4, (0, 255, 255), 5) 82 | 83 | if x + 500 < fx < x + 405 + w and y < fy < y + h - 95: #500 - 95 = 405 84 | counter_G += 1 85 | cv2.rectangle(img, (x + 500, y), (w + 500, h), (255, 255, 0), cv2.FILLED) 86 | if counter_G == 1: 87 | G_on = not G_on 88 | 89 | else: 90 | counter_G = 0 91 | if G_on: 92 | G_val = 1 93 | cv2.rectangle(img, (x + 500, y), (w + 500, h), (0, 255, 0), cv2.FILLED) 94 | cv2.putText(img, "ON", (X + 500, Y), cv2.FONT_HERSHEY_PLAIN, 95 | 4, (255, 255, 255), 5) 96 | else: 97 | G_val = 0 98 | cv2.rectangle(img, (x + 500, y), (w + 500, h), (150, 150, 150), cv2.FILLED) 99 | cv2.putText(img, "OFF", (X-15 + 500, Y), cv2.FONT_HERSHEY_PLAIN, 100 | 4, (0, 255, 0), 5) 101 | 102 | board.digital[pinR].write(R_val) 103 | board.digital[pinY].write(Y_val) 104 | board.digital[pinG].write(G_val) 105 | 106 | 107 | cv2.imshow("Image", img) 108 | cv2.waitKey(1) 109 | 110 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # virtual-button-opencv 2 | 3 | **Python packages required:** 4 | * cvzone 1.4.1 5 | * mediapipe 0.8.7 6 | * pyfirmata 1.1.0 7 | 8 | **Arduino Configuration** 9 | - Open IDE Arduino 10 | - Select File -> Example -> Firmata -> StandardFirmata 11 | - Select Tools -> Board -> Arduino/Genuino Uno 12 | - Select Tools -> Port -> *choose your port COM* 13 | - Upload 14 | -------------------------------------------------------------------------------- /handtracking.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import pyfirmata 3 | from cvzone.HandTrackingModule import HandDetector 4 | import numpy as np 5 | 6 | cap = cv2.VideoCapture(0) 7 | #Set size screen 8 | ws, hs = 1280, 720 9 | cap.set(3, ws) 10 | cap.set(4, hs) 11 | 12 | if not cap.isOpened(): 13 | print("Camera couldn't Access") 14 | exit() 15 | 16 | detector = HandDetector(detectionCon=0.7) 17 | 18 | port = "COM7" 19 | board = pyfirmata.Arduino(port) 20 | servo_pinX = board.get_pin('d:5:s') #pin 5 Arduino 21 | servo_pinY = board.get_pin('d:6:s') #pin 6 Arduino 22 | 23 | 24 | while True: 25 | success, img = cap.read() 26 | img = cv2.flip(img, 1) 27 | img = detector.findHands(img) 28 | lmList, bboxInfo = detector.findPosition(img) 29 | 30 | if lmList : 31 | 32 | fx, fy = lmList[9][0], lmList[9][1] 33 | posFinger = [fx, fy] 34 | # convert coordinat to servo degree 35 | servoX = np.interp(fx, [0, ws], [0, 180]) 36 | servoY = np.interp(fy, [0, hs], [0, 180]) 37 | 38 | 39 | cv2.circle(img, (fx-20, fy+50), 15, (0, 0, 255), cv2.FILLED) # draw circle on center of hand 40 | cv2.putText(img, str(posFinger), (fx-10, fy+40), cv2.FONT_HERSHEY_PLAIN, 2, (0, 0, 255), 3) 41 | cv2.line(img, (0, fy+50), (ws, fy+50), (0,0,0), 2) # x line 42 | cv2.line(img, (fx-20, hs), (fx-20, 0), (0, 0, 0), 2)# y line 43 | cv2.putText(img, f'Servo X: {int(servoX)} deg', (50, 50), cv2.FONT_HERSHEY_PLAIN, 2, (255, 0, 0), 2) 44 | cv2.putText(img, f'Servo Y: {int(servoY)} deg', (50, 100), cv2.FONT_HERSHEY_PLAIN, 2, (255, 0, 0), 2) 45 | 46 | servo_pinX.write(servoX) 47 | servo_pinY.write(servoY) 48 | 49 | 50 | cv2.imshow("Image", img) 51 | cv2.waitKey(1) 52 | 53 | -------------------------------------------------------------------------------- /mqtt/mqttESP8266/mqttESP8266.ino: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | // Update these with values suitable for your network. 5 | 6 | const char* ssid = "Room105"; 7 | const char* password = "indonesiaraya"; 8 | const char* mqtt_server = "broker.mqtt-dashboard.com"; 9 | 10 | WiFiClient espClient; 11 | PubSubClient client(espClient); 12 | unsigned long lastMsg = 0; 13 | #define MSG_BUFFER_SIZE (50) 14 | char msg[MSG_BUFFER_SIZE]; 15 | int value = 0; 16 | 17 | void setup_wifi() { 18 | 19 | delay(10); 20 | // We start by connecting to a WiFi network 21 | Serial.println(); 22 | Serial.print("Connecting to "); 23 | Serial.println(ssid); 24 | 25 | WiFi.mode(WIFI_STA); 26 | WiFi.begin(ssid, password); 27 | 28 | while (WiFi.status() != WL_CONNECTED) { 29 | delay(500); 30 | Serial.print("."); 31 | } 32 | 33 | randomSeed(micros()); 34 | 35 | Serial.println(""); 36 | Serial.println("WiFi connected"); 37 | Serial.println("IP address: "); 38 | Serial.println(WiFi.localIP()); 39 | } 40 | 41 | void callback(char* topic, byte* payload, unsigned int length) { 42 | Serial.print("Message arrived ["); 43 | Serial.print(topic); 44 | Serial.print("] "); 45 | for (int i = 0; i < length; i++) { 46 | Serial.print((char)payload[i]); 47 | } 48 | Serial.println(); 49 | 50 | // Switch on the LED if an 1 was received as first character 51 | if ((char)payload[0] == '1') { 52 | digitalWrite(D1, HIGH); // Turn the LED on (Note that LOW is the voltage level 53 | } else { 54 | digitalWrite(D1, LOW); // Turn the LED off by making the voltage HIGH 55 | } 56 | if ((char)payload[1] == '1') { 57 | digitalWrite(D2, HIGH); // Turn the LED on (Note that LOW is the voltage level 58 | } else { 59 | digitalWrite(D2, LOW); // Turn the LED off by making the voltage HIGH 60 | } 61 | if ((char)payload[2] == '1') { 62 | digitalWrite(D3, HIGH); // Turn the LED on (Note that LOW is the voltage level 63 | } else { 64 | digitalWrite(D3, LOW); // Turn the LED off by making the voltage HIGH 65 | } 66 | 67 | } 68 | 69 | void reconnect() { 70 | // Loop until we're reconnected 71 | while (!client.connected()) { 72 | Serial.print("Attempting MQTT connection..."); 73 | // Create a random client ID 74 | String clientId = "ESP8266Client-"; 75 | clientId += String(random(0xffff), HEX); 76 | // Attempt to connect 77 | if (client.connect(clientId.c_str())) { 78 | Serial.println("connected"); 79 | // Once connected, publish an announcement... 80 | // client.publish("rizky2", "hello world"); 81 | // ... and resubscribe 82 | client.subscribe("RizkyProject/fingersup"); 83 | } else { 84 | Serial.print("failed, rc="); 85 | Serial.print(client.state()); 86 | Serial.println(" try again in 5 seconds"); 87 | // Wait 5 seconds before retrying 88 | delay(5000); 89 | } 90 | } 91 | } 92 | 93 | void setup() { 94 | pinMode(D1, OUTPUT); // Initialize the BUILTIN_LED pin as an output 95 | pinMode(D2, OUTPUT); 96 | pinMode(D3, OUTPUT); 97 | Serial.begin(115200); 98 | setup_wifi(); 99 | client.setServer(mqtt_server, 1883); 100 | client.setCallback(callback); 101 | } 102 | 103 | void loop() { 104 | 105 | if (!client.connected()) { 106 | reconnect(); 107 | } 108 | client.loop(); 109 | 110 | unsigned long now = millis(); 111 | if (now - lastMsg > 2000) { 112 | lastMsg = now; 113 | ++value; 114 | // snprintf (msg, MSG_BUFFER_SIZE, "hello world #%ld", value); 115 | // Serial.print("Publish message: "); 116 | // Serial.println(msg); 117 | // client.publish("rizky2", msg); 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /mqtt/mqttPython.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | from cvzone.HandTrackingModule import HandDetector 3 | from cvzone.FPS import FPS 4 | import paho.mqtt.client as mqtt 5 | 6 | cap = cv2.VideoCapture(0) 7 | #Set size screen 8 | x_max, y_max = 1280, 720 9 | cap.set(3, x_max) 10 | cap.set(4, y_max) 11 | 12 | if not cap.isOpened(): 13 | print("Camera couldn't Access") 14 | exit() 15 | 16 | broker_address="mqtt-dashboard.com" 17 | client = mqtt.Client("P1") #create new instance 18 | client.connect(broker_address) #connect to broker 19 | 20 | fpsReader = FPS() 21 | fps = fpsReader.update() 22 | 23 | detector = HandDetector(detectionCon=0.7) 24 | 25 | 26 | counter_R, counter_Y, counter_G = 0, 0, 0 27 | R_on, Y_on, G_on = False, False, False 28 | 29 | while True: 30 | success, img = cap.read() 31 | img = cv2.flip(img, 1) 32 | img = detector.findHands(img) 33 | fps, img = fpsReader.update(img) 34 | lmList, bboxInfo = detector.findPosition(img) 35 | 36 | if lmList : 37 | x, y = 100, 100 38 | w, h = 225, 225 39 | X, Y = 120, 190 40 | 41 | fx, fy = lmList[8][0], lmList[8][1] #index fingertip 42 | posFinger = [fx, fy] 43 | cv2.circle(img, (fx, fy), 15, (255, 0, 255), cv2.FILLED) #draw circle on index fingertip 44 | # cv2.putText(img, str(posFinger), (fx+10, fy-10), cv2.FONT_HERSHEY_PLAIN, 2, (255, 255, 0), 3) 45 | # cv2.line(img, (0, fy), (x_max, fy), (255,255,0), 2) # x line 46 | # cv2.line(img, (fx, y_max), (fx, 0), (255, 255, 0), 2)# y line 47 | 48 | if x < fx < x + w - 95 and y < fy < y + h - 95: 49 | counter_R += 1 50 | cv2.rectangle(img, (x, y), (w, h), (255, 255, 0), cv2.FILLED) 51 | if counter_R == 1: 52 | R_on = not R_on 53 | else: 54 | counter_R = 0 55 | if R_on: 56 | R_val = 1 57 | cv2.rectangle(img, (x, y), (w, h), (0, 0, 255), cv2.FILLED) 58 | cv2.putText(img, "ON", (X, Y), cv2.FONT_HERSHEY_PLAIN, 59 | 4, (255, 255, 255), 5) 60 | else: 61 | R_val = 0 62 | cv2.rectangle(img, (x, y), (w, h), (150, 150, 150), cv2.FILLED) 63 | cv2.putText(img, "OFF", (X - 15, Y), cv2.FONT_HERSHEY_PLAIN, 64 | 4, (0, 0, 255), 5) 65 | 66 | if x + 250 < fx < x + 155 + w and y < fy < y + h - 95: # 155 = 250 - 95 67 | counter_Y += 1 68 | cv2.rectangle(img, (x + 250, y), (w + 250, h), (255, 255, 0), cv2.FILLED) 69 | if counter_Y == 1: 70 | Y_on = not Y_on 71 | else: 72 | counter_Y = 0 73 | if Y_on: 74 | Y_val = 1 75 | cv2.rectangle(img, (x + 250, y), (w + 250, h), (0, 255, 255), cv2.FILLED) 76 | cv2.putText(img, "ON", (X + 250, Y), cv2.FONT_HERSHEY_PLAIN, 77 | 4, (255, 255, 255), 5) 78 | else: 79 | Y_val = 0 80 | cv2.rectangle(img, (x + 250, y), (w + 250, h), (150, 150, 150), cv2.FILLED) 81 | cv2.putText(img, "OFF", (X - 15 + 250, Y), cv2.FONT_HERSHEY_PLAIN, 82 | 4, (0, 255, 255), 5) 83 | 84 | if x + 500 < fx < x + 405 + w and y < fy < y + h - 95: # 500 - 95 = 405 85 | counter_G += 1 86 | cv2.rectangle(img, (x + 500, y), (w + 500, h), (255, 255, 0), cv2.FILLED) 87 | if counter_G == 1: 88 | G_on = not G_on 89 | 90 | else: 91 | counter_G = 0 92 | if G_on: 93 | G_val = 1 94 | cv2.rectangle(img, (x + 500, y), (w + 500, h), (0, 255, 0), cv2.FILLED) 95 | cv2.putText(img, "ON", (X + 500, Y), cv2.FONT_HERSHEY_PLAIN, 96 | 4, (255, 255, 255), 5) 97 | else: 98 | G_val = 0 99 | cv2.rectangle(img, (x + 500, y), (w + 500, h), (150, 150, 150), cv2.FILLED) 100 | cv2.putText(img, "OFF", (X - 15 + 500, Y), cv2.FONT_HERSHEY_PLAIN, 101 | 4, (0, 255, 0), 5) 102 | 103 | val = str(R_val) + str(Y_val) + str(G_val) 104 | client.publish("RizkyProject/fingersup", val) # publish 105 | 106 | 107 | cv2.imshow("Image", img) 108 | cv2.waitKey(1) 109 | 110 | --------------------------------------------------------------------------------