├── .gitignore ├── ESP32CamObjectDetection ├── ESP32CamObjectDetection.ino ├── camera_pins.h ├── camera_wrap.cpp ├── camera_wrap.h └── remote │ ├── imageThread.py │ ├── main.py │ ├── remoteQt.py │ ├── remoteQtUI.py │ ├── remoteQtUI.ui │ ├── requirementVirtualenv.txt │ ├── setting.ini │ ├── ssdlite_mobilenet_v2_quantized.tflite │ ├── ssdmobilenetv2lite.py │ ├── test_images │ └── image_info.txt │ ├── utils.py │ └── yolov3_keras.py ├── LICENSE └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | *.jpg 2 | __pycache__ -------------------------------------------------------------------------------- /ESP32CamObjectDetection/ESP32CamObjectDetection.ino: -------------------------------------------------------------------------------- 1 | /* 2 | BSD 2-Clause License 3 | 4 | Copyright (c) 2020, longpth 5 | All rights reserved. 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are met: 9 | 10 | 1. Redistributions of source code must retain the above copyright notice, this 11 | list of conditions and the following disclaimer. 12 | 13 | 2. Redistributions in binary form must reproduce the above copyright notice, 14 | this list of conditions and the following disclaimer in the documentation 15 | and/or other materials provided with the distribution. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 21 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 25 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | */ 28 | 29 | #include 30 | #include 31 | #include 32 | #include "camera_wrap.h" 33 | 34 | #define DEBUG 35 | // #define SAVE_IMG 36 | 37 | const char* ssid = "your_ssid"; //replace with your wifi ssid 38 | const char* password = "your_password"; //replace with your wifi password 39 | //holds the current upload 40 | int cameraInitState = -1; 41 | uint8_t* jpgBuff = new uint8_t[68123]; 42 | size_t jpgLength = 0; 43 | //Creating UDP Listener Object. 44 | WiFiUDP UDPServer; 45 | unsigned int UDPPort = 6868; 46 | IPAddress addrRemote; 47 | int portRemote; 48 | 49 | // Use WiFiClient class to create TCP connections 50 | WiFiClient tcpClient; 51 | bool clientConnected = false; 52 | 53 | WebSocketsServer webSocket = WebSocketsServer(86); 54 | 55 | const int RECVLENGTH = 8; 56 | byte packetBuffer[RECVLENGTH]; 57 | 58 | unsigned long previousMillis = 0; 59 | unsigned long previousMillisServo = 0; 60 | const unsigned long interval = 30; 61 | const unsigned long intervalServo = 100; 62 | 63 | bool bStream = false; 64 | int debugCnt=0; 65 | 66 | bool reqLeft = false; 67 | bool reqRight = false; 68 | bool reqFw = false; 69 | bool reqBw = false; 70 | 71 | const int PIN_SERVO_YAW = 12; 72 | const int PIN_SERVO_PITCH = 2; 73 | const int LED_BUILTIN = 4; 74 | const int SERVO_RESOLUTION = 16; 75 | int ledState = LOW; 76 | 77 | int posYaw = 90; 78 | int posPitch = 30; 79 | int delta = 1; 80 | const int angleMax = 180; 81 | uint8_t camNo = 0; 82 | 83 | void processData(){ 84 | int cb = UDPServer.parsePacket(); 85 | if (cb) { 86 | UDPServer.read(packetBuffer, RECVLENGTH); 87 | addrRemote = UDPServer.remoteIP(); 88 | portRemote = UDPServer.remotePort(); 89 | 90 | String strPackage = String((const char*)packetBuffer); 91 | #ifdef DEBUG 92 | Serial.print("receive: "); 93 | // for (int y = 0; y < RECVLENGTH; y++){ 94 | // Serial.print(packetBuffer[y]); 95 | // Serial.print("\n"); 96 | // } 97 | Serial.print(strPackage); 98 | Serial.print(" from: "); 99 | Serial.println(addrRemote); 100 | #endif 101 | if(strPackage.equals("whoami")){ 102 | UDPServer.beginPacket(addrRemote, portRemote); 103 | String res = "ESP32-CAM"; 104 | UDPServer.write((const uint8_t*)res.c_str(),res.length()); 105 | UDPServer.endPacket(); 106 | Serial.println("response"); 107 | }else if(strPackage.equals("fwon")){ 108 | reqFw = true; 109 | }else if(strPackage.equals("bwon")){ 110 | reqBw = true; 111 | }else if(strPackage.equals("leon")){ 112 | reqLeft = true; 113 | }else if(strPackage.equals("rion")){ 114 | reqRight = true; 115 | }else if(strPackage.equals("fwoff")){ 116 | reqFw = false; 117 | }else if(strPackage.equals("bwoff")){ 118 | reqBw = false; 119 | }else if(strPackage.equals("leoff")){ 120 | reqLeft = false; 121 | }else if(strPackage.equals("rioff")){ 122 | reqRight = false; 123 | } 124 | } 125 | memset(packetBuffer, 0, RECVLENGTH); 126 | } 127 | 128 | void servoWrite(uint8_t channel, uint8_t angle) { 129 | // regarding the datasheet of sg90 servo, pwm period is 20 ms and duty is 1->2ms 130 | uint32_t maxDuty = (pow(2,SERVO_RESOLUTION)-1)/10; 131 | uint32_t minDuty = (pow(2,SERVO_RESOLUTION)-1)/20; 132 | uint32_t duty = (maxDuty-minDuty)*angle/180 + minDuty; 133 | ledcWrite(channel, duty); 134 | } 135 | 136 | void controlServos(){ 137 | if(reqFw){ 138 | if(posPitch<60){ 139 | posPitch += 1; 140 | } 141 | } 142 | if(reqBw){ 143 | if(posPitch>0){ 144 | posPitch -= 1; 145 | } 146 | } 147 | if(reqLeft){ 148 | if(posYaw<180){ 149 | posYaw += 1; 150 | } 151 | } 152 | if(reqRight){ 153 | if(posYaw>0){ 154 | posYaw -= 1; 155 | } 156 | } 157 | 158 | servoWrite(2,posPitch); 159 | servoWrite(4,posYaw); 160 | } 161 | 162 | void webSocketEvent(uint8_t num, WStype_t type, uint8_t * payload, size_t length) { 163 | 164 | switch(type) { 165 | case WStype_DISCONNECTED: 166 | Serial.printf("[%u] Disconnected!\n", num); 167 | camNo = num; 168 | clientConnected = false; 169 | break; 170 | case WStype_CONNECTED: 171 | Serial.printf("[%u] Connected!\n", num); 172 | clientConnected = true; 173 | break; 174 | case WStype_TEXT: 175 | case WStype_BIN: 176 | case WStype_ERROR: 177 | case WStype_FRAGMENT_TEXT_START: 178 | case WStype_FRAGMENT_BIN_START: 179 | case WStype_FRAGMENT: 180 | case WStype_FRAGMENT_FIN: 181 | Serial.println(type); 182 | break; 183 | } 184 | } 185 | 186 | void setup(void) { 187 | Serial.begin(115200); 188 | Serial.print("\n"); 189 | #ifdef DEBUG 190 | Serial.setDebugOutput(true); 191 | #endif 192 | 193 | cameraInitState = initCamera(); 194 | 195 | Serial.printf("camera init state %d\n", cameraInitState); 196 | 197 | // pinMode(LED_BUILTIN, OUTPUT); 198 | 199 | if(cameraInitState != 0){ 200 | // digitalWrite(LED_BUILTIN, HIGH); 201 | return; 202 | } 203 | 204 | //WIFI INIT 205 | Serial.printf("Connecting to %s\n", ssid); 206 | if (String(WiFi.SSID()) != String(ssid)) { 207 | WiFi.mode(WIFI_STA); 208 | WiFi.begin(ssid, password); 209 | } 210 | 211 | while (WiFi.status() != WL_CONNECTED) { 212 | delay(500); 213 | // if the LED is off turn it on and vice-versa: 214 | if (ledState == LOW) { 215 | ledState = HIGH; 216 | } else { 217 | ledState = LOW; 218 | } 219 | 220 | // set the LED with the ledState of the variable: 221 | // digitalWrite(LED_BUILTIN, ledState); 222 | Serial.print("."); 223 | } 224 | // digitalWrite(LED_BUILTIN, LOW); 225 | Serial.println(""); 226 | Serial.print("Connected! IP address: "); 227 | Serial.println(WiFi.localIP()); 228 | 229 | UDPServer.begin(UDPPort); 230 | webSocket.begin(); 231 | webSocket.onEvent(webSocketEvent); 232 | 233 | // 1. 50hz ==> period = 20ms (sg90 servo require 20ms pulse, duty cycle is 1->2ms: -90=>90degree) 234 | // 2. resolution = 16, maximum value is 2^16-1=65535 235 | // From 1 and 2 => -90=>90 degree or 0=>180degree ~ 3276=>6553 236 | ledcSetup(4, 50, SERVO_RESOLUTION);//channel, freq, resolution 237 | ledcAttachPin(PIN_SERVO_YAW, 4);// pin, channel 238 | 239 | ledcSetup(2, 50, SERVO_RESOLUTION);//channel, freq, resolution 240 | ledcAttachPin(PIN_SERVO_PITCH, 2);// pin, channel 241 | } 242 | 243 | void loop(void) { 244 | webSocket.loop(); 245 | if(clientConnected == true){ 246 | grabImage(jpgLength, jpgBuff); 247 | webSocket.sendBIN(camNo, jpgBuff, jpgLength); 248 | // Serial.print("send img: "); 249 | // Serial.println(jpgLength); 250 | } 251 | unsigned long currentMillis = millis(); 252 | if (currentMillis - previousMillis >= interval) { 253 | previousMillis = currentMillis; 254 | processData(); 255 | } 256 | if (currentMillis - previousMillisServo >= intervalServo) { 257 | previousMillisServo = currentMillis; 258 | controlServos(); 259 | } 260 | } 261 | -------------------------------------------------------------------------------- /ESP32CamObjectDetection/camera_pins.h: -------------------------------------------------------------------------------- 1 | 2 | #if defined(CAMERA_MODEL_WROVER_KIT) 3 | #define PWDN_GPIO_NUM -1 4 | #define RESET_GPIO_NUM -1 5 | #define XCLK_GPIO_NUM 21 6 | #define SIOD_GPIO_NUM 26 7 | #define SIOC_GPIO_NUM 27 8 | 9 | #define Y9_GPIO_NUM 35 10 | #define Y8_GPIO_NUM 34 11 | #define Y7_GPIO_NUM 39 12 | #define Y6_GPIO_NUM 36 13 | #define Y5_GPIO_NUM 19 14 | #define Y4_GPIO_NUM 18 15 | #define Y3_GPIO_NUM 5 16 | #define Y2_GPIO_NUM 4 17 | #define VSYNC_GPIO_NUM 25 18 | #define HREF_GPIO_NUM 23 19 | #define PCLK_GPIO_NUM 22 20 | 21 | #elif defined(CAMERA_MODEL_ESP_EYE) 22 | #define PWDN_GPIO_NUM -1 23 | #define RESET_GPIO_NUM -1 24 | #define XCLK_GPIO_NUM 4 25 | #define SIOD_GPIO_NUM 18 26 | #define SIOC_GPIO_NUM 23 27 | 28 | #define Y9_GPIO_NUM 36 29 | #define Y8_GPIO_NUM 37 30 | #define Y7_GPIO_NUM 38 31 | #define Y6_GPIO_NUM 39 32 | #define Y5_GPIO_NUM 35 33 | #define Y4_GPIO_NUM 14 34 | #define Y3_GPIO_NUM 13 35 | #define Y2_GPIO_NUM 34 36 | #define VSYNC_GPIO_NUM 5 37 | #define HREF_GPIO_NUM 27 38 | #define PCLK_GPIO_NUM 25 39 | 40 | #elif defined(CAMERA_MODEL_M5STACK_PSRAM) 41 | #define PWDN_GPIO_NUM -1 42 | #define RESET_GPIO_NUM 15 43 | #define XCLK_GPIO_NUM 27 44 | #define SIOD_GPIO_NUM 25 45 | #define SIOC_GPIO_NUM 23 46 | 47 | #define Y9_GPIO_NUM 19 48 | #define Y8_GPIO_NUM 36 49 | #define Y7_GPIO_NUM 18 50 | #define Y6_GPIO_NUM 39 51 | #define Y5_GPIO_NUM 5 52 | #define Y4_GPIO_NUM 34 53 | #define Y3_GPIO_NUM 35 54 | #define Y2_GPIO_NUM 32 55 | #define VSYNC_GPIO_NUM 22 56 | #define HREF_GPIO_NUM 26 57 | #define PCLK_GPIO_NUM 21 58 | 59 | #elif defined(CAMERA_MODEL_M5STACK_WIDE) 60 | #define PWDN_GPIO_NUM -1 61 | #define RESET_GPIO_NUM 15 62 | #define XCLK_GPIO_NUM 27 63 | #define SIOD_GPIO_NUM 22 64 | #define SIOC_GPIO_NUM 23 65 | 66 | #define Y9_GPIO_NUM 19 67 | #define Y8_GPIO_NUM 36 68 | #define Y7_GPIO_NUM 18 69 | #define Y6_GPIO_NUM 39 70 | #define Y5_GPIO_NUM 5 71 | #define Y4_GPIO_NUM 34 72 | #define Y3_GPIO_NUM 35 73 | #define Y2_GPIO_NUM 32 74 | #define VSYNC_GPIO_NUM 25 75 | #define HREF_GPIO_NUM 26 76 | #define PCLK_GPIO_NUM 21 77 | 78 | #elif defined(CAMERA_MODEL_AI_THINKER) 79 | #define PWDN_GPIO_NUM 32 80 | #define RESET_GPIO_NUM -1 81 | #define XCLK_GPIO_NUM 0 82 | #define SIOD_GPIO_NUM 26 83 | #define SIOC_GPIO_NUM 27 84 | 85 | #define Y9_GPIO_NUM 35 86 | #define Y8_GPIO_NUM 34 87 | #define Y7_GPIO_NUM 39 88 | #define Y6_GPIO_NUM 36 89 | #define Y5_GPIO_NUM 21 90 | #define Y4_GPIO_NUM 19 91 | #define Y3_GPIO_NUM 18 92 | #define Y2_GPIO_NUM 5 93 | #define VSYNC_GPIO_NUM 25 94 | #define HREF_GPIO_NUM 23 95 | #define PCLK_GPIO_NUM 22 96 | 97 | #else 98 | #error "Camera model not selected" 99 | #endif 100 | -------------------------------------------------------------------------------- /ESP32CamObjectDetection/camera_wrap.cpp: -------------------------------------------------------------------------------- 1 | #include "camera_wrap.h" 2 | 3 | // Select camera model 4 | // #define CAMERA_MODEL_WROVER_KIT 5 | //#define CAMERA_MODEL_ESP_EYE 6 | // #define CAMERA_MODEL_M5STACK_PSRAM 7 | // #define CAMERA_MODEL_M5STACK_WIDE 8 | #define CAMERA_MODEL_AI_THINKER 9 | 10 | #include "camera_pins.h" 11 | 12 | int initCamera(){ 13 | camera_config_t config; 14 | config.ledc_channel = LEDC_CHANNEL_0; 15 | config.ledc_timer = LEDC_TIMER_0; 16 | config.pin_d0 = Y2_GPIO_NUM; 17 | config.pin_d1 = Y3_GPIO_NUM; 18 | config.pin_d2 = Y4_GPIO_NUM; 19 | config.pin_d3 = Y5_GPIO_NUM; 20 | config.pin_d4 = Y6_GPIO_NUM; 21 | config.pin_d5 = Y7_GPIO_NUM; 22 | config.pin_d6 = Y8_GPIO_NUM; 23 | config.pin_d7 = Y9_GPIO_NUM; 24 | config.pin_xclk = XCLK_GPIO_NUM; 25 | config.pin_pclk = PCLK_GPIO_NUM; 26 | config.pin_vsync = VSYNC_GPIO_NUM; 27 | config.pin_href = HREF_GPIO_NUM; 28 | config.pin_sscb_sda = SIOD_GPIO_NUM; 29 | config.pin_sscb_scl = SIOC_GPIO_NUM; 30 | config.pin_pwdn = PWDN_GPIO_NUM; 31 | config.pin_reset = RESET_GPIO_NUM; 32 | config.xclk_freq_hz = 20000000; 33 | config.pixel_format = PIXFORMAT_JPEG; 34 | //init with high specs to pre-allocate larger buffers 35 | if(psramFound()){ 36 | config.frame_size = FRAMESIZE_UXGA; 37 | config.jpeg_quality = 10; 38 | config.fb_count = 2; 39 | } else { 40 | // config.frame_size = FRAMESIZE_SVGA; 41 | config.frame_size = FRAMESIZE_VGA; 42 | config.jpeg_quality = 12; 43 | config.fb_count = 1; 44 | } 45 | 46 | #if defined(CAMERA_MODEL_ESP_EYE) 47 | pinMode(13, INPUT_PULLUP); 48 | pinMode(14, INPUT_PULLUP); 49 | #endif 50 | 51 | // camera init 52 | esp_err_t err = esp_camera_init(&config); 53 | if (err != ESP_OK) { 54 | Serial.printf("Camera init failed with error 0x%x", err); 55 | return -1; 56 | } 57 | 58 | sensor_t * s = esp_camera_sensor_get(); 59 | //initial sensors are flipped vertically and colors are a bit saturated 60 | if (s->id.PID == OV3660_PID) { 61 | s->set_vflip(s, 1);//flip it back 62 | s->set_brightness(s, 1);//up the blightness just a bit 63 | s->set_saturation(s, -2);//lower the saturation 64 | } 65 | //drop down frame size for higher initial frame rate 66 | // s->set_framesize(s, FRAMESIZE_QVGA); 67 | // s->set_framesize(s, FRAMESIZE_SVGA); 68 | s->set_framesize(s, FRAMESIZE_VGA); 69 | 70 | #if defined(CAMERA_MODEL_M5STACK_WIDE) 71 | s->set_vflip(s, 1); 72 | s->set_hmirror(s, 1); 73 | #endif 74 | return 0; 75 | } 76 | 77 | esp_err_t grabImage( size_t& jpg_buf_len, uint8_t *jpg_buf){ 78 | camera_fb_t * fb = NULL; 79 | esp_err_t res = ESP_OK; 80 | fb = esp_camera_fb_get(); 81 | uint8_t *jpg_buf_tmp = NULL; 82 | if (!fb) { 83 | Serial.println("Camera capture failed"); 84 | res = ESP_FAIL; 85 | }else{ 86 | if(fb->format != PIXFORMAT_JPEG){ 87 | bool jpeg_converted = frame2jpg(fb, 80, &jpg_buf_tmp, &jpg_buf_len); 88 | memcpy(jpg_buf, jpg_buf_tmp, jpg_buf_len); 89 | fb = NULL; 90 | if(!jpeg_converted){ 91 | // Serial.println("JPEG compression failed"); 92 | res = ESP_FAIL; 93 | } 94 | } else { 95 | jpg_buf_len = fb->len; 96 | memcpy(jpg_buf, fb->buf, jpg_buf_len); 97 | // Serial.println("Image is in jpg format"); 98 | } 99 | esp_camera_fb_return(fb); 100 | } 101 | return res; 102 | } -------------------------------------------------------------------------------- /ESP32CamObjectDetection/camera_wrap.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "esp_camera.h" 4 | #include "Arduino.h" 5 | #include 6 | 7 | extern int initCamera(); 8 | extern esp_err_t grabImage( size_t& jpg_buf_len, uint8_t *jpg_buf); -------------------------------------------------------------------------------- /ESP32CamObjectDetection/remote/imageThread.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import socket 3 | import cv2 4 | import pickle 5 | from PyQt5.QtCore import QThread, pyqtSignal 6 | from time import sleep 7 | import yolov3_keras 8 | import ssdmobilenetv2lite 9 | import time 10 | import io 11 | import asyncio 12 | import select 13 | import re 14 | from configparser import ConfigParser 15 | import time 16 | from websocket import create_connection 17 | 18 | udpServerAddr = ('255.255.255.255', 6868) # replace with your network address, it usually 192.168.1.255 (255 in this case means broadcast) 19 | 20 | RECV_BUFF_SIZE = 8192*8 21 | HEADER_SIZE = 4 22 | DEBUG = False 23 | 24 | # Subclassing QThread 25 | # http://qt-project.org/doc/latest/qthread.html 26 | class ImageThread(QThread): 27 | 28 | new_image = pyqtSignal() 29 | stop_signal = pyqtSignal() 30 | pause_signal = pyqtSignal() 31 | resume_signal = pyqtSignal() 32 | 33 | def __init__(self): 34 | QThread.__init__(self) 35 | self.udp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 36 | self.udp_socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) 37 | self.udp_socket.settimeout(1) 38 | self.udp_socket.bind(("0.0.0.0",12345)) 39 | self.bufferSize = RECV_BUFF_SIZE 40 | self.isStop = False 41 | self.isPause = False 42 | self.stop_signal.connect(self.requestStop) 43 | self.pause_signal.connect(self.requestPause) 44 | self.resume_signal.connect(self.requestResume) 45 | # self.delta = 0 46 | self.initRequestCnt = 5 47 | self.frame = None 48 | 49 | configure = ConfigParser() 50 | print (configure.read('setting.ini')) 51 | print ("Sections : ", configure.sections()) 52 | if(int(configure.get('system','GPU')) == 1): 53 | self.model = yolov3_keras.yolo3_keras_model('./yolov3.h5') 54 | else: 55 | self.model = ssdmobilenetv2lite.ssdMobilenetV2('./ssdlite_mobilenet_v2_quantized.tflite') 56 | 57 | def requestStop(self): 58 | while self.initRequestCnt > 0: 59 | self.udp_socket.sendto(b'stop', udpServerAddr) 60 | self.initRequestCnt -= 1 61 | self.initRequestCnt = 5 62 | self.isStop = True 63 | 64 | def requestPause(self): 65 | while self.initRequestCnt > 0: 66 | self.udp_socket.sendto(b'stop', udpServerAddr) 67 | self.initRequestCnt -= 1 68 | self.initRequestCnt = 5 69 | self.isPause = True 70 | 71 | def requestResume(self): 72 | while self.initRequestCnt > 0: 73 | self.udp_socket.sendto(b'stream', udpServerAddr) 74 | self.initRequestCnt -= 1 75 | self.initRequestCnt = 5 76 | self.isPause = False 77 | 78 | def requestLeft(self, val): 79 | print('[DEBUG]ImageThread request left {}'.format(val)) 80 | while self.initRequestCnt > 0: 81 | if val: 82 | self.udp_socket.sendto(b'leon', udpServerAddr) 83 | else: 84 | self.udp_socket.sendto(b'leoff', udpServerAddr) 85 | self.initRequestCnt -= 1 86 | self.initRequestCnt = 5 87 | 88 | def requestRight(self, val): 89 | print('[DEBUG]ImageThread request right {}'.format(val)) 90 | while self.initRequestCnt > 0: 91 | if val: 92 | self.udp_socket.sendto(b'rion', udpServerAddr) 93 | else: 94 | self.udp_socket.sendto(b'rioff', udpServerAddr) 95 | self.initRequestCnt -= 1 96 | self.initRequestCnt = 5 97 | 98 | def requestFw(self, val): 99 | print('[DEBUG]ImageThread request backward {}'.format(val)) 100 | while self.initRequestCnt > 0: 101 | if val: 102 | self.udp_socket.sendto(b'bwon', udpServerAddr) 103 | else: 104 | self.udp_socket.sendto(b'bwoff', udpServerAddr) 105 | self.initRequestCnt -= 1 106 | self.initRequestCnt = 5 107 | 108 | def requestBw(self, val): 109 | print('[DEBUG]ImageThread request forward {}'.format(val)) 110 | while self.initRequestCnt > 0: 111 | if val: 112 | self.udp_socket.sendto(b'fwon', udpServerAddr) 113 | else: 114 | self.udp_socket.sendto(b'fwoff', udpServerAddr) 115 | self.initRequestCnt -= 1 116 | self.initRequestCnt = 5 117 | 118 | def run(self): 119 | ws = None 120 | while self.initRequestCnt > 0: 121 | try: 122 | self.udp_socket.sendto(b'whoami', udpServerAddr) 123 | data, server = self.udp_socket.recvfrom(1024) 124 | ws = create_connection("ws://{}:86/websocket".format(server[0])) 125 | break 126 | except socket.timeout: 127 | self.initRequestCnt -= 1 128 | print('REQUEST TIMED OUT') 129 | self.initRequestCnt = 5 130 | cnt = 0 131 | 132 | frame_data = b'' 133 | 134 | start_time = time.time() 135 | 136 | while not self.isStop: 137 | if self.isPause: 138 | time.sleep(0.1) 139 | continue 140 | frame_data = ws.recv() 141 | frame_stream = io.BytesIO(frame_data) 142 | frame_stream.seek(0) 143 | file_bytes = np.asarray(bytearray(frame_stream.read()), dtype=np.uint8) 144 | frame = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR) 145 | if frame is not None: 146 | self.frame = frame[:,:,::-1].copy() 147 | self.frame, boxes = self.model.do_inference(self.frame) 148 | self.fps = 1/(time.time() - start_time) 149 | self.fps = round(self.fps,2) 150 | print ("---receive and processing frame {} time: {} seconds ---".format(cnt, (time.time() - start_time))) 151 | start_time = time.time() 152 | 153 | self.new_image.emit() 154 | 155 | if DEBUG: 156 | cv2.imwrite('test_{}.jpg'.format(0), self.frame[:,:,::-1]) 157 | 158 | if ws is not None: 159 | ws.close() 160 | self.udp_socket.close() 161 | print('remote exit') 162 | 163 | def getImage(self): 164 | return self.frame, self.fps 165 | -------------------------------------------------------------------------------- /ESP32CamObjectDetection/remote/main.py: -------------------------------------------------------------------------------- 1 | from PyQt5 import QtCore, QtGui, QtWidgets 2 | from PyQt5.QtWidgets import QApplication 3 | import sys 4 | import remoteQt 5 | 6 | class ExampleApp(QtWidgets.QMainWindow, remoteQt.Ui_MainWindow): 7 | def __init__(self, parent=None): 8 | super(ExampleApp, self).__init__(parent) 9 | self.setupUi(self) 10 | 11 | def main(): 12 | app = QApplication(sys.argv) 13 | form = ExampleApp() 14 | form.show() 15 | app.exec_() 16 | 17 | if __name__ == '__main__': 18 | main() 19 | -------------------------------------------------------------------------------- /ESP32CamObjectDetection/remote/remoteQt.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Form implementation generated from reading ui file 'remoteQt.ui' 4 | # 5 | # Created by: PyQt5 UI code generator 5.10.1 6 | # 7 | # WARNING! All changes made in this file will be lost! 8 | 9 | import PyQt5 10 | from PyQt5 import QtCore, QtGui, QtWidgets 11 | from PyQt5.QtGui import QPixmap, QImage 12 | from PyQt5.QtCore import QThread, pyqtSignal, QRectF 13 | from time import sleep 14 | import asyncore 15 | import numpy as np 16 | import pickle 17 | from imageThread import ImageThread 18 | import cv2 19 | import time 20 | 21 | DEBUG = False 22 | 23 | class Ui_MainWindow(object): 24 | 25 | def __init__(self): 26 | self.setupUi(self) 27 | 28 | self.ImageThread = ImageThread() 29 | self.ImageThread.new_image.connect(self.viewImage) 30 | 31 | def setupUi(self, MainWindow): 32 | MainWindow.setObjectName("MainWindow") 33 | MainWindow.resize(1305, 913) 34 | MainWindow.setFocusPolicy(QtCore.Qt.StrongFocus) 35 | self.centralwidget = QtWidgets.QWidget(MainWindow) 36 | self.centralwidget.setObjectName("centralwidget") 37 | self.fwButton = QtWidgets.QPushButton(self.centralwidget) 38 | self.fwButton.setGeometry(QtCore.QRect(640, 700, 51, 41)) 39 | self.fwButton.setAutoRepeat(False) 40 | self.fwButton.setAutoRepeatInterval(10) 41 | self.fwButton.setObjectName("fwButton") 42 | self.bwButton = QtWidgets.QPushButton(self.centralwidget) 43 | self.bwButton.setGeometry(QtCore.QRect(640, 820, 51, 41)) 44 | self.bwButton.setObjectName("bwButton") 45 | self.leftButton = QtWidgets.QPushButton(self.centralwidget) 46 | self.leftButton.setGeometry(QtCore.QRect(570, 760, 51, 41)) 47 | self.leftButton.setObjectName("leftButton") 48 | self.rightButton = QtWidgets.QPushButton(self.centralwidget) 49 | self.rightButton.setGeometry(QtCore.QRect(710, 760, 51, 41)) 50 | self.rightButton.setObjectName("rightButton") 51 | self.streamButton = QtWidgets.QPushButton(self.centralwidget) 52 | self.streamButton.setGeometry(QtCore.QRect(630, 750, 71, 61)) 53 | self.streamButton.setObjectName("streamButton") 54 | self.label = QtWidgets.QLabel(self.centralwidget) 55 | self.label.setGeometry(QtCore.QRect(10, 10, 1271, 671)) 56 | self.label.setObjectName("label") 57 | MainWindow.setCentralWidget(self.centralwidget) 58 | self.menubar = QtWidgets.QMenuBar(MainWindow) 59 | self.menubar.setGeometry(QtCore.QRect(0, 0, 1305, 22)) 60 | self.menubar.setObjectName("menubar") 61 | MainWindow.setMenuBar(self.menubar) 62 | self.statusbar = QtWidgets.QStatusBar(MainWindow) 63 | self.statusbar.setObjectName("statusbar") 64 | MainWindow.setStatusBar(self.statusbar) 65 | self.label_2 = QtWidgets.QLabel(self.centralwidget) 66 | self.label_2.setGeometry(QtCore.QRect(0, 700, 71, 21)) 67 | self.label_2.setObjectName("label_2") 68 | 69 | self.fwButton.setCheckable(True) 70 | self.fwButton.toggled.connect(self.buttontoggled) 71 | self.bwButton.setCheckable(True) 72 | self.bwButton.toggled.connect(self.buttontoggled) 73 | self.leftButton.setCheckable(True) 74 | self.leftButton.toggled.connect(self.buttontoggled) 75 | self.rightButton.setCheckable(True) 76 | self.rightButton.toggled.connect(self.buttontoggled) 77 | self.streamButton.setCheckable(True) 78 | self.streamButton.toggled.connect(self.buttontoggled) 79 | 80 | MainWindow.setCentralWidget(self.centralwidget) 81 | self.menubar = QtWidgets.QMenuBar(MainWindow) 82 | self.menubar.setGeometry(QtCore.QRect(0, 0, 942, 22)) 83 | self.menubar.setObjectName("menubar") 84 | MainWindow.setMenuBar(self.menubar) 85 | self.statusbar = QtWidgets.QStatusBar(MainWindow) 86 | self.statusbar.setObjectName("statusbar") 87 | MainWindow.setStatusBar(self.statusbar) 88 | 89 | self.retranslateUi(MainWindow) 90 | QtCore.QMetaObject.connectSlotsByName(MainWindow) 91 | 92 | def viewImage(self): 93 | img, fps = self.ImageThread.getImage() 94 | if img is not None: 95 | img = cv2.resize(img, (self.label.size().width(), self.label.size().height())) 96 | # img = img[:, :, ::-1] 97 | height, width, channel = img.shape 98 | bytesPerLine = 3 * width 99 | self.qImg = QImage(img.data, width, height, bytesPerLine, QImage.Format_RGB888) 100 | pixMap = QPixmap.fromImage(self.qImg) 101 | self.label.setPixmap(pixMap) 102 | self.label_2.setText('FPS: {}'.format(fps)) 103 | # print('label size {}'.format(self.label.size())) 104 | # print('image size {}'.format(self.qImg.size())) 105 | 106 | if DEBUG: 107 | cv2.imwrite('test2.jpg', self.ImageThread.getImage()) 108 | 109 | def closeEvent(self, event): 110 | self.ImageThread.stop_signal.emit() 111 | self.ImageThread.wait() 112 | 113 | def retranslateUi(self, MainWindow): 114 | _translate = QtCore.QCoreApplication.translate 115 | MainWindow.setWindowTitle(_translate("MainWindow", "ControlPanel")) 116 | self.fwButton.setText(_translate("MainWindow", "FW")) 117 | self.fwButton.setShortcut(_translate("MainWindow", "W")) 118 | self.bwButton.setText(_translate("MainWindow", "BW")) 119 | self.bwButton.setShortcut(_translate("MainWindow", "S")) 120 | self.leftButton.setText(_translate("MainWindow", "LEFT")) 121 | self.leftButton.setShortcut(_translate("MainWindow", "A")) 122 | self.rightButton.setText(_translate("MainWindow", "RIGHT")) 123 | self.rightButton.setShortcut(_translate("MainWindow", "D")) 124 | self.streamButton.setText(_translate("MainWindow", "STREAM")) 125 | self.streamButton.setShortcut(_translate("MainWindow", "Q")) 126 | self.label.setText(_translate("MainWindow", "

Image View

")) 127 | self.label_2.setText(_translate("MainWindow", "

FPS:

")) 128 | 129 | def buttontoggled(self): 130 | sender = self.sender() # This is what you need 131 | if sender.isChecked(): 132 | if sender.text() == 'STREAM': 133 | print('START') 134 | self.streamButton.setChecked(True) 135 | if self.ImageThread.isPause: 136 | self.ImageThread.resume_signal.emit() 137 | else: 138 | self.ImageThread.start() 139 | if sender.text() == 'LEFT': 140 | self.ImageThread.requestLeft(True) 141 | if sender.text() == 'RIGHT': 142 | self.ImageThread.requestRight(True) 143 | if sender.text() == 'FW': 144 | self.ImageThread.requestFw(True) 145 | if sender.text() == 'BW': 146 | self.ImageThread.requestBw(True) 147 | self.statusBar().showMessage(sender.text() + ' is pressed') 148 | else: 149 | if sender.text() == 'STREAM': 150 | print('PAUSE') 151 | self.streamButton.setChecked(False) 152 | self.ImageThread.pause_signal.emit() 153 | if sender.text() == 'LEFT': 154 | self.ImageThread.requestLeft(False) 155 | if sender.text() == 'RIGHT': 156 | self.ImageThread.requestRight(False) 157 | if sender.text() == 'FW': 158 | self.ImageThread.requestFw(False) 159 | if sender.text() == 'BW': 160 | self.ImageThread.requestBw(False) 161 | self.statusBar().showMessage(sender.text() + ' is released') 162 | 163 | def onkeyPressEvent(self,event): 164 | if event.key() == ord('W') and not event.isAutoRepeat(): 165 | 166 | self.fwButton.setChecked(True) 167 | 168 | if event.key() == ord('S') and not event.isAutoRepeat(): 169 | self.bwButton.setChecked(True) 170 | 171 | if event.key() == ord('A') and not event.isAutoRepeat(): 172 | self.leftButton.setChecked(True) 173 | 174 | if event.key() == ord('D') and not event.isAutoRepeat(): 175 | self.rightButton.setChecked(True) 176 | 177 | if not event.isAutoRepeat(): 178 | print(event.key()) 179 | 180 | def onkeyReleaseEvent(self, event): 181 | if event.key() == ord('W') and not event.isAutoRepeat(): 182 | self.fwButton.setChecked(False) 183 | 184 | if event.key() == ord('S') and not event.isAutoRepeat(): 185 | self.bwButton.setChecked(False) 186 | 187 | if event.key() == ord('A') and not event.isAutoRepeat(): 188 | self.leftButton.setChecked(False) 189 | 190 | if event.key() == ord('D') and not event.isAutoRepeat(): 191 | self.rightButton.setChecked(False) 192 | 193 | if not event.isAutoRepeat(): 194 | print(event.key()) 195 | 196 | -------------------------------------------------------------------------------- /ESP32CamObjectDetection/remote/remoteQtUI.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Form implementation generated from reading ui file 'remoteQtUI.ui' 4 | # 5 | # Created by: PyQt5 UI code generator 5.14.2 6 | # 7 | # WARNING! All changes made in this file will be lost! 8 | 9 | 10 | from PyQt5 import QtCore, QtGui, QtWidgets 11 | 12 | 13 | class Ui_MainWindow(object): 14 | def setupUi(self, MainWindow): 15 | MainWindow.setObjectName("MainWindow") 16 | MainWindow.resize(1305, 913) 17 | MainWindow.setFocusPolicy(QtCore.Qt.StrongFocus) 18 | self.centralwidget = QtWidgets.QWidget(MainWindow) 19 | self.centralwidget.setObjectName("centralwidget") 20 | self.fwButton = QtWidgets.QPushButton(self.centralwidget) 21 | self.fwButton.setGeometry(QtCore.QRect(640, 700, 51, 41)) 22 | self.fwButton.setAutoRepeat(False) 23 | self.fwButton.setAutoRepeatInterval(10) 24 | self.fwButton.setObjectName("fwButton") 25 | self.bwButton = QtWidgets.QPushButton(self.centralwidget) 26 | self.bwButton.setGeometry(QtCore.QRect(640, 820, 51, 41)) 27 | self.bwButton.setObjectName("bwButton") 28 | self.leftButton = QtWidgets.QPushButton(self.centralwidget) 29 | self.leftButton.setGeometry(QtCore.QRect(570, 760, 51, 41)) 30 | self.leftButton.setObjectName("leftButton") 31 | self.rightButton = QtWidgets.QPushButton(self.centralwidget) 32 | self.rightButton.setGeometry(QtCore.QRect(710, 760, 51, 41)) 33 | self.rightButton.setObjectName("rightButton") 34 | self.streamButton = QtWidgets.QPushButton(self.centralwidget) 35 | self.streamButton.setGeometry(QtCore.QRect(630, 750, 71, 61)) 36 | self.streamButton.setObjectName("streamButton") 37 | self.label = QtWidgets.QLabel(self.centralwidget) 38 | self.label.setGeometry(QtCore.QRect(10, 10, 1271, 671)) 39 | self.label.setObjectName("label") 40 | self.label_2 = QtWidgets.QLabel(self.centralwidget) 41 | self.label_2.setGeometry(QtCore.QRect(0, 700, 71, 21)) 42 | self.label_2.setObjectName("label_2") 43 | MainWindow.setCentralWidget(self.centralwidget) 44 | self.menubar = QtWidgets.QMenuBar(MainWindow) 45 | self.menubar.setGeometry(QtCore.QRect(0, 0, 1305, 22)) 46 | self.menubar.setObjectName("menubar") 47 | MainWindow.setMenuBar(self.menubar) 48 | self.statusbar = QtWidgets.QStatusBar(MainWindow) 49 | self.statusbar.setObjectName("statusbar") 50 | MainWindow.setStatusBar(self.statusbar) 51 | 52 | self.retranslateUi(MainWindow) 53 | QtCore.QMetaObject.connectSlotsByName(MainWindow) 54 | 55 | def retranslateUi(self, MainWindow): 56 | _translate = QtCore.QCoreApplication.translate 57 | MainWindow.setWindowTitle(_translate("MainWindow", "Terminator")) 58 | self.fwButton.setText(_translate("MainWindow", "FW")) 59 | self.fwButton.setShortcut(_translate("MainWindow", "W")) 60 | self.bwButton.setText(_translate("MainWindow", "BW")) 61 | self.bwButton.setShortcut(_translate("MainWindow", "S")) 62 | self.leftButton.setText(_translate("MainWindow", "LEFT")) 63 | self.leftButton.setShortcut(_translate("MainWindow", "A")) 64 | self.rightButton.setText(_translate("MainWindow", "RIGHT")) 65 | self.rightButton.setShortcut(_translate("MainWindow", "D")) 66 | self.streamButton.setText(_translate("MainWindow", "STREAM")) 67 | self.streamButton.setShortcut(_translate("MainWindow", "Q")) 68 | self.label.setText(_translate("MainWindow", "

Image View

")) 69 | self.label_2.setText(_translate("MainWindow", "

FPS:

")) 70 | -------------------------------------------------------------------------------- /ESP32CamObjectDetection/remote/remoteQtUI.ui: -------------------------------------------------------------------------------- 1 | 2 | 3 | MainWindow 4 | 5 | 6 | 7 | 0 8 | 0 9 | 1305 10 | 913 11 | 12 | 13 | 14 | Qt::StrongFocus 15 | 16 | 17 | Terminator 18 | 19 | 20 | 21 | 22 | 23 | 640 24 | 700 25 | 51 26 | 41 27 | 28 | 29 | 30 | FW 31 | 32 | 33 | W 34 | 35 | 36 | false 37 | 38 | 39 | 10 40 | 41 | 42 | 43 | 44 | 45 | 640 46 | 820 47 | 51 48 | 41 49 | 50 | 51 | 52 | BW 53 | 54 | 55 | S 56 | 57 | 58 | 59 | 60 | 61 | 570 62 | 760 63 | 51 64 | 41 65 | 66 | 67 | 68 | LEFT 69 | 70 | 71 | A 72 | 73 | 74 | 75 | 76 | 77 | 710 78 | 760 79 | 51 80 | 41 81 | 82 | 83 | 84 | RIGHT 85 | 86 | 87 | D 88 | 89 | 90 | 91 | 92 | 93 | 630 94 | 750 95 | 71 96 | 61 97 | 98 | 99 | 100 | STREAM 101 | 102 | 103 | Q 104 | 105 | 106 | 107 | 108 | 109 | 10 110 | 10 111 | 1271 112 | 671 113 | 114 | 115 | 116 | <html><head/><body><p align="center">Image View</p></body></html> 117 | 118 | 119 | 120 | 121 | 122 | 0 123 | 700 124 | 71 125 | 21 126 | 127 | 128 | 129 | <html><head/><body><p><span style=" font-size:16pt;">FPS: </span></p></body></html> 130 | 131 | 132 | 133 | 134 | 135 | 136 | 0 137 | 0 138 | 1305 139 | 22 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | -------------------------------------------------------------------------------- /ESP32CamObjectDetection/remote/requirementVirtualenv.txt: -------------------------------------------------------------------------------- 1 | absl-py==0.9.0 2 | astor==0.8.1 3 | click==7.1.1 4 | cycler==0.10.0 5 | Flask==1.1.2 6 | gast==0.2.2 7 | gevent==20.5.0 8 | google-pasta==0.2.0 9 | greenlet==0.4.15 10 | grpcio==1.28.1 11 | h5py==2.10.0 12 | itsdangerous==1.1.0 13 | Jinja2==2.11.1 14 | Keras==2.2.4 15 | Keras-Applications==1.0.8 16 | Keras-Preprocessing==1.1.0 17 | kiwisolver==1.2.0 18 | Markdown==3.2.1 19 | MarkupSafe==1.1.1 20 | matplotlib==3.2.1 21 | numpy==1.18.2 22 | opencv-python==3.4.6.27 23 | opt-einsum==3.2.0 24 | Pillow==7.1.1 25 | pkg-resources==0.0.0 26 | protobuf==3.11.3 27 | pyparsing==2.4.7 28 | PyQt5==5.14.2 29 | PyQt5-sip==12.7.2 30 | pyserial==3.4 31 | python-dateutil==2.8.1 32 | PyYAML==5.3.1 33 | scipy==1.4.1 34 | six==1.14.0 35 | tensorboard==1.15.0 36 | tensorflow-estimator==1.15.1 37 | tensorflow-gpu==1.15.0 38 | termcolor==1.1.0 39 | websocket==0.2.1 40 | websocket-client==0.57.0 41 | Werkzeug==1.0.1 42 | wrapt==1.12.1 43 | -------------------------------------------------------------------------------- /ESP32CamObjectDetection/remote/setting.ini: -------------------------------------------------------------------------------- 1 | [system] 2 | GPU = 0 -------------------------------------------------------------------------------- /ESP32CamObjectDetection/remote/ssdlite_mobilenet_v2_quantized.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/longpth/ESP32-CAM-ObjectDetection/b2f5d0597bca6ce1378164512ab88240bd962a6c/ESP32CamObjectDetection/remote/ssdlite_mobilenet_v2_quantized.tflite -------------------------------------------------------------------------------- /ESP32CamObjectDetection/remote/ssdmobilenetv2lite.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import os 3 | import numpy as np 4 | import tensorflow as tf 5 | from tensorflow.lite.python import interpreter as interpreter_wrapper 6 | import time 7 | 8 | tf.config.threading.set_inter_op_parallelism_threads( 9 | 4 10 | ) 11 | 12 | IMAGE_MEAN = 128.0 13 | IMAGE_STD = 128.0 14 | 15 | labelsSSD = [ 16 | "person", 17 | "bicycle", 18 | "car", 19 | "motorcycle", 20 | "airplane", 21 | "bus", 22 | "train", 23 | "truck", 24 | "boat", 25 | "traffic light", 26 | "fire hydrant", 27 | "street sign", 28 | "stop sign", 29 | "parking meter", 30 | "bench", 31 | "bird", 32 | "cat", 33 | "dog", 34 | "horse", 35 | "sheep", 36 | "cow", 37 | "elephant", 38 | "bear", 39 | "zebra", 40 | "giraffe", 41 | "hat", 42 | "backpack", 43 | "umbrella", 44 | "shoe", 45 | "eye glasses", 46 | "handbag", 47 | "tie", 48 | "suitcase", 49 | "frisbee", 50 | "skis", 51 | "snowboard", 52 | "sports ball", 53 | "kite", 54 | "baseball bat", 55 | "baseball glove", 56 | "skateboard", 57 | "surfboard", 58 | "tennis racket", 59 | "bottle", 60 | "plate", 61 | "wine glass", 62 | "cup", 63 | "fork", 64 | "knife", 65 | "spoon", 66 | "bowl", 67 | "banana", 68 | "apple", 69 | "sandwich", 70 | "orange", 71 | "broccoli", 72 | "carrot", 73 | "hot dog", 74 | "pizza", 75 | "donut", 76 | "cake", 77 | "chair", 78 | "couch", 79 | "potted plant", 80 | "bed", 81 | "mirror", 82 | "dining table", 83 | "window", 84 | "desk", 85 | "toilet", 86 | "door", 87 | "tv", 88 | "laptop", 89 | "mouse", 90 | "remote", 91 | "keyboard", 92 | "cell phone", 93 | "microwave", 94 | "oven", 95 | "toaster", 96 | "sink", 97 | "refrigerator", 98 | "blender", 99 | "book", 100 | "clock", 101 | "vase", 102 | "scissors", 103 | "teddy bear", 104 | "hair drier", 105 | "toothbrush", 106 | "hair brush" 107 | ] 108 | 109 | class ssdMobilenetV2(): 110 | def __init__(self, model_path): 111 | self.interpreter = interpreter_wrapper.Interpreter(model_path=model_path) 112 | self.interpreter.allocate_tensors() 113 | 114 | self.input_details = self.interpreter.get_input_details() 115 | self.output_details = self.interpreter.get_output_details() 116 | 117 | print(self.input_details) 118 | print(self.output_details) 119 | 120 | def do_inference(self, img): 121 | orig = img.copy() 122 | resized_image = cv2.resize(orig, (300, 300), cv2.INTER_AREA) 123 | 124 | if self.input_details[0]['dtype'] == np.float32: 125 | resized_image = resized_image.astype('float32') 126 | mean_image = np.full((300,300,3), IMAGE_MEAN, dtype='float32') 127 | resized_image = (resized_image - mean_image)/IMAGE_STD 128 | 129 | resized_image = resized_image[np.newaxis,...] 130 | 131 | start_time = time.time() 132 | self.interpreter.set_tensor(self.input_details[0]['index'], resized_image) 133 | self.interpreter.invoke() 134 | print("---cnn inference time: {} seconds ---".format((time.time() - start_time))) 135 | 136 | output_data0 = self.interpreter.get_tensor(self.output_details[0]['index']) 137 | output_data1 = self.interpreter.get_tensor(self.output_details[1]['index']) 138 | output_data2 = self.interpreter.get_tensor(self.output_details[2]['index']) 139 | 140 | output_data0 = np.squeeze(output_data0) 141 | output_data1 = np.squeeze(output_data1) 142 | output_data2 = np.squeeze(output_data2) 143 | detection_boxes = output_data0 144 | 145 | indices = np.where(output_data2>=0.3) 146 | 147 | detection_boxes = detection_boxes[indices] 148 | detection_boxes [:, 0] = detection_boxes [:, 0]*orig.shape[0] 149 | detection_boxes [:, 1] = detection_boxes [:, 1]*orig.shape[1] 150 | detection_boxes [:, 2] = detection_boxes [:, 2]*orig.shape[0] 151 | detection_boxes [:, 3] = detection_boxes [:, 3]*orig.shape[1] 152 | 153 | cnt = 0 154 | for detection_box in detection_boxes: 155 | print('{} {} {} {} {}'.format(labelsSSD[int(output_data1[cnt])], detection_box[1], detection_box[0], detection_box[3], detection_box[2])) 156 | cv2.rectangle( orig, (detection_box[1],detection_box[0]), (detection_box[3],detection_box[2]), (0,255,0),3) 157 | cv2.rectangle( orig, (int(detection_box[1]), int(detection_box[0])), (int(detection_box[1]+130), int(detection_box[0]+40)), (0,255,0),-1) 158 | cv2.putText( orig, labelsSSD[int(output_data1[cnt])], (int(detection_box[1]+10), int(detection_box[0]+30)), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 0, 255), lineType=cv2.LINE_AA) 159 | cnt += 1 160 | 161 | return orig, detection_boxes 162 | 163 | def main(): 164 | 165 | # model_path = './ssdlite_mobilenet_v2.tflite' 166 | model_path = './ssdlite_mobilenet_v2_quantized.tflite' 167 | 168 | interpreter = interpreter_wrapper.Interpreter(model_path=model_path) 169 | interpreter.allocate_tensors() 170 | 171 | input_details = interpreter.get_input_details() 172 | output_details = interpreter.get_output_details() 173 | 174 | print(input_details) 175 | print(output_details) 176 | 177 | cap = cv2.VideoCapture(0) 178 | 179 | ret, orig = cap.read(0) 180 | 181 | while ret: 182 | 183 | resized_image = cv2.resize(orig, (300, 300), cv2.INTER_AREA) 184 | 185 | # print(resized_image) 186 | # resized_image = resized_image.astype('float32') 187 | # mean_image = np.full((300,300,3), IMAGE_MEAN, dtype='float32') 188 | # resized_image = (resized_image - mean_image)/IMAGE_STD 189 | 190 | resized_image = resized_image[np.newaxis,...] 191 | 192 | start = time.time() 193 | interpreter.set_tensor(input_details[0]['index'], resized_image) 194 | interpreter.invoke() 195 | end = time.time() 196 | 197 | output_data0 = interpreter.get_tensor(output_details[0]['index']) 198 | output_data1 = interpreter.get_tensor(output_details[1]['index']) 199 | output_data2 = interpreter.get_tensor(output_details[2]['index']) 200 | 201 | print(output_data0.shape) 202 | print(output_data1.shape) 203 | print(output_data2.shape) 204 | 205 | output_data0 = np.squeeze(output_data0) 206 | output_data1 = np.squeeze(output_data1) 207 | output_data2 = np.squeeze(output_data2) 208 | detection_boxes = output_data0 209 | 210 | indices = np.where(output_data2>=0.3) 211 | 212 | detection_boxes = detection_boxes[indices] 213 | detection_boxes [:, 0] = detection_boxes [:, 0]*orig.shape[0] 214 | detection_boxes [:, 1] = detection_boxes [:, 1]*orig.shape[1] 215 | detection_boxes [:, 2] = detection_boxes [:, 2]*orig.shape[0] 216 | detection_boxes [:, 3] = detection_boxes [:, 3]*orig.shape[1] 217 | 218 | cnt = 0 219 | for detection_box in detection_boxes: 220 | print('{} {} {} {} {}'.format(labelsSSD[int(output_data1[cnt])], detection_box[1], detection_box[0], detection_box[3], detection_box[2])) 221 | cv2.rectangle( orig, (detection_box[1],detection_box[0]), (detection_box[3],detection_box[2]), (255,0,0),2) 222 | # print(labelsSSD[int(output_data1[cnt])]) 223 | cv2.putText( orig, labelsSSD[int(output_data1[cnt])], (int(detection_box[1]+10), int(detection_box[0]+10)), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 0, 0), lineType=cv2.LINE_AA) 224 | cnt += 1 225 | 226 | print("time execution {}".format(end-start)) 227 | 228 | cv2.imshow('result', orig) 229 | key = cv2.waitKey(1) 230 | 231 | if key == 27: 232 | break 233 | 234 | ret, orig = cap.read(0) 235 | 236 | 237 | if __name__ == '__main__': 238 | main() 239 | 240 | -------------------------------------------------------------------------------- /ESP32CamObjectDetection/remote/test_images/image_info.txt: -------------------------------------------------------------------------------- 1 | 2 | Image provenance: 3 | image1.jpg: https://commons.wikimedia.org/wiki/File:Baegle_dwa.jpg 4 | image2.jpg: Michael Miley, 5 | https://www.flickr.com/photos/mike_miley/4678754542/in/photolist-88rQHL-88oBVp-88oC2B-88rS6J-88rSqm-88oBLv-88oBC4 6 | 7 | -------------------------------------------------------------------------------- /ESP32CamObjectDetection/remote/utils.py: -------------------------------------------------------------------------------- 1 | ####################################################### 2 | #Reference: https://github.com/experiencor/keras-yolo3# 3 | ####################################################### 4 | 5 | import numpy as np 6 | import os 7 | import cv2 8 | from scipy.special import expit 9 | 10 | class BoundBox: 11 | def __init__(self, xmin, ymin, xmax, ymax, c = None, classes = None): 12 | self.xmin = xmin 13 | self.ymin = ymin 14 | self.xmax = xmax 15 | self.ymax = ymax 16 | 17 | self.c = c 18 | self.classes = classes 19 | 20 | self.label = -1 21 | self.score = -1 22 | 23 | def get_label(self): 24 | if self.label == -1: 25 | self.label = np.argmax(self.classes) 26 | 27 | return self.label 28 | 29 | def get_score(self): 30 | if self.score == -1: 31 | self.score = self.classes[self.get_label()] 32 | 33 | return self.score 34 | 35 | def get_box(self): 36 | return (self.xmin, self.ymin, self.xmax, self.ymax) 37 | 38 | def _sigmoid(x): 39 | return expit(x) 40 | 41 | def _softmax(x, axis=-1): 42 | x = x - np.amax(x, axis, keepdims=True) 43 | e_x = np.exp(x) 44 | 45 | return e_x / e_x.sum(axis, keepdims=True) 46 | 47 | def preprocess_input(img, w, h): 48 | 49 | ih, iw, _ = img.shape 50 | scale = min(w/iw, h/ih) 51 | nw = int(iw*scale) 52 | nh = int(ih*scale) 53 | image_data = cv2.resize(img, (nw,nh)) 54 | new_image = np.full((h,w,3), (128,128,128), dtype='uint8') 55 | new_image[(h-nh)//2 : (h+nh)//2, (w-nw)//2:(w+nw)//2] = image_data 56 | image_data = new_image.astype('float')/255.0 57 | image_data = image_data[np.newaxis, ...] 58 | 59 | return image_data 60 | 61 | def decode_netout(netout, anchors, obj_thresh, net_h, net_w): 62 | grid_h, grid_w = netout.shape[:2] 63 | nb_box = 3 64 | netout = netout.reshape((grid_h, grid_w, nb_box, -1)) 65 | nb_class = netout.shape[-1] - 5 66 | 67 | boxes = [] 68 | 69 | netout[..., :2] = _sigmoid(netout[..., :2]) 70 | netout[..., 4] = _sigmoid(netout[..., 4]) 71 | netout[..., 5:] = netout[..., 4][..., np.newaxis] * _softmax(netout[..., 5:]) 72 | netout[..., 5:] *= netout[..., 5:] > obj_thresh 73 | 74 | for i in range(grid_h*grid_w): 75 | row = i // grid_w 76 | col = i % grid_w 77 | 78 | for b in range(nb_box): 79 | # 4th element is objectness score 80 | objectness = netout[row, col, b, 4] 81 | 82 | if(objectness <= obj_thresh): continue 83 | 84 | # first 4 elements are x, y, w, and h 85 | x, y, w, h = netout[row,col,b,:4] 86 | 87 | x = (col + x) / grid_w # center position, unit: image width 88 | y = (row + y) / grid_h # center position, unit: image height 89 | w = anchors[2 * b + 0] * np.exp(w) / net_w # unit: image width 90 | h = anchors[2 * b + 1] * np.exp(h) / net_h # unit: image height 91 | 92 | # last elements are class probabilities 93 | classes = netout[row,col,b,5:] 94 | 95 | box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, objectness, classes) 96 | 97 | boxes.append(box) 98 | 99 | return boxes 100 | 101 | def do_nms(boxes, nms_thresh): 102 | if len(boxes) > 0: 103 | nb_class = len(boxes[0].classes) 104 | else: 105 | return [] 106 | 107 | for c in range(nb_class): 108 | sorted_indices = np.argsort([-box.classes[c] for box in boxes]) 109 | 110 | for i in range(len(sorted_indices)): 111 | index_i = sorted_indices[i] 112 | 113 | if boxes[index_i].classes[c] == 0: continue 114 | 115 | for j in range(i+1, len(sorted_indices)): 116 | index_j = sorted_indices[j] 117 | 118 | if bbox_iou(boxes[index_i], boxes[index_j]) >= nms_thresh: 119 | boxes[index_j].classes[c] = 0 120 | 121 | return boxes 122 | 123 | def get_yolo_boxes(model, images, net_h, net_w, anchors, obj_thresh, nms_thresh): 124 | image_h, image_w, _ = images[0].shape 125 | nb_images = len(images) 126 | batch_input = np.zeros((nb_images, net_h, net_w, 3)) 127 | 128 | # preprocess the input 129 | for i in range(nb_images): 130 | batch_input[i] = preprocess_input(images[i], net_h, net_w) 131 | 132 | # run the prediction 133 | batch_output = model.predict_on_batch(batch_input) 134 | batch_boxes = [None]*nb_images 135 | 136 | for i in range(nb_images): 137 | yolos = [batch_output[0][i], batch_output[1][i], batch_output[2][i]] 138 | boxes = [] 139 | 140 | # decode the output of the network 141 | for j in range(len(yolos)): 142 | yolo_anchors = anchors[(2-j)*6:(3-j)*6] # config['model']['anchors'] 143 | boxes += decode_netout(yolos[j], yolo_anchors, obj_thresh, net_h, net_w) 144 | 145 | # correct the sizes of the bounding boxes 146 | correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w) 147 | 148 | # suppress non-maximal boxes 149 | do_nms(boxes, nms_thresh) 150 | 151 | batch_boxes[i] = boxes 152 | 153 | return batch_boxes 154 | 155 | def correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w): 156 | if (float(net_w)/image_w) < (float(net_h)/image_h): 157 | new_w = net_w 158 | new_h = (image_h*net_w)/image_w 159 | else: 160 | new_h = net_w 161 | new_w = (image_w*net_h)/image_h 162 | 163 | for i in range(len(boxes)): 164 | x_offset, x_scale = (net_w - new_w)/2./net_w, float(new_w)/net_w 165 | y_offset, y_scale = (net_h - new_h)/2./net_h, float(new_h)/net_h 166 | 167 | boxes[i].xmin = int((boxes[i].xmin - x_offset) / x_scale * image_w) 168 | boxes[i].xmax = int((boxes[i].xmax - x_offset) / x_scale * image_w) 169 | boxes[i].ymin = int((boxes[i].ymin - y_offset) / y_scale * image_h) 170 | boxes[i].ymax = int((boxes[i].ymax - y_offset) / y_scale * image_h) 171 | return boxes 172 | 173 | def compute_overlap(a, b): 174 | """ 175 | Code originally from https://github.com/rbgirshick/py-faster-rcnn. 176 | Parameters 177 | ---------- 178 | a: (N, 4) ndarray of float 179 | b: (K, 4) ndarray of float 180 | Returns 181 | ------- 182 | overlaps: (N, K) ndarray of overlap between boxes and query_boxes 183 | """ 184 | area = (b[:, 2] - b[:, 0]) * (b[:, 3] - b[:, 1]) 185 | 186 | iw = np.minimum(np.expand_dims(a[:, 2], axis=1), b[:, 2]) - np.maximum(np.expand_dims(a[:, 0], 1), b[:, 0]) 187 | ih = np.minimum(np.expand_dims(a[:, 3], axis=1), b[:, 3]) - np.maximum(np.expand_dims(a[:, 1], 1), b[:, 1]) 188 | 189 | iw = np.maximum(iw, 0) 190 | ih = np.maximum(ih, 0) 191 | 192 | ua = np.expand_dims((a[:, 2] - a[:, 0]) * (a[:, 3] - a[:, 1]), axis=1) + area - iw * ih 193 | 194 | ua = np.maximum(ua, np.finfo(float).eps) 195 | 196 | intersection = iw * ih 197 | 198 | return intersection / ua 199 | 200 | def _interval_overlap(interval_a, interval_b): 201 | x1, x2 = interval_a 202 | x3, x4 = interval_b 203 | 204 | if x3 < x1: 205 | if x4 < x1: 206 | return 0 207 | else: 208 | return min(x2,x4) - x1 209 | else: 210 | if x2 < x3: 211 | return 0 212 | else: 213 | return min(x2,x4) - x3 214 | 215 | def bbox_iou(box1, box2): 216 | intersect_w = _interval_overlap([box1.xmin, box1.xmax], [box2.xmin, box2.xmax]) 217 | intersect_h = _interval_overlap([box1.ymin, box1.ymax], [box2.ymin, box2.ymax]) 218 | 219 | intersect = intersect_w * intersect_h 220 | 221 | w1, h1 = box1.xmax-box1.xmin, box1.ymax-box1.ymin 222 | w2, h2 = box2.xmax-box2.xmin, box2.ymax-box2.ymin 223 | 224 | union = w1*h1 + w2*h2 - intersect 225 | 226 | return float(intersect) / union 227 | -------------------------------------------------------------------------------- /ESP32CamObjectDetection/remote/yolov3_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import keras 4 | import utils 5 | import glob 6 | import os 7 | from keras.models import load_model 8 | import time 9 | import tensorflow as tf 10 | 11 | from keras.backend.tensorflow_backend import set_session 12 | config = tf.ConfigProto() 13 | config.gpu_options.per_process_gpu_memory_fraction = 0.7 14 | set_session(tf.Session(config=config)) 15 | 16 | labelsCoco = [ 17 | "person", 18 | "bicycle", 19 | "car", 20 | "motorbike", 21 | "aeroplane", 22 | "bus", 23 | "train", 24 | "truck", 25 | "boat", 26 | "traffic_light", 27 | "fire_hydrant", 28 | "stop_sign", 29 | "parking_meter", 30 | "bench", 31 | "bird", 32 | "cat", 33 | "dog", 34 | "horse", 35 | "sheep", 36 | "cow", 37 | "elephant", 38 | "bear", 39 | "zebra", 40 | "giraffe", 41 | "backpack", 42 | "umbrella", 43 | "handbag", 44 | "tie", 45 | "suitcase", 46 | "frisbee", 47 | "skis", 48 | "snowboard", 49 | "sports_ball", 50 | "kite", 51 | "baseball_bat", 52 | "baseball_glove", 53 | "skateboard", 54 | "surfboard", 55 | "tennis_racket", 56 | "bottle", 57 | "wine_glass", 58 | "cup", 59 | "fork", 60 | "knife", 61 | "spoon", 62 | "bowl", 63 | "banana", 64 | "apple", 65 | "sandwich", 66 | "orange", 67 | "broccoli", 68 | "carrot", 69 | "hot_dog", 70 | "pizza", 71 | "donut", 72 | "cake", 73 | "chair", 74 | "sofa", 75 | "pottedplant", 76 | "bed", 77 | "diningtable", 78 | "toilet", 79 | "tvmonitor", 80 | "laptop", 81 | "mouse", 82 | "remote", 83 | "keyboard", 84 | "cell_phone", 85 | "microwave", 86 | "oven", 87 | "toaster", 88 | "sink", 89 | "refrigerator", 90 | "book", 91 | "clock", 92 | "vase", 93 | "scissors", 94 | "teddy_bear", 95 | "hair_drier", 96 | "toothbrush" 97 | ] 98 | 99 | # anchors = [[81,82, 135,169, 344,319], [23,27, 37,58, 81,82]] #tiny 100 | anchors = [[116,90, 156,198, 373,326], [30,61, 62,45, 59,119], [10,13, 16,30, 33,23]] 101 | 102 | obj_threshold = 0.5 103 | nms_threshold = 0.3 104 | # network size 105 | net_w, net_h = 416,416 106 | 107 | class yolo3_keras_model(): 108 | 109 | def __init__(self, model_path): 110 | self.model = load_model(model_path) 111 | self.model._make_predict_function() 112 | self.model.summary() 113 | 114 | def draw_boxes(self, boxes, img): 115 | # draw boxes onto image 116 | for box in boxes: 117 | if box.get_score() > 0: 118 | cv2.rectangle( img, (box.xmin, box.ymin), (box.xmax, box.ymax), (0,255,0),3) 119 | cv2.rectangle( img, (box.xmin, box.ymin), (box.xmin+130, box.ymin+40), (0,255,0),-1) 120 | cv2.putText( img, labelsCoco[box.get_label()], (box.get_box()[0]+10, box.get_box()[1]+30), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 0, 255), lineType=cv2.LINE_AA) 121 | return img 122 | 123 | def do_inference(self, img): 124 | 125 | # image size 126 | ih, iw, _ = img.shape 127 | 128 | # preprocess input image 129 | img_rgb = img[:,:,::-1] 130 | image_data = utils.preprocess_input(img_rgb, net_w, net_h) 131 | 132 | start_time = time.time() 133 | # prediction 134 | out = self.model.predict(image_data) 135 | print("---cnn inference time: {} seconds ---".format((time.time() - start_time))) 136 | out[0] = np.squeeze(out[0]) 137 | out[1] = np.squeeze(out[1]) 138 | out[2] = np.squeeze(out[2]) 139 | 140 | boxes = list() 141 | # for i in range(2): #tiny 142 | for i in range(3): 143 | # decode the output of the network 144 | boxes += utils.decode_netout(out[i], anchors[i], obj_threshold, 416, 416) 145 | 146 | boxes = utils.correct_yolo_boxes(boxes, ih, iw, net_w, net_h) 147 | 148 | boxes = utils.do_nms(boxes, nms_threshold) 149 | 150 | # draw boxes onto image 151 | self.draw_boxes(boxes, img) 152 | 153 | return img, boxes 154 | 155 | # def main(): 156 | # model = yolo3_keras_model('./yolov3.h5') 157 | 158 | # img_paths = glob.glob(os.path.join('/media/p4f/My Passport/02.dataset/coco/val2017','*.jpg')) 159 | 160 | # for img_path in img_paths: 161 | # print('inference on image: {}'.format(img_path)) 162 | # img = cv2.imread(img_path) 163 | # image, boxes = model.do_inference(img) 164 | # cv2.imshow('result', image) 165 | # key = cv2.waitKey(0) 166 | # if key == 27: 167 | # break 168 | 169 | def main(): 170 | model = yolo3_keras_model('./yolov3.h5') 171 | 172 | cap = cv2.VideoCapture(0) 173 | ret, img = cap.read() 174 | while ret: 175 | start_time = time.time() 176 | image, boxes = model.do_inference(img) 177 | print("--- %s seconds ---" % (time.time() - start_time)) 178 | cv2.imshow('result', image) 179 | key = cv2.waitKey(1) 180 | if key == 27: 181 | break 182 | ret, img = cap.read() 183 | 184 | if __name__ == '__main__': 185 | main() 186 | 187 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | 3 | Copyright (c) 2020, longpth 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ESP32-CAM: remoted object detection camera # 2 | 3 | ## Demonstration 4 | 5 | [![ESP32-CAM: Remote Control Object Detection Camera](http://img.youtube.com/vi/4a_r6fCYZ3U/0.jpg)](https://www.youtube.com/watch?v=4a_r6fCYZ3U "ESP32-CAM: Remote Control Object Detection Camera") 6 | 7 | ## Descrition 8 | 9 | The ESP32-CAM would stream the images to PC via WebSocket packages, then the remote software in PC would receive the images and do the object detection with Tensorflow Lite(SSD Mobilenetv2) or Keras (YOLOv3). 10 | 11 | ### Hardware setup 12 | Components are used: ESP32-CAM module, RC Servo x2, 5V Battery.
13 | 14 | ### Software setup 15 | The software contains 2 parts
16 | 1. ESP32-CAM firmware
17 | - Dependency: Download https://github.com/Links2004/arduinoWebSockets as zip file and add this library to your Arduino Libraries by Sketch/Include Library/Add Zip...
18 | - Use Arduino IDE to flash firmware to ESP32-CAM module.
19 | 2. Remote control software (Can be run on Ubuntu or Windows PC)
20 | - ubuntu: Install virtualenv
21 | ```bash 22 | sudo apt-get install python3-pip 23 | sudo pip3 install virtualenv 24 | cd ESP32CamObjectDetection/remote 25 | virtualenv -p python3 26 | source /bin/activate 27 | ()$ pip install -r requirementVirtualenv.txt 28 | ``` 29 | - windows 10: Not yet checked but it should be similar
30 | 31 | Run remote control:
32 | ```bash 33 | ()$ cd remote 34 | ()$ python3 main.py 35 | ``` 36 | Note: If your computer does not have GPU, update setting.ini by changing GPU = 0, or you can change like this to run the code in CPU.
37 | 38 | For use GPU, download the pretrained model [here](https://drive.google.com/file/d/13azCyG6wulYYfzFFTdpxxOx16kh6ysSg/view?usp=sharing) and place in the remote folder. 39 | 40 | ### CNN models reference sources 41 | SSD Mobilenetv2: 42 | https://github.com/tensorflow/models/tree/master/research/object_detection 43 | 44 | Keras-Yolov3 45 | https://github.com/qqwweee/keras-yolo3 46 | 47 | --------------------------------------------------------------------------------