├── DetectTarget.py ├── GeneralYolov3.py ├── README.md ├── UI.py └── VideoProcess.py /DetectTarget.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -* 2 | """ 3 | -------------------------------------------------------------------- 4 | File : DetectTarget.py 5 | Author : Amo 6 | Time : 2019-3-26 7 | Description : 小车检测主函数 8 | Update : -- 2019/4/11 9 | 1.优化了class MainUi 中的方法 10 | 2.提取出了带检测目标的中心坐标 11 | 3.下一步进行特定目标的跟踪 12 | 13 | -- 2019/4/12 14 | #################### 21:05 #################### 15 | 1.纠正错误思路:目标追踪不能一直做检测（效率太低）， 16 | K 帧(K = k1 + (K - k1)) 图像，需要检测 k1 帧， 17 | 剩下的 K - k1 帧由跟踪算法进行预测！ 18 | 19 | -- 2019/4/14 20 | #################### 20:20 #################### 21 | 修改思路：1.（选定）视频加载完成后截取一帧进行检测，选择"person"为目标 22 | 2.（跟踪）选定目标后用OpenCV的"CSRT"算法跟踪 23 | 3.（寻回）丢失目标后重新进行检测 24 | 25 | #################### 22:42 #################### 26 | 移植没有完成，在self.tracker.init报错！！！ 27 | pedestrian_detect_track() 逻辑结构不够明了，功能实现后进行重写 28 | 29 | -- 2019/4/15 30 | #################### 10:01 #################### 31 | 1.将 class VideoProcess 和 class GeneralYolov3 分离 32 | 2.在 github/Object-detect 创建新分支 dev 33 | #################### 10:28 #################### 34 | 3.跟踪功能实现（之前 self.tracker.init 报错是因为 self.initBB 传参错误造成） 35 | 4.在 pedestrian_detect_track() 中将坐标发送加上 36 | 5.速度太慢，可以尝试优化，寻回功能还没写 37 | #################### 15:55 #################### 38 | 6.send_msg() 优化，整体功能基本实现 39 | 40 | -------------------------------------------------------------------- 41 | """ 42 | 43 | import sys 44 | import cv2 45 | import time 46 | from imutils.video import FPS 47 | from PyQt5.QtWidgets import QApplication 48 | from PyQt5.QtCore import QTimer 49 | from UI import DetectUi 50 | from VideoProcess import VideoProcess 51 | from GeneralYolov3 import GeneralYolov3 52 | 53 | MODIFY = True 54 | 55 | DETECTORTRACK = True # 检测 or 跟踪 56 | TINYORNOT = True # 使用YOLOv3-Tiny? 57 | 58 | YOLOV3PATH = "D:/Py/YOLOv3/" # YOLO v3 配置文件目录和权重目录 59 | 60 | # OpenCV 目标跟踪工具 61 | OPENCV_OBJECT_TRACKERS = { 62 | "csrt": cv2.TrackerCSRT_create, 63 | "kcf": cv2.TrackerKCF_create, 64 | "goturn": cv2.TrackerGOTURN_create, 65 | "boosting": cv2.TrackerBoosting_create, 66 | "mil": cv2.TrackerMIL_create, 67 | "tld": cv2.TrackerTLD_create, 68 | "medianflow": cv2.TrackerMedianFlow_create, 69 | "mosse": cv2.TrackerMOSSE_create 70 | } 71 | 72 | 73 | # class MainUI 主窗口 74 | class MainUi(DetectUi): 75 | def __init__(self): 76 | super().__init__() 77 | self.tic = time.time() 78 | 79 | # 目标检测/跟踪标志位 80 | # True--> 检测 81 | # False--> 跟踪 82 | self.dect_trac = DETECTORTRACK 83 | 84 | # initialize the bounding box coordinates of the object we are going to 85 | # track 86 | self.initBB = None 87 | 88 | # 选择 tracker 89 | self.tracker = OPENCV_OBJECT_TRACKERS["csrt"]() 90 | 91 | # message 发送标志 92 | self.msg_count = 0 93 | 94 | # FPS 95 | self.fps = None 96 | 97 | # 构建神经网络对象 98 | self.yolov3 = GeneralYolov3(YOLOV3PATH, is_tiny=TINYORNOT) 99 | 100 | # 构建视频对象 101 | self.video = VideoProcess(cv2.VideoCapture(0)) 102 | time.sleep(1) 103 | 104 | # 构建定时器 105 | self.timer = QTimer(self) 106 | # 设定定时器溢出时间 107 | self.timer.start(33) 108 | 109 | # 定时器溢出时发送信号 110 | self.timer.timeout.connect(self.load_video) 111 | 112 | # 开始加载视频 113 | self.load_video() 114 | 115 | def load_video(self): 116 | """ 117 | 视频加载函数 118 | :param : None 119 | :return : None 120 | """ 121 | self.msg_count += 1 122 | self.video.capture_next_frame() 123 | self.tic = time.time() 124 | 125 | if MODIFY: 126 | # YOLOV3 + Tracker 127 | self.pedestrian_detect_track() 128 | else: 129 | # YOLOV3 130 | self.pedestrian_detect_yolo() 131 | 132 | self.videoFrame.setPixmap(self.video.convert_frame()) 133 | self.videoFrame.setScaledContents(True) 134 | 135 | print("%.4f" % (time.time() - self.tic)) 136 | 137 | ############################################################## 138 | # TODO: YOLO 检测 # 139 | # 2019/3/29 # 140 | ############################################################## 141 | def pedestrian_detect_yolo(self): 142 | """ 143 | 使用 YOLO v3 进行行人识别 144 | :return: 145 | """ 146 | results, res_find = self.predict() 147 | if res_find: 148 | label = "正在跟踪目标...\n坐标:" 149 | # self.draw_speed() 150 | coordinates = self.processing_bounding_box(results) 151 | res_coordinate = self.compute_center(coordinates) 152 | else: 153 | label = "没有找到目标，正在重新寻找..." 154 | res_coordinate = None 155 | 156 | self.send_msg(finded=res_find, 157 | coordinate=res_coordinate, 158 | text=label) 159 | 160 | def predict(self): 161 | # 创建网络输入的 4D blob. 162 | # blobFromImage()将对图像进行预处理 163 | # - 平均减法（用于帮助对抗数据集中输入图像的光照变化） 164 | # - 按某种因素进行缩放 165 | blob = cv2.dnn.blobFromImage(self.video.currentFrame.copy(), 166 | 1.0 / 255, 167 | (self.yolov3.net_width, 168 | self.yolov3.net_height), 169 | [0, 0, 0], 170 | swapRB=False, 171 | crop=False) 172 | # 设置模型的输入 blob 173 | self.yolov3.yolov3_model.setInput(blob) 174 | 175 | # 前向计算 176 | outputs = self.yolov3.yolov3_model.forward(self.yolov3.outputs_names) 177 | # 后处理 178 | results = self.yolov3.postprocess( 179 | self.video.currentFrame.copy(), outputs) 180 | 181 | return results, len(results) 182 | 183 | def processing_bounding_box(self, results): 184 | """ 185 | 处理边界框 186 | 187 | left,top------------- 188 | | | 189 | | | 190 | | | 191 | | | 192 | | | 193 | | | 194 | ---------right,bottom 195 | 196 | :param results: 197 | :return: coordinate : 边界框中心坐标 198 | """ 199 | 200 | coordinate = None 201 | coord_temp = {} 202 | 203 | for result in results: 204 | left, top, right, bottom = result["box"] 205 | cv2.rectangle(self.video.currentFrame, (left, top), 206 | (right, bottom), (255, 178, 50), 3) 207 | 208 | # coord_temp["class_id"] = results["class_id"] 209 | coord_temp["coordinate"] = (left, top, right, bottom) 210 | 211 | coordinate = coord_temp["coordinate"] 212 | 213 | return coordinate 214 | 215 | def draw_speed(self): 216 | """ 217 | 计算速率信息 # 218 | getPerfProfile() 函数返回模型的推断总时间以及 219 | 每一网络层的耗时(in layersTimes) 220 | :return: None 221 | """ 222 | t, _ = self.yolov3.yolov3_model.getPerfProfile() 223 | label = 'Inference time: %.2f ms' % \ 224 | (t * 1000.0 / cv2.getTickFrequency()) 225 | cv2.putText(self.video.currentFrame, label, (10, 465), 226 | cv2.FONT_HERSHEY_SIMPLEX, 0.5, (95, 255, 127)) 227 | 228 | def send_msg(self, **kwargs): 229 | """ 230 | 向文本浏览框发送信息 231 | :param results: 232 | :return: 233 | """ 234 | if (self.msg_count % 20 == 0) and kwargs["finded"]: 235 | self.msg_count = 0 236 | self.textBrowser.append("%s %s,%s" % (kwargs["text"], 237 | kwargs["coordinate"][0][0], 238 | kwargs["coordinate"][0][1])) 239 | elif self.msg_count % 20 == 0: 240 | self.msg_count = 0 241 | self.textBrowser.append("%s" % kwargs["text"]) 242 | 243 | def compute_center(self, coordinates): 244 | """ 245 | 计算中心坐标 246 | :param coordinates: tuple 类型 （left, top, right, bottum) 247 | :return: 248 | """ 249 | res_coord = [] 250 | if len(coordinates): 251 | # for ind in coordinates: 252 | # # 计算中心坐标 253 | # center_H = int( 254 | # (ind["coordinate"][0] + ind["coordinate"][2]) / 2) 255 | # center_V = int( 256 | # (ind["coordinate"][1] + ind["coordinate"][3]) / 2) 257 | 258 | center_H = int((coordinates[0] + coordinates[2]) / 2) 259 | 260 | center_V = int((coordinates[1] + coordinates[3]) / 2) 261 | 262 | res_coord.append((center_H, center_V)) 263 | print(res_coord[0]) 264 | 265 | cv2.circle(self.video.currentFrame, (center_H, center_V), 266 | 10, (255, 0, 0), -5) 267 | 268 | return res_coord 269 | ############################################################## 270 | # END # 271 | ############################################################## 272 | 273 | ############################################################## 274 | # TODO: YOLO 检测 + Tracker # 275 | # 2019/4/14 # 276 | ############################################################## 277 | def pedestrian_detect_track(self): 278 | if self.dect_trac: 279 | if self.msg_count == 0: 280 | pass # 加载第一帧 281 | else: 282 | # 检测 283 | results, res_find = self.predict() 284 | 285 | if res_find: # 如果检测到目标 286 | self.dect_trac = False 287 | 288 | self.initBB = self.processing_bounding_box(results) 289 | 290 | # self.initBB = (coordinates[0]["coordinate"]) 291 | 292 | self.tracker.init( 293 | self.video.currentFrame.copy(), self.initBB) 294 | 295 | self.fps = FPS().start() 296 | 297 | else: 298 | self.send_msg(finded=False, text="没有找到目标，正在重新寻找...") 299 | else: 300 | # 跟踪 301 | if self.initBB is not None: 302 | # 捕捉目标的新边界坐标 303 | (success, box) = self.tracker.update(self.video.currentFrame) 304 | 305 | # 查看是否捕捉成功 306 | if success: 307 | (x, y, w, h) = [int(v) for v in box] 308 | cv2.rectangle(self.video.currentFrame, (x, y), 309 | (x + w, y + h), (0, 255, 0), 2) 310 | 311 | res_coordinate = self.compute_center((x, y, x + w, y + h)) 312 | 313 | self.send_msg(finded=True, 314 | coordinate=res_coordinate, 315 | text="正在跟踪目标...\n坐标:") 316 | 317 | # 更新FPS 318 | self.fps.update() 319 | self.fps.stop() 320 | ############################################################## 321 | # END # 322 | ############################################################## 323 | 324 | 325 | def main(): 326 | app = QApplication(sys.argv) 327 | 328 | detect_target = MainUi() 329 | 330 | detect_target.show() 331 | 332 | sys.exit(app.exec_()) 333 | 334 | 335 | if __name__ == '__main__': 336 | main() 337 | -------------------------------------------------------------------------------- /GeneralYolov3.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -* 2 | """ 3 | -------------------------------------------------------------------- 4 | File : GeneralYlolov3.py 5 | Time : 2019-4-14 6 | Author : Amo 7 | Description : Yolo v3 模型加载、预测 8 | Update : 9 | -------------------------------------------------------------------- 10 | """ 11 | 12 | import cv2 13 | import numpy as np 14 | import os 15 | 16 | COCONAMEPATH = "D:/Py/Car/coco.names" # CoCo数据集分类类名 17 | 18 | """ 19 | class GeneralYolove 用于： 20 | - 设置网络参数 21 | - 加载 YOLO v3 模型 22 | - 加载 CoCo 分类类名 23 | - 处理分类结果 24 | """ 25 | 26 | 27 | class GeneralYolov3(object): 28 | def __init__(self, modelpath, is_tiny=False): 29 | self.conf_threshold = 0.5 # 置信度阈值 30 | self.nms_threshold = 0.4 # 非最大值抑制阈值 31 | self.net_width = 416 # 网络输入图像宽度 32 | self.net_height = 416 # 网络输入图像高度 33 | 34 | self.classes = self.get_coco_names() 35 | self.yolov3_model = self.get_yolov3_model(modelpath, is_tiny) 36 | self.outputs_names = self.get_outputs_names() 37 | 38 | def get_coco_names(self): 39 | """ 40 | 获取COCO数据集标签 41 | :return: classes : coco 分类标签 42 | """ 43 | # COCO 物体类别名 44 | classes = None 45 | with open(COCONAMEPATH, 'rt') as f: 46 | classes = f.read().rstrip('\n').split('\n') 47 | print("[INFO] COCO.names loaded ...") 48 | 49 | return classes 50 | 51 | def get_yolov3_model(self, modelpath, is_tiny): 52 | """ 53 | :param modelpath: 模型路径 54 | :param is_tiny: 加载 YOLOv3-tiny 模型或 YOLOv3 模型 55 | :return: net : 返回网络 56 | """ 57 | if is_tiny: 58 | cfg_file = os.path.join(modelpath, "yolov3-tiny.cfg") 59 | weights_file = os.path.join(modelpath, "yolov3-tiny.weights") 60 | print("[INFO] YOLOV3-tiny model loaded ...") 61 | else: 62 | cfg_file = os.path.join(modelpath, "yolov3.cfg") 63 | weights_file = os.path.join(modelpath, "yolov3.weights") 64 | print("[INFO] YOLOV3 model loaded ...") 65 | 66 | net = cv2.dnn.readNetFromDarknet(cfg_file, weights_file) 67 | net.setPreferableBackend(cv2.dnn.DNN_BACKEND_OPENCV) 68 | net.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU) 69 | 70 | return net 71 | 72 | def get_outputs_names(self): 73 | """ 74 | 获取所有网络层名 75 | :return: 76 | """ 77 | layersNames = self.yolov3_model.getLayerNames() 78 | print("[INFO] LayersNames loaded ...") 79 | # 输出网络层名，如无连接输出的网络层. 80 | return [layersNames[i[0] - 1] for i in 81 | self.yolov3_model.getUnconnectedOutLayers()] 82 | 83 | def postprocess(self, img_cv2, outputs): 84 | """ 85 | 检测结果后处理 86 | :param img_cv2: 输入图像 87 | :param outputs: 前向计算输出 88 | :return: results : 返回结果 89 | """ 90 | # 采用 NMS 移除低 confidence 的边界框 91 | img_height, img_width, _ = img_cv2.shape 92 | 93 | # 只保留高 confidence scores 的输出边界框 94 | # 将最高 score 的类别标签作为边界框的类别标签 95 | class_ids = [] 96 | confidences = [] 97 | boxes = [] 98 | for output in outputs: 99 | for detection in output: 100 | scores = detection[5:] 101 | class_id = np.argmax(scores) # 返回索引值中最大值的id 102 | # 区分出 "person" 103 | if class_id == 0: 104 | confidence = scores[class_id] 105 | # 置信度大于阈值 106 | if confidence > self.conf_threshold: 107 | center_x = int(detection[0] * img_width) 108 | center_y = int(detection[1] * img_height) 109 | width = int(detection[2] * img_width) 110 | height = int(detection[3] * img_height) 111 | left = int(center_x - width / 2) 112 | top = int(center_y - height / 2) 113 | class_ids.append(class_id) 114 | confidences.append(float(confidence)) 115 | boxes.append([left, top, width, height]) 116 | 117 | # NMS 处理， 消除 lower confidences 的冗余重叠边界框 118 | indices = cv2.dnn.NMSBoxes(boxes, 119 | confidences, 120 | self.conf_threshold, 121 | self.nms_threshold) 122 | results = [] 123 | for ind in indices: 124 | res_box = {} 125 | res_box["class_id"] = class_ids[ind[0]] 126 | res_box["score"] = confidences[ind[0]] 127 | 128 | box = boxes[ind[0]] 129 | res_box["box"] = (box[0], 130 | box[1], 131 | box[0] + box[2], 132 | box[1] + box[3]) 133 | results.append(res_box) 134 | 135 | return results 136 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Object-detecting-and-tracking 2 | 小车目标检测和跟踪 3 | -------------------------------------------------------------------------------- /UI.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -* 2 | """ 3 | -------------------------------------------------------------------- 4 | File : UI.py 5 | Time : 2019-3-26 6 | Author : Amo 7 | Description : 小车UI窗口 8 | Update : 9 | -------------------------------------------------------------------- 10 | """ 11 | 12 | from PyQt5.QtGui import QIcon, QFont 13 | from PyQt5.QtWidgets import QWidget, QTextBrowser, QLabel, QFrame 14 | 15 | 16 | class DetectUi(QWidget): 17 | def __init__(self): 18 | super().__init__() 19 | # ——————————构建视频帧显示标签——————————— # 20 | self.videoFrame = QLabel(self) 21 | # ——————————构建一个文本浏览框——————————— # 22 | self.textBrowser = QTextBrowser(self) 23 | # ——————————————设置标签——————————————— # 24 | self.label_1 = QLabel(self) 25 | # ———————————————设置边框——————————————— # 26 | self.line_1 = QFrame(self) 27 | self.line_2 = QFrame(self) 28 | self.line_3 = QFrame(self) 29 | self.line_4 = QFrame(self) 30 | self.line_5 = QFrame(self) 31 | # ———————————————构建窗口——————————————— # 32 | self.setGeometry(100, 100, 625, 400) 33 | # 构建布局 34 | self.setupUi() 35 | 36 | def setupUi(self): 37 | """ 38 | 构建窗口布局 39 | :param : None 40 | :return : None 41 | """ 42 | # ———————————————设置字体——————————————— # 43 | font = QFont() 44 | font.setPointSize(14) 45 | font.setBold(True) 46 | font.setWeight(75) 47 | 48 | self.videoFrame.setGeometry(30, 70, 320, 240) 49 | self.videoFrame.setObjectName("videoFrame") 50 | 51 | self.textBrowser.setGeometry(370, 70, 220, 240) 52 | 53 | self.label_1.setGeometry(230, 30, 180, 20) 54 | self.label_1.setFont(font) 55 | self.label_1.setText("自动跟随智能购物车") 56 | 57 | self.line_1.setGeometry(30, 70, 320, 3) 58 | self.line_1.setFrameShape(QFrame.HLine) 59 | self.line_1.setFrameShadow(QFrame.Sunken) 60 | self.line_1.setObjectName("line_1") 61 | 62 | self.line_2.setGeometry(30, 310, 320, 3) 63 | self.line_2.setFrameShape(QFrame.HLine) 64 | self.line_2.setFrameShadow(QFrame.Sunken) 65 | self.line_2.setObjectName("line_2") 66 | 67 | self.line_3.setGeometry(30, 70, 3, 240) 68 | self.line_3.setFrameShape(QFrame.VLine) 69 | self.line_3.setFrameShadow(QFrame.Sunken) 70 | self.line_3.setObjectName("line_3") 71 | 72 | self.line_4.setGeometry(350, 70, 3, 240) 73 | self.line_4.setFrameShape(QFrame.VLine) 74 | self.line_4.setFrameShadow(QFrame.Sunken) 75 | self.line_4.setObjectName("line_4") 76 | 77 | self.line_5.setGeometry(360, 70, 3, 240) 78 | self.line_5.setFrameShape(QFrame.VLine) 79 | self.line_5.setFrameShadow(QFrame.Sunken) 80 | self.line_5.setObjectName("line_5") 81 | 82 | self.setWindowTitle('自动跟随智能购物车 CQUT') # 设置窗口标题 83 | self.setWindowIcon(QIcon('Icon/cqutLog.jpg')) # 设置窗口图标 84 | -------------------------------------------------------------------------------- /VideoProcess.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -* 2 | """ 3 | -------------------------------------------------------------------- 4 | File : VideoProcess.py 5 | Time : 2019-4-14 6 | Author : Amo 7 | Description : 视频流处理 8 | Update : 9 | -------------------------------------------------------------------- 10 | """ 11 | 12 | import cv2 13 | import numpy as np 14 | from PyQt5.QtGui import QPixmap, QImage 15 | """ 16 | 视频流处理 17 | """ 18 | 19 | 20 | class VideoProcess(object): 21 | def __init__(self, capture): 22 | self.capture = capture 23 | self.currentFrame = np.array([]) 24 | self.originalFrame = np.array([]) 25 | 26 | def capture_frame(self): 27 | """ 28 | 捕获帧 29 | :param : None 30 | :return : Frame 31 | """ 32 | ret, Frame = self.capture.read() 33 | return Frame 34 | 35 | def capture_next_frame(self): 36 | """ 37 | 捕获下一帧 38 | :param : None 39 | :return : None 40 | """ 41 | ret, self.originalFrame = self.capture.read() 42 | if ret is True: 43 | self.currentFrame = cv2.cvtColor( 44 | self.originalFrame, cv2.COLOR_BGR2RGB) 45 | 46 | def convert_frame(self): 47 | """ 48 | 帧格式转换 49 | :param : None 50 | :return : img 51 | """ 52 | try: 53 | height, width = self.currentFrame.shape[:2] 54 | # 用QImage加载当前帧 55 | # 需要注意QImage参数类型 56 | img = QImage( 57 | np.ndarray.tobytes(self.currentFrame), 58 | int(width), 59 | int(height), 60 | QImage.Format_RGB888) 61 | img = QPixmap.fromImage(img) 62 | return img 63 | except BaseException: 64 | return None 65 | --------------------------------------------------------------------------------