├── .gitignore ├── screenshot.png ├── common.py ├── people_detect.py ├── README.md ├── lucas_kanade.py └── people_counter.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | *.pyc 3 | data/ 4 | -------------------------------------------------------------------------------- /screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hrnr/people-counter/HEAD/screenshot.png -------------------------------------------------------------------------------- /common.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Common utilities used though this project 3 | ''' 4 | 5 | import cv2 as cv 6 | 7 | def draw_str(dst, target, s): 8 | """ 9 | Draws text with shadow 10 | """ 11 | x, y = target 12 | cv.putText(dst, s, (x+1, y+1), cv.FONT_HERSHEY_PLAIN, 1.0, (0, 0, 0), thickness = 2, lineType=cv.LINE_AA) 13 | cv.putText(dst, s, (x, y), cv.FONT_HERSHEY_PLAIN, 1.0, (255, 255, 255), lineType=cv.LINE_AA) 14 | 15 | # some nice colors for visualisation 16 | color_palette = [ 17 | (0,0,0), 18 | (230,159,0), 19 | (86,180,223), 20 | (0,158,115), 21 | (240,228,66), 22 | (0,114,178), 23 | (213,94,0), 24 | (204,121,167), 25 | ] 26 | def randColor(n): 27 | """ 28 | Pick color from color palette based on n 29 | """ 30 | return color_palette[n % len(color_palette)] 31 | 32 | class Rect: 33 | """ 34 | 2D rectangle. reimplements Rect from OpenCV, which is not exported to Python bindings 35 | """ 36 | x = 0 37 | y = 0 38 | width = 0 39 | height = 0 40 | 41 | def __init__(self, pt1, pt2): 42 | """ 43 | Initializes Rectangle using 2 points 44 | """ 45 | self.x = min(pt1[0], pt2[0]); 46 | self.y = min(pt1[1], pt2[1]); 47 | self.width = max(pt1[0], pt2[0]) - self.x; 48 | self.height = max(pt1[1], pt2[1]) - self.y; 49 | 50 | def tl(self): 51 | """ 52 | Returns top-left corner 53 | """ 54 | return (self.x, self.y) 55 | 56 | def br(self): 57 | """ 58 | Returns bottom-right corner 59 | """ 60 | return (self.x + self.width, self.y + self.height) 61 | 62 | def area(self): 63 | """ 64 | Returns area of the rectangle 65 | """ 66 | return self.width * self.height 67 | 68 | def contains(self, pt): 69 | """ 70 | Returns true when rectangle contains given Point pt 71 | """ 72 | return self.x <= pt[0] and pt[0] < self.x + self.width and self.y <= pt[1] and pt[1] < self.y + self.height 73 | 74 | -------------------------------------------------------------------------------- /people_detect.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | 4 | """ 5 | People detector based on MobileNet-SSD. 6 | 7 | This is used by people_counter.py 8 | """ 9 | 10 | import numpy as np 11 | import cv2 as cv 12 | 13 | from common import Rect 14 | 15 | # input dimensions for MobileNet-SSD object detection network 16 | # https://github.com/chuanqi305/MobileNet-SSD/ 17 | inWidth = 300 18 | inHeight = 300 19 | WHRatio = inWidth / float(inHeight) 20 | inScaleFactor = 0.007843 21 | meanVal = 127.5 22 | 23 | # classes that network detects 24 | classNames = ['background', 25 | 'aeroplane', 'bicycle', 'bird', 'boat', 26 | 'bottle', 'bus', 'car', 'cat', 'chair', 27 | 'cow', 'diningtable', 'dog', 'horse', 28 | 'motorbike', 'person', 'pottedplant', 29 | 'sheep', 'sofa', 'train', 'tvmonitor'] 30 | 31 | class PeopleDetector: 32 | def __init__(self, proto, model, confidence): 33 | # load net for the OpenCV from caffe model and weights 34 | self.net = cv.dnn.readNetFromCaffe(proto, model) 35 | # confidence threshold for detections 36 | self.confidence = confidence 37 | # detected people 38 | self.people = [] 39 | # detected other classes than people 40 | self.other_objects = [] 41 | 42 | def update(self, image): 43 | """ 44 | Detects people and other object in the frame 45 | """ 46 | # remove previous detections 47 | self.people = [] 48 | self.other_objects = [] 49 | frame = image.copy() 50 | 51 | # normalize the image for the network input 52 | blob = cv.dnn.blobFromImage(frame, inScaleFactor, (inWidth, inHeight), (meanVal, meanVal, meanVal), False, False) 53 | # run the frame though the network 54 | self.net.setInput(blob) 55 | detections = self.net.forward() 56 | 57 | # rows and columns of the image 58 | cols = frame.shape[1] 59 | rows = frame.shape[0] 60 | 61 | # get detections with confidence higher than threshold 62 | for i in range(detections.shape[2]): 63 | # confidence for this detection 64 | confidence = detections[0, 0, i, 2] 65 | # type of detected object 66 | class_id = int(detections[0, 0, i, 1]) 67 | if confidence < self.confidence: 68 | continue 69 | if class_id >= len(classNames): 70 | # unknown object 71 | continue 72 | 73 | # bounding box coordinates 74 | xLeftBottom = int(detections[0, 0, i, 3] * cols) 75 | yLeftBottom = int(detections[0, 0, i, 4] * rows) 76 | xRightTop = int(detections[0, 0, i, 5] * cols) 77 | yRightTop = int(detections[0, 0, i, 6] * rows) 78 | # create rectangle for bounding box 79 | roi = Rect((xLeftBottom, yLeftBottom), (xRightTop, yRightTop)) 80 | 81 | # save people and other object separately 82 | if classNames[class_id] == 'person': 83 | # save roi 84 | self.people.append(roi) 85 | else: 86 | # save just class name for other objects 87 | self.other_objects.append(classNames[class_id]) 88 | 89 | def visualise(self, frame): 90 | """ 91 | Visualises detections in current frame 92 | """ 93 | # draw rectangle for each person 94 | for roi in self.people: 95 | cv.rectangle(frame, roi.tl(), roi.br(), (0, 255, 0)) 96 | 97 | cv.imshow("detections", frame) 98 | 99 | def main(): 100 | """ 101 | main method to run this as indendent script 102 | """ 103 | import sys 104 | try: 105 | video_src = sys.argv[1] 106 | except: 107 | video_src = 0 108 | 109 | print(__doc__) 110 | detector = PeopleDetector(sys.argv[2], sys.argv[3], 0.5) 111 | cam = cv.VideoCapture(video_src) 112 | while True: 113 | ret, frame = cam.read() 114 | if not ret: 115 | break 116 | detector.update(frame) 117 | detector.visualise(frame) 118 | if cv.waitKey(1) >= 0: 119 | break 120 | cam.release() 121 | cv.destroyAllWindows() 122 | 123 | if __name__ == '__main__': 124 | main() 125 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | People counter 2 | ============== 3 | 4 | Count people passing in videos. This demo uses OpenCV deep neural networks module together with optical flow to track people in images. Find more in the [technical report](https://1drv.ms/b/s!Asq_3FI_n-vzjBEKaKWhpGi50TzJ). 5 | 6 | ![screenshot from the tracker](screenshot.png) 7 | 8 | Dependencies 9 | ------------ 10 | 11 | The main dependency of the project is OpenCV. You need a recent version of OpenCV, at least 3.4.0 or latest master. OpenCV needs to be compiled with dnn module (enabled by default) since that is what we use for running neural network. 12 | 13 | The project has been tested with `opencv-python` package version 3.4.0.12 and later available from PyPi via pip at https://pypi.python.org/pypi/opencv-python It should be enough to run 14 | 15 | ``` 16 | sudo pip3 install opencv-python 17 | ``` 18 | 19 | Another dependency is numpy, but that is required anyway for OpenCV bindings. 20 | 21 | The project supports only Python 3. 22 | 23 | Running 24 | ------- 25 | 26 | The main script is `people_counter.py` the synopsis is: 27 | 28 | ``` 29 | usage: people_counter.py [-h] [--video VIDEO] [--proto PROTO] [--model MODEL] 30 | [--confidence CONFIDENCE] 31 | [--min_tracks_for_match MIN_TRACKS_FOR_MATCH] 32 | [--min_track_length MIN_TRACK_LENGTH] 33 | [--detect_interval DETECT_INTERVAL] 34 | 35 | Counting people in videos 36 | 37 | optional arguments: 38 | -h, --help show this help message and exit 39 | --video VIDEO path to video file. If empty, camera's stream will be 40 | used 41 | --proto PROTO Path to text network file: 42 | MobileNetSSD_deploy.prototxt 43 | --model MODEL Path to weights: MobileNetSSD_deploy.caffemodel 44 | --confidence CONFIDENCE 45 | confidence threshold to filter out weak detections 46 | --min_tracks_for_match MIN_TRACKS_FOR_MATCH 47 | minimum number of points that must match between 48 | detection to be considered one track 49 | --min_track_length MIN_TRACK_LENGTH 50 | minimum number of detections in one track to count one 51 | person 52 | --detect_interval DETECT_INTERVAL 53 | each detect_interval frames people detection and 54 | corner detection runs. In between people are tracked 55 | only using Lucas-Kanade method. 56 | ``` 57 | 58 | `--proto` and `--model` parameters must be provided and point to required files for neural network. The default values are `data/MobileNetSSD_deploy.prototxt` and `data/MobileNetSSD_deploy.caffemodel` respectively, which might be fine if you are going to use a provided data folder (see below). If not these files can be downloaded at https://github.com/chuanqi305/MobileNet-SSD. 59 | 60 | `--video` is video file to analyse. If not provided it will try to use the first attached camera (usually a webcam). 61 | 62 | Other parameters are optional, they are explained thoroughly in the paper. 63 | 64 | Datasets 65 | -------- 66 | 67 | `data` folder with video used for experiments and required files for neural network is available at https://1drv.ms/u/s!Asq_3FI_n-vzjA-1AtArIz9rct0y 68 | 69 | Extract the `data` folder from the archive alongside `people_counter.py` and you should be able to run examples below. Visualisation should start automatically. 70 | 71 | The examples expects a unix system with `/` as directory separator, adjust it accordingly on non-unix systems. You might also need to specify `--proto` and `--model` parameters explicitly (default values expect data folder and `/` separator). 72 | 73 | ``` 74 | ./people_counter.py --video data/P1E_S1_C1.mkv 75 | ./people_counter.py --video data/P1L_S3_C2.mkv 76 | ./people_counter.py --video data/P2E_S2_C2.2.mkv 77 | ./people_counter.py --video data/P2E_S5_C2.1.mkv 78 | ``` 79 | 80 | ``` 81 | ./people_counter.py --video data/mv2_001.mkv --detect_interval=15 82 | ./people_counter.py --video data/Venice-1.mp4 --detect_interval=15 83 | ``` 84 | 85 | ``` 86 | ./people_counter.py --video data/TUD-Campus.mp4 --detect_interval=2 87 | ./people_counter.py --video data/TUD-Crossing.mp4 --detect_interval=2 88 | ``` 89 | 90 | When video stream ends the script outputs total people passed in video to stdout. 91 | -------------------------------------------------------------------------------- /lucas_kanade.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Lucas-Kanade tracker 5 | ==================== 6 | 7 | Lucas-Kanade sparse optical flow based tracker. Uses goodFeaturesToTrack 8 | for track initialization and back-tracking for match verification 9 | between frames. 10 | 11 | This is used by people_detect.py 12 | """ 13 | 14 | import numpy as np 15 | import cv2 as cv 16 | from common import draw_str 17 | from time import clock 18 | import random 19 | 20 | class LucasKanadeTracker: 21 | def __init__(self, track_len): 22 | # how much long tracks to keep 23 | self.track_len = track_len 24 | # list of lists of tracked points 25 | self.tracks = [] 26 | # Lucas-Kanade parameters 27 | self.lk_params = dict(winSize = (15, 15), maxLevel = 2, 28 | criteria = (cv.TERM_CRITERIA_EPS | cv.TERM_CRITERIA_COUNT, 10, 0.03)) 29 | # parameters for goodFeaturesToTrack 30 | self.feature_params = dict(maxCorners = 600, qualityLevel = 0.1, minDistance = 7, blockSize = 7) 31 | 32 | def update(self, image): 33 | """ 34 | Runs fast Lucas-Kanade tracking without re-detectiing the feature points 35 | """ 36 | # convert to grayscale 37 | frame_gray = cv.cvtColor(image, cv.COLOR_BGR2GRAY) 38 | 39 | if len(self.tracks) > 0: 40 | # image to compute flow between 41 | img0, img1 = self.prev_gray, frame_gray 42 | # last known points positions (from previous frame) 43 | p0 = np.float32([tr[-1] for tr in self.tracks]).reshape(-1, 1, 2) 44 | # forward flow 45 | p1, st, err = cv.calcOpticalFlowPyrLK(img0, img1, p0, None, **self.lk_params) 46 | # backwards flow 47 | p0r, st, err = cv.calcOpticalFlowPyrLK(img1, img0, p1, None, **self.lk_params) 48 | # error between forward flow and backwards flow 49 | d = abs(p0-p0r).reshape(-1, 2).max(-1) 50 | # mask for features that have matching forward and backwards flow 51 | good = d < 1 52 | new_tracks = [] 53 | for tr, (x, y), good_flag in zip(self.tracks, p1.reshape(-1, 2), good): 54 | if not good_flag: 55 | # skip unreliable points 56 | continue 57 | # add new point to the track 58 | tr.append((x, y)) 59 | if len(tr) > self.track_len: 60 | # remove oldest point if the track is too long 61 | del tr[0] 62 | new_tracks.append(tr) 63 | # save the new tracks 64 | self.tracks = new_tracks 65 | # save the current image 66 | self.prev_gray = frame_gray 67 | 68 | def detect(self, image): 69 | """ 70 | Detects new feature points 71 | """ 72 | # convert to gray 73 | frame_gray = cv.cvtColor(image, cv.COLOR_BGR2GRAY) 74 | # initialize mask 75 | mask = np.zeros_like(frame_gray) 76 | mask[:] = 255 77 | # create mask to supress detecting features around points we already track 78 | for x, y in [np.int32(tr[-1]) for tr in self.tracks]: 79 | cv.circle(mask, (x, y), 5, 0, -1) 80 | # detect feature points using minimal eigenvalue of gradient matrices method 81 | p = cv.goodFeaturesToTrack(frame_gray, mask = mask, **self.feature_params) 82 | if p is not None: 83 | for x, y in np.float32(p).reshape(-1, 2): 84 | # add new feature point for tracking 85 | self.tracks.append([(x, y)]) 86 | 87 | def visualise(self, frame): 88 | """ 89 | Draw visualised tracks to frame 90 | """ 91 | # draw point for each feature point 92 | for track in self.tracks: 93 | cv.circle(frame, track[-1], 2, (0, 255, 0), -1) 94 | # draw path for each track 95 | cv.polylines(frame, [np.int32(tr) for tr in self.tracks], False, (0, 255, 0)) 96 | # print how many tracks we have altogether 97 | draw_str(frame, (20, 20), 'track count: %d' % len(self.tracks)) 98 | 99 | def main(): 100 | """ 101 | This can be used to run only Lucas-Kande tracker independently 102 | """ 103 | import sys 104 | try: 105 | video_src = sys.argv[1] 106 | except: 107 | video_src = 0 108 | 109 | print(__doc__) 110 | tracker = LucasKanadeTracker(100) 111 | cam = cv.VideoCapture(video_src) 112 | while True: 113 | ret, frame = cam.read() 114 | if not ret: 115 | break 116 | tracker.update(frame) 117 | tracker.detect(frame) 118 | tracker.visualise(frame) 119 | cv.imshow('lk_track', frame) 120 | if cv.waitKey(1) >= 0: 121 | break 122 | cam.release() 123 | cv.destroyAllWindows() 124 | 125 | if __name__ == '__main__': 126 | main() 127 | -------------------------------------------------------------------------------- /people_counter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Counts people in videos 5 | """ 6 | 7 | import argparse 8 | import random 9 | import time 10 | from collections import namedtuple 11 | 12 | # numpy and OpenCV required 13 | import numpy as np 14 | import cv2 as cv 15 | 16 | from people_detect import PeopleDetector 17 | from lucas_kanade import LucasKanadeTracker 18 | from common import draw_str, randColor 19 | 20 | 21 | def matchRoisFromFlow(old_roi, new_roi, tracks, step): 22 | """ 23 | Matches rois across frames using Lucas-Kanade tracks (tracked points) 24 | 25 | returns: number of matched tracks 26 | """ 27 | # number of tracks that match 28 | matched_tracks = 0 29 | for track in tracks: 30 | # we need at least tracks that are in the this frame and frame 31 | # with previous detection 32 | if len(track) < step + 1: 33 | continue 34 | # if the track goes through both rois 35 | if new_roi.contains(track[-1]) and old_roi.contains(track[-1 - step]): 36 | matched_tracks += 1 37 | return matched_tracks 38 | 39 | def main(): 40 | # accepted commanline arguments. proto and model must point to valid files 41 | parser = argparse.ArgumentParser(description='Counting people in videos') 42 | parser.add_argument("--video", help="path to video file. If empty, camera's stream will be used") 43 | parser.add_argument("--proto", default="data/MobileNetSSD_deploy.prototxt", help='Path to text network file: MobileNetSSD_deploy.prototxt') 44 | parser.add_argument("--model", default="data/MobileNetSSD_deploy.caffemodel", help='Path to weights: MobileNetSSD_deploy.caffemodel') 45 | parser.add_argument("--confidence", default=0.6, type=float, help="confidence threshold to filter out weak detections") 46 | parser.add_argument("--min_tracks_for_match", default=7, type=int, help="minimum number of points that must match between detection to be considered one track") 47 | parser.add_argument("--min_track_length", default=4, type=int, help="minimum number of detections in one track to count one person") 48 | parser.add_argument("--detect_interval", default=8, type=int, 49 | help="each detect_interval frames people detection and corner detection runs. In between people are tracked only using Lucas-Kanade method.") 50 | args = parser.parse_args() 51 | 52 | # run main counter 53 | PeopleCounter(args).run() 54 | 55 | # Stamped rectangle. Stores roi of detected person and stamp - 56 | # frame index when the dection occured 57 | RectStamped = namedtuple('RectStamped', ['roi', 'stamp']) 58 | 59 | class PeopleCounter: 60 | def __init__(self, args): 61 | # see above for parameters descriptions 62 | self.min_tracks_for_match = args.min_tracks_for_match 63 | self.detect_interval = args.detect_interval 64 | self.finish_tracking_after = self.detect_interval * 3 65 | self.min_track_length = args.min_track_length 66 | 67 | # detector for people detection 68 | self.detector = PeopleDetector(args.proto, args.model, args.confidence) 69 | # tracker providing optical flow tracks 70 | self.tracker = LucasKanadeTracker(4 * self.detect_interval) 71 | # tracker people in video 72 | self.people = [] 73 | # people that already passed in video 74 | self.count_passed = 0 75 | # current processed frame in video 76 | self.frame_idx = 0 77 | # total time in seconds the processing took 78 | self.total_time = 0.01 79 | if args.video: 80 | # use video file 81 | self.cap = cv.VideoCapture(args.video) 82 | else: 83 | # use webcam 84 | self.cap = cv.VideoCapture(0) 85 | 86 | def run(self): 87 | while True: 88 | # read frame from video source 89 | ret, frame = self.cap.read() 90 | if not ret: 91 | break 92 | 93 | # measure time 94 | start = time.clock() 95 | 96 | # optical flow update - fast 97 | self.tracker.update(frame) 98 | 99 | if self.frame_idx % self.detect_interval == 0: 100 | # detect new keypoints 101 | self.tracker.detect(frame) 102 | # detect people 103 | self.detector.update(frame) 104 | # count people 105 | self.count() 106 | 107 | # measure time 108 | stop = time.clock() 109 | self.total_time += stop - start 110 | 111 | # visualisation 112 | self.tracker.visualise(frame) 113 | self.visualise(frame) 114 | # wait for visualisation 115 | c = cv.waitKey(1) 116 | if c == 27: # esc press 117 | break 118 | elif c == 115: # s press 119 | # save visualialised image on s press 120 | cv.imwrite('vis-%s.png' % self.frame_idx, frame) 121 | 122 | self.frame_idx += 1 123 | 124 | # count also people currently in the frame 125 | self.frame_idx += self.finish_tracking_after * 2 126 | self._count_finished_tracks() 127 | # cleanup windows 128 | self.cap.release() 129 | cv.destroyAllWindows() 130 | print('passed people: ', self.count_passed) 131 | 132 | def count(self): 133 | """ 134 | Updates tracks of people in video, counts finished tracks 135 | """ 136 | self._add_new_people() 137 | self._count_finished_tracks() 138 | 139 | def _add_new_people(self): 140 | """ 141 | Add new detections to existing tracks of people of create new ones 142 | """ 143 | # find if we have new person in the image 144 | for new_person in self.detector.people: 145 | # try to match detected persons to the new ones 146 | matches = np.zeros(len(self.people)) 147 | for i,person_track in enumerate(self.people): 148 | # count tracks that match 149 | matches[i] = matchRoisFromFlow(person_track[-1].roi, new_person, 150 | self.tracker.tracks, self.frame_idx - person_track[-1].stamp) 151 | if self.people: 152 | # choose person with maximum matches 153 | max_ind = np.argmax(matches) 154 | matched_tracks = matches[max_ind] 155 | else: 156 | matched_tracks = 0 157 | 158 | if matched_tracks > self.min_tracks_for_match: 159 | # we matched an old person 160 | # add to detecction to person's track 161 | self.people[max_ind].append(RectStamped(new_person, self.frame_idx)) 162 | else: 163 | # we haven't found a matching person, lets add new one 164 | self.people.append([RectStamped(new_person, self.frame_idx)]) 165 | 166 | def _count_finished_tracks(self): 167 | """ 168 | Counts finished tracks (not updated for quite some time) 169 | """ 170 | for person_track in self.people: 171 | if self.frame_idx - person_track[-1].stamp < self.finish_tracking_after: 172 | # this track is still new 173 | continue 174 | # track is too old 175 | self.people.remove(person_track) 176 | # remove outliers - too short tracks are probably just noise 177 | if len(person_track) < self.min_track_length: 178 | continue 179 | # increase counters 180 | self.count_passed += 1 181 | 182 | def visualise(self, frame): 183 | """ 184 | Visualise tracks and optical flow 185 | """ 186 | # print statistics in top left corner 187 | draw_str(frame, (20, 40), 'passed people: %d' % self.count_passed) 188 | draw_str(frame, (20, 60), 'fps: %d' % (self.frame_idx / self.total_time)) 189 | if self.detector.other_objects: 190 | draw_str(frame, (20, 80), 'other objects: %s' % ', '.join(self.detector.other_objects)) 191 | # draw people's tracks 192 | for track in self.people: 193 | # choose color from color palette 194 | color = randColor(hash(track[0])) 195 | # draw rectangle for each detection in track 196 | for roi, _ in track: 197 | cv.rectangle(frame, roi.tl(), roi.br(), color) 198 | 199 | # show visualisation in window 200 | cv.imshow("counter", frame) 201 | 202 | # run main when this runs as a script 203 | if __name__ == "__main__": 204 | main() 205 | --------------------------------------------------------------------------------