├── images ├── .gitkeep ├── SocialDistancing1.png ├── SocialDistancing2.png ├── SocialDistancing3.png ├── SocialDistancing4.png ├── SocialDistancing5.png ├── SocialDistancing6.png ├── SocialDistancing3-1.png ├── SocialDistancing4-1.png └── SocialDistancing5-1.png ├── T.mat ├── resources └── project │ ├── Project.xml │ ├── Root.type.Files │ ├── LICENSE.type.File.xml │ ├── README.md.type.File.xml │ ├── images.type.File.xml │ ├── yolov3Helper.type.File.xml │ ├── images.type.File │ │ ├── .gitkeep.type.File.xml │ │ ├── 1.type.DIR_SIGNIFIER.xml │ │ ├── SocialDistancing1.png.type.File.xml │ │ ├── SocialDistancing2.png.type.File.xml │ │ ├── SocialDistancing3.png.type.File.xml │ │ ├── SocialDistancing4.png.type.File.xml │ │ ├── SocialDistancing5.png.type.File.xml │ │ ├── SocialDistancing6.png.type.File.xml │ │ ├── SocialDistancing3-1.png.type.File.xml │ │ ├── SocialDistancing4-1.png.type.File.xml │ │ └── SocialDistancing5-1.png.type.File.xml │ ├── yolov3Helper.type.File │ │ ├── 1.type.DIR_SIGNIFIER.xml │ │ ├── upsampleLayer.m.type.File.xml │ │ ├── generateTargets.m.type.File.xml │ │ └── generateYOLOv3Detections.m.type.File.xml │ ├── T.mat.type.File.xml │ ├── TRDATA.mat.type.File.xml │ ├── PeopleYOLOv3.mat.type.File.xml │ ├── importGroundTruth.m.type.File.xml │ ├── durationForVideo.mat.type.File.xml │ ├── COVID19_PeopleExtractGT.mlx.type.File.xml │ ├── COVID19_TrainPeopleYOLOv3.mlx.type.File.xml │ ├── COVID19_PeopleVideoRunning.mlx.type.File.xml │ ├── COVID19_SocialDistancing.mlapp.type.File.xml │ └── COVID19_SocialDistancingScript.mlx.type.File.xml │ ├── ProjectData.type.Info.xml │ ├── uuid-8a4d8e10-e660-4470-812b-18af87e9e63d.xml │ └── Root.type.Categories │ ├── FileClassCategory.type.Category │ ├── none.type.Label.xml │ ├── other.type.Label.xml │ ├── test.type.Label.xml │ ├── artifact.type.Label.xml │ ├── derived.type.Label.xml │ ├── design.type.Label.xml │ └── convenience.type.Label.xml │ └── FileClassCategory.type.Category.xml ├── TRDATA.mat ├── COVID19_PeopleExtractGT.mlx ├── COVID19_PeopleVideoRunning.mlx ├── COVID19_SocialDistancing.mlapp ├── COVID19_TrainPeopleYOLOv3.mlx ├── COVID19_SocialDistancingScript.mlx ├── GithubSubmission.prj ├── yolov3Helper ├── upsampleLayer.m ├── generateYOLOv3Detections.m └── generateTargets.m ├── LICENSE └── README.md /images/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /T.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matlab-deep-learning/Social-Distancing-Monitoring-System/HEAD/T.mat -------------------------------------------------------------------------------- /resources/project/Project.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/LICENSE.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/README.md.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/images.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /TRDATA.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matlab-deep-learning/Social-Distancing-Monitoring-System/HEAD/TRDATA.mat -------------------------------------------------------------------------------- /resources/project/Root.type.Files/yolov3Helper.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/ProjectData.type.Info.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/uuid-8a4d8e10-e660-4470-812b-18af87e9e63d.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/images.type.File/.gitkeep.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/images.type.File/1.type.DIR_SIGNIFIER.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/yolov3Helper.type.File/1.type.DIR_SIGNIFIER.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/images.type.File/SocialDistancing1.png.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/images.type.File/SocialDistancing2.png.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/images.type.File/SocialDistancing3.png.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/images.type.File/SocialDistancing4.png.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/images.type.File/SocialDistancing5.png.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/images.type.File/SocialDistancing6.png.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/images.type.File/SocialDistancing3-1.png.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/images.type.File/SocialDistancing4-1.png.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/images.type.File/SocialDistancing5-1.png.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /COVID19_PeopleExtractGT.mlx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matlab-deep-learning/Social-Distancing-Monitoring-System/HEAD/COVID19_PeopleExtractGT.mlx -------------------------------------------------------------------------------- /images/SocialDistancing1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matlab-deep-learning/Social-Distancing-Monitoring-System/HEAD/images/SocialDistancing1.png -------------------------------------------------------------------------------- /images/SocialDistancing2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matlab-deep-learning/Social-Distancing-Monitoring-System/HEAD/images/SocialDistancing2.png -------------------------------------------------------------------------------- /images/SocialDistancing3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matlab-deep-learning/Social-Distancing-Monitoring-System/HEAD/images/SocialDistancing3.png -------------------------------------------------------------------------------- /images/SocialDistancing4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matlab-deep-learning/Social-Distancing-Monitoring-System/HEAD/images/SocialDistancing4.png -------------------------------------------------------------------------------- /images/SocialDistancing5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matlab-deep-learning/Social-Distancing-Monitoring-System/HEAD/images/SocialDistancing5.png -------------------------------------------------------------------------------- /images/SocialDistancing6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matlab-deep-learning/Social-Distancing-Monitoring-System/HEAD/images/SocialDistancing6.png -------------------------------------------------------------------------------- /COVID19_PeopleVideoRunning.mlx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matlab-deep-learning/Social-Distancing-Monitoring-System/HEAD/COVID19_PeopleVideoRunning.mlx -------------------------------------------------------------------------------- /COVID19_SocialDistancing.mlapp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matlab-deep-learning/Social-Distancing-Monitoring-System/HEAD/COVID19_SocialDistancing.mlapp -------------------------------------------------------------------------------- /COVID19_TrainPeopleYOLOv3.mlx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matlab-deep-learning/Social-Distancing-Monitoring-System/HEAD/COVID19_TrainPeopleYOLOv3.mlx -------------------------------------------------------------------------------- /images/SocialDistancing3-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matlab-deep-learning/Social-Distancing-Monitoring-System/HEAD/images/SocialDistancing3-1.png -------------------------------------------------------------------------------- /images/SocialDistancing4-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matlab-deep-learning/Social-Distancing-Monitoring-System/HEAD/images/SocialDistancing4-1.png -------------------------------------------------------------------------------- /images/SocialDistancing5-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matlab-deep-learning/Social-Distancing-Monitoring-System/HEAD/images/SocialDistancing5-1.png -------------------------------------------------------------------------------- /COVID19_SocialDistancingScript.mlx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matlab-deep-learning/Social-Distancing-Monitoring-System/HEAD/COVID19_SocialDistancingScript.mlx -------------------------------------------------------------------------------- /resources/project/Root.type.Categories/FileClassCategory.type.Category/none.type.Label.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Categories/FileClassCategory.type.Category/other.type.Label.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Categories/FileClassCategory.type.Category/test.type.Label.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Categories/FileClassCategory.type.Category/artifact.type.Label.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Categories/FileClassCategory.type.Category/derived.type.Label.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Categories/FileClassCategory.type.Category/design.type.Label.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Categories/FileClassCategory.type.Category.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /resources/project/Root.type.Categories/FileClassCategory.type.Category/convenience.type.Label.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /GithubSubmission.prj: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/T.mat.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/TRDATA.mat.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/PeopleYOLOv3.mat.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/importGroundTruth.m.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/durationForVideo.mat.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/COVID19_PeopleExtractGT.mlx.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/COVID19_TrainPeopleYOLOv3.mlx.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/COVID19_PeopleVideoRunning.mlx.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/COVID19_SocialDistancing.mlapp.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/COVID19_SocialDistancingScript.mlx.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/yolov3Helper.type.File/upsampleLayer.m.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/yolov3Helper.type.File/generateTargets.m.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /resources/project/Root.type.Files/yolov3Helper.type.File/generateYOLOv3Detections.m.type.File.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /yolov3Helper/upsampleLayer.m: -------------------------------------------------------------------------------- 1 | % Upsample by replicating neighbouring pixel values. 2 | 3 | % Copyright 2019 The MathWorks, Inc. 4 | 5 | classdef upsampleLayer < nnet.layer.Layer 6 | properties 7 | 8 | % factor to upsample the input. 9 | UpSampleFactor 10 | end 11 | 12 | methods 13 | function layer = upsampleLayer(factor, name) 14 | 15 | % Set layer name. 16 | layer.Name = name; 17 | 18 | % Set layer description. 19 | layer.Description = "upSamplingLayer with factor " + factor; 20 | 21 | % Stride by which layer is upsampled. 22 | layer.UpSampleFactor = factor; 23 | end 24 | 25 | function Z = predict(layer, X) 26 | % Z = predict(layer, X) forwards the input data X through the 27 | % layer and outputs the result Z. 28 | 29 | Z = repelem(X,layer.UpSampleFactor,layer.UpSampleFactor); 30 | end 31 | end 32 | end -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020, The MathWorks, Inc. 2 | All rights reserved. 3 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 4 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 5 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 6 | 3. In all cases, the software is, and all modifications and derivatives of the software shall be, licensed to you solely for use in conjunction with MathWorks products and service offerings. 7 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /yolov3Helper/generateYOLOv3Detections.m: -------------------------------------------------------------------------------- 1 | function [bboxes,scores,labels] = generateYOLOv3Detections(detections, confidenceThreshold, imageSize, classes) 2 | % Apply following post processing steps to filter the detections: 3 | % * Filter detections based on threshold. 4 | % * Convert bboxes from spatial to pixel dimension. 5 | 6 | % Combine the prediction from different heads. 7 | detections(:,1:5) = cellfun(@ reshapePredictions,detections(:,1:5), 'UniformOutput', false); 8 | detections(:,6) = cellfun(@(a,b) reshapeClasses(a,b),detections(:,6),repmat({numel(classes)}, size(detections(:,6))),'UniformOutput', false); 9 | detections = cell2mat(detections); 10 | 11 | % Keep detections whose objectness score is greater than thresh. 12 | confidenceTmp = detections(:,1); 13 | detections = detections(confidenceTmp>confidenceThreshold,:); 14 | 15 | % Initialize bboxes,scores,labels. 16 | bboxes = zeros(0,'single'); 17 | scores = zeros(0,'single'); 18 | labels = categorical(cell(0,1)); 19 | 20 | % Filter the classes based on (confidence score * class probability). 21 | if ~isempty(detections) 22 | [classProbs, classIdx] = max(detections(:,6:end),[],2); 23 | detections(:,1) = detections(:,1).*classProbs; 24 | detections(:,6) = classIdx; 25 | detections = detections(detections(:,1)>=confidenceThreshold,:); 26 | if ~isempty(detections) 27 | 28 | bboxes = detections(:,2:5); 29 | scale = [imageSize(2) imageSize(1) imageSize(2) imageSize(1)]; 30 | bboxes = bboxes.*scale; 31 | 32 | % Convert x and y position of detections from centre to top-left. 33 | % Resize boxes to image size. 34 | bboxes = convertCenterToTopLeft(bboxes); 35 | 36 | scores = detections(:,1); 37 | labels = detections(:,6); 38 | classes = classes'; 39 | labels = categorical(classes(labels)); 40 | end 41 | end 42 | end 43 | 44 | % Convert x and y position of detections from centre to top-left. 45 | function bboxes = convertCenterToTopLeft(bboxes) 46 | bboxes(:,1) = bboxes(:,1)- bboxes(:,3)/2 + 0.5; 47 | bboxes(:,2) = bboxes(:,2)- bboxes(:,4)/2 + 0.5; 48 | bboxes = floor(bboxes); 49 | bboxes(bboxes<1) = 1; 50 | end 51 | 52 | function x = reshapePredictions(pred) 53 | [h,w,c,n] = size(pred); 54 | x = reshape(pred,h*w*c,1,n); 55 | end 56 | 57 | function x = reshapeClasses(pred,numclasses) 58 | [h,w,c,n] = size(pred); 59 | numanchors = c/numclasses; 60 | x = reshape(pred,h*w,numclasses,numanchors,n); 61 | x = permute(x,[1,3,2,4]); 62 | [h,w,c,n] = size(x); 63 | x = reshape(x,h*w,c,n); 64 | end -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Social Distancing detection using Deep Learning 2 | ============= 3 | Final Result

4 | 5 | 6 | **Background** 7 | 8 | Coronavirus disease (COVID-19) is a new strain of disease in humans discovered in 2019 that has never been identified in the past. 9 | Coronavirus is a large family of viruses that causes illness in patients ranging from common cold to advanced respiratory syndromes such as Middle East Respiratory Syndrome (MERS-COV) and Severe Acute Respiratory Syndrome (SARS-COV). 10 | Many people are currently affected and are being treated across the world causing a global pandemic. 11 | Several countries have declared a national emergency and have quarantined millions of people. 12 | 13 | To be a part of the worldwide trend, I've created a Social Distancing detection system. 14 | 15 | Application Workflow 16 | ------------- 17 |

18 | * Detect Pedestrians in the perspective view 19 | * Train YOLOv3 detector for pedestrian detection 20 | * Convert perspective view into Bird's-eye view 21 | * Morphs the perspective view into a bird’s-eye (top-down) view 22 | * We assumes that every person is standing on the same flat ground plane. 23 | * Measure the distances between persons in Bird's-eye view 24 | * Estimate pedestrian's (x, y) location in the bird's-eye view. 25 | * Selecting bottom-center point of each person's bounding box in the perspective view and transform the points into bird's-eye view. 26 | 27 | 28 | Part1 - Detect Pedestrians in the perspective view 29 | ------------- 30 |

31 | 32 | #### COVID19_PeopleExtractGT.mlx 33 | This file extracts dataset provider given ground truth information, [Oxford Town Centre Dataset](http://www.robots.ox.ac.uk/ActiveVision/Research/Projects/2009bbenfold_headpose/project.html)[1], and get ready for the training. 34 | #### COVID19_TrainPeopleYOLOv3.mlx 35 | Train the pedestrian detection model from ground truth data, extracted from above file. 36 | 37 | Part2 - Convert perspective view into Bird's-eye view & Measure the distances between persons in Bird's-eye view 38 | ------------- 39 |

40 | 41 | #### COVID19_SocialDistancingScript.mlx 42 | This file includes the entire social distancing system development script with pretrained people detection model. 43 | It covers calibration process from perspective view into bird's-eye view, extract person's location and measure the distances each other. 44 | Measurement

45 | 46 | Part3 - Test application and App for interactive execution 47 | ------------- 48 | ### COVID19_PeopleVideoRunning.mlx 49 | With the pretrained people detector, and using COVID19_SocialDistancingScript.mlx, it can run social distancing detector for existing video. 50 | Final Result

51 | ### COVID19_SocialDistancing.mlapp 52 | I have built a lightweight tool that enables even non-technical users to create the system for their own. 53 | App for Social Distancing

54 | 55 | Requires 56 | - [MATLAB](https://www.mathworks.com/products/matlab.html) 57 | - [Deep Learning Toolbox](https://www.mathworks.com/products/deep-learning.html) 58 | - [Image Processing Toolbox](https://www.mathworks.com/products/image.html) 59 | - [Computer Vision Toolbox](https://www.mathworks.com/products/computer-vision.html) 60 | - [Parallel Computing Toolbox](https://www.mathworks.com/products/parallel-computing.html) 61 | - [MATLAB Coder](https://www.mathworks.com/products/matlab-coder.html) 62 | - [GPU Coder](https://www.mathworks.com/products/gpu-coder.html) 63 | 64 | 65 | For more information on Deep Learning in MATLAB 66 | ------------- 67 | [![View Social-Distancing-Monitoring-System on File Exchange](https://www.mathworks.com/matlabcentral/images/matlab-file-exchange.svg)](https://kr.mathworks.com/matlabcentral/fileexchange/76760-social-distancing-monitoring-system) 68 | 69 | **[Download a free MATLAB trial for Deep Learning](https://www.mathworks.com/products/deep-learning.html)** 70 | 71 | 72 | References 73 | ------------- 74 | [1] B. Benfold and I. Reid. Guiding visual surveillance by tracking human attention. In BMVC, 2009 75 | 76 | Copyright 2020 The MathWorks, Inc. 77 | -------------------------------------------------------------------------------- /yolov3Helper/generateTargets.m: -------------------------------------------------------------------------------- 1 | function [boxDeltaTarget, objectnessTarget, classTarget, maskTarget, boxErrorScaleTarget] = generateTargets(YPredCellGathered, groundTruth, inputImageSize, anchorBoxes, anchorBoxMask, penaltyThreshold) 2 | % generateTargets creates target array for every prediction element 3 | % x, y, width, height, confidence scores and class probabilities. 4 | 5 | boxDeltaTarget = cell(size(YPredCellGathered,1),4); 6 | objectnessTarget = cell(size(YPredCellGathered,1),1); 7 | classTarget = cell(size(YPredCellGathered,1),1); 8 | maskTarget = cell(size(YPredCellGathered,1),3); 9 | boxErrorScaleTarget = cell(size(YPredCellGathered,1),1); 10 | 11 | % Normalize the ground truth boxes w.r.t image input size. 12 | gtScale = [inputImageSize(2) inputImageSize(1) inputImageSize(2) inputImageSize(1)]; 13 | groundTruth(:,1:4,:,:) = groundTruth(:,1:4,:,:)./gtScale; 14 | 15 | for numPred = 1:size(YPredCellGathered,1) 16 | 17 | % Select anchor boxes based on anchor box mask indices. 18 | anchors = anchorBoxes(anchorBoxMask{numPred},:); 19 | 20 | bx = YPredCellGathered{numPred,2}; 21 | by = YPredCellGathered{numPred,3}; 22 | bw = YPredCellGathered{numPred,4}; 23 | bh = YPredCellGathered{numPred,5}; 24 | predClasses = YPredCellGathered{numPred,6}; 25 | 26 | gridSize = size(bx); 27 | if numel(gridSize)== 3 28 | gridSize(4) = 1; 29 | end 30 | numClasses = size(predClasses,3)./size(anchors,1); 31 | 32 | % Initialize the required variables. 33 | mask = single(zeros(size(bx))); 34 | confMask = single(ones(size(bx))); 35 | classMask = single(zeros(size(predClasses))); 36 | tx = single(zeros(size(bx))); 37 | ty = single(zeros(size(by))); 38 | tw = single(zeros(size(bw))); 39 | th = single(zeros(size(bh))); 40 | tconf = single(zeros(size(bx))); 41 | tclass = single(zeros(size(predClasses))); 42 | boxErrorScale = single(ones(size(bx))); 43 | 44 | % Get the IOU of predictions with groundtruth. 45 | iou = getMaxIOUPredictedWithGroundTruth(bx,by,bw,bh,groundTruth); 46 | 47 | % Donot penalize the predictions which has iou greater than penalty 48 | % threshold. 49 | confMask(iou > penaltyThreshold) = 0; 50 | 51 | for batch = 1:gridSize(4) 52 | truthBatch = groundTruth(:,1:5,:,batch); 53 | truthBatch = truthBatch(all(truthBatch,2),:); 54 | 55 | % Get boxes with center as 0. 56 | gtPred = [0-truthBatch(:,3)/2,0-truthBatch(:,4)/2,truthBatch(:,3),truthBatch(:,4)]; 57 | anchorPrior = [0-anchorBoxes(:,2)/(2*inputImageSize(2)),0-anchorBoxes(:,1)/(2*inputImageSize(1)),anchorBoxes(:,2)/inputImageSize(2),anchorBoxes(:,1)/inputImageSize(1)]; 58 | 59 | % Get the iou of best matching anchor box. 60 | overLap = bboxOverlapRatio(gtPred,anchorPrior); 61 | [~,bestAnchorIdx] = max(overLap,[],2); 62 | 63 | % Select gt that are within the mask. 64 | index = ismember(bestAnchorIdx,anchorBoxMask{numPred}); 65 | truthBatch = truthBatch(index,:); 66 | bestAnchorIdx = bestAnchorIdx(index,:); 67 | bestAnchorIdx = bestAnchorIdx - anchorBoxMask{numPred}(1,1) + 1; 68 | 69 | if ~isempty(truthBatch) 70 | % Convert top left position of ground-truth to centre coordinates. 71 | truthBatch = [truthBatch(:,1)+truthBatch(:,3)./2,truthBatch(:,2)+truthBatch(:,4)./2,truthBatch(:,3),truthBatch(:,4),truthBatch(:,5)]; 72 | 73 | errorScale = 2 - truthBatch(:,3).*truthBatch(:,4); 74 | truthBatch = [truthBatch(:,1)*gridSize(2),truthBatch(:,2)*gridSize(1),truthBatch(:,3)*inputImageSize(2),truthBatch(:,4)*inputImageSize(1),truthBatch(:,5)]; 75 | for t = 1:size(truthBatch,1) 76 | 77 | % Get the position of ground-truth box in the grid. 78 | colIdx = ceil(truthBatch(t,1)); 79 | colIdx(colIdx<1) = 1; 80 | colIdx(colIdx>gridSize(2)) = gridSize(2); 81 | rowIdx = ceil(truthBatch(t,2)); 82 | rowIdx(rowIdx<1) = 1; 83 | rowIdx(rowIdx>gridSize(1)) = gridSize(1); 84 | pos = [rowIdx,colIdx]; 85 | anchorIdx = bestAnchorIdx(t,1); 86 | 87 | mask(pos(1,1),pos(1,2),anchorIdx,batch) = 1; 88 | confMask(pos(1,1),pos(1,2),anchorIdx,batch) = 1; 89 | 90 | % Calculate the shift in ground-truth boxes. 91 | tShiftX = truthBatch(t,1)-pos(1,2)+1; 92 | tShiftY = truthBatch(t,2)-pos(1,1)+1; 93 | tShiftW = log(truthBatch(t,3)/anchors(anchorIdx,2)); 94 | tShiftH = log(truthBatch(t,4)/anchors(anchorIdx,1)); 95 | 96 | % Update the target box. 97 | tx(pos(1,1),pos(1,2),anchorIdx,batch) = tShiftX; 98 | ty(pos(1,1),pos(1,2),anchorIdx,batch) = tShiftY; 99 | tw(pos(1,1),pos(1,2),anchorIdx,batch) = tShiftW; 100 | th(pos(1,1),pos(1,2),anchorIdx,batch) = tShiftH; 101 | boxErrorScale(pos(1,1),pos(1,2),anchorIdx,batch) = errorScale(t); 102 | tconf(rowIdx,colIdx,anchorIdx,batch) = 1; 103 | classIdx = (numClasses*(anchorIdx-1))+truthBatch(t,5); 104 | tclass(rowIdx,colIdx,classIdx,batch) = 1; 105 | classMask(rowIdx,colIdx,(numClasses*(anchorIdx-1))+(1:numClasses),batch) = 1; 106 | end 107 | end 108 | end 109 | boxDeltaTarget(numPred,:) = [{tx} {ty} {tw} {th}]; 110 | objectnessTarget{numPred,1} = tconf; 111 | classTarget{numPred,1} = tclass; 112 | maskTarget(numPred,:) = [{mask} {confMask} {classMask}]; 113 | boxErrorScaleTarget{numPred,:} = boxErrorScale; 114 | end 115 | end 116 | 117 | function iou = getMaxIOUPredictedWithGroundTruth(predx,predy,predw,predh,truth) 118 | % getMaxIOUPredictedWithGroundTruth computes the maximum intersection over 119 | % union scores for every pair of predictions and ground-truth boxes. 120 | 121 | [h,w,c,n] = size(predx); 122 | iou = zeros([h w c n],'like',predx); 123 | 124 | % For each batch prepare the predictions and ground-truth. 125 | for batchSize = 1:n 126 | truthBatch = truth(:,1:4,1,batchSize); 127 | truthBatch = truthBatch(all(truthBatch,2),:); 128 | predxb = predx(:,:,:,batchSize); 129 | predyb = predy(:,:,:,batchSize); 130 | predwb = predw(:,:,:,batchSize); 131 | predhb = predh(:,:,:,batchSize); 132 | predb = [predxb(:),predyb(:),predwb(:),predhb(:)]; 133 | 134 | % Convert from center xy coordinate to topleft xy coordinate. 135 | predb = [predb(:,1)-predb(:,3)./2, predb(:,2)-predb(:,4)./2, predb(:,3), predb(:,4)]; 136 | 137 | % Compute and extract the maximum IOU of predictions with ground-truth. 138 | overlap = bboxOverlapRatio(predb,truthBatch); 139 | maxOverlap = max(overlap,[],2); 140 | iou(:,:,:,batchSize) = reshape(maxOverlap,h,w,c); 141 | end 142 | end --------------------------------------------------------------------------------