├── .gitignore
├── LICENSE
├── README.md
├── SocialDistancingChecker
    ├── pom.xml
    └── src
    │   └── main
    │       ├── java
    │           ├── CameraStreamer.java
    │           ├── SocialDistanceChecker.java
    │           ├── SocialDistanceCheckerVideo.java
    │           └── SocialDistanceCheckerWebcam.java
    │       └── resources
    │           └── log4j.properties
├── crowd_topview_gray_1280x720.jpg
└── poster.png


/.gitignore:
--------------------------------------------------------------------------------
 1 | .idea
 2 | target/
 3 | *.mp4
 4 | *.iml
 5 | 
 6 | # Compiled class file
 7 | *.class
 8 | 
 9 | # Log file
10 | *.log
11 | 
12 | # BlueJ files
13 | *.ctxt
14 | 
15 | # Mobile Tools for Java (J2ME)
16 | .mtj.tmp/
17 | 
18 | # Package Files #
19 | *.jar
20 | *.war
21 | *.nar
22 | *.ear
23 | *.zip
24 | *.tar.gz
25 | *.rar
26 | 
27 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
28 | hs_err_pid*
29 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2020 Choo Wilson
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Social Distancing Checker in Java
 2 | A demo project to showcase how you can create a social distancing monitor using Eclipse Deeplearning4j and JavaCV. The program is targeted to process camera video frames for security cameras which has top down view footages.
 3 | 
 4 | ![Social Distancing Monitor](poster.png "Social Distancing Monitor")
 5 | 
 6 | # Setup
 7 | 1. This repository uses Eclipse Deeplearning4j and JavaCV
 8 | 2. Open this using IntelliJ and open the pom.xml as a Maven Project.
 9 | 
10 | # Run
11 | 1. Get the model file [here](https://drive.google.com/file/d/12aD2k96gA9yrtU06fACrJmzqvcyF8f_S/view?usp=sharing)
12 | 2. Change the PATH to your test image or test video.
13 | 


--------------------------------------------------------------------------------
/SocialDistancingChecker/pom.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
 3 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 5 |     <modelVersion>4.0.0</modelVersion>
 6 | 
 7 |     <groupId>Certifai</groupId>
 8 |     <artifactId>SocialDistancingChecker</artifactId>
 9 |     <version>1.0-SNAPSHOT</version>
10 | 
11 |     <properties>
12 |         <maven.compiler.source>1.8</maven.compiler.source>
13 |         <maven.compiler.target>1.8</maven.compiler.target>
14 |         <nd4j.backend>nd4j-native-platform</nd4j.backend>
15 |         <nd4j.backend>nd4j-cuda-10.1-platform</nd4j.backend>
16 |         <nd4j.version>1.0.0-beta7</nd4j.version>
17 |         <dl4j.version>1.0.0-beta7</dl4j.version>
18 |         <datavec.version>1.0.0-beta7</datavec.version>
19 |     </properties>
20 | 
21 |     <repositories>
22 |         <repository>
23 |             <id>snapshots-repo</id>
24 |             <url>https://oss.sonatype.org/content/repositories/snapshots</url>
25 |             <releases>
26 |                 <enabled>false</enabled>
27 |             </releases>
28 |             <snapshots>
29 |                 <enabled>true</enabled>
30 |             </snapshots>
31 |         </repository>
32 |     </repositories>
33 |     <dependencies>
34 |         <dependency>
35 |             <groupId>org.nd4j</groupId>
36 |             <artifactId>${nd4j.backend}</artifactId>
37 |             <version>${nd4j.version}</version>
38 |         </dependency>
39 |         <dependency>
40 |             <groupId>org.deeplearning4j</groupId>
41 |             <artifactId>deeplearning4j-cuda-10.1</artifactId>
42 |             <version>${dl4j.version}</version>
43 |         </dependency>
44 |         <dependency>
45 |             <groupId>org.bytedeco</groupId>
46 |             <artifactId>javacv-platform</artifactId>
47 |             <version>1.5.3</version>
48 |         </dependency>
49 |         <dependency>
50 |             <groupId>org.deeplearning4j</groupId>
51 |             <artifactId>deeplearning4j-ui</artifactId>
52 |             <version>${dl4j.version}</version>
53 |         </dependency>
54 |         <!--        Logging libraries-->
55 |         <dependency>
56 |             <groupId>org.slf4j</groupId>
57 |             <artifactId>slf4j-api</artifactId>
58 |             <version>2.0.0-alpha1</version>
59 |         </dependency>
60 |         <dependency>
61 |             <groupId>org.slf4j</groupId>
62 |             <artifactId>slf4j-log4j12</artifactId>
63 |             <version>2.0.0-alpha1</version>
64 |         </dependency>
65 |     </dependencies>
66 | </project>


--------------------------------------------------------------------------------
/SocialDistancingChecker/src/main/java/CameraStreamer.java:
--------------------------------------------------------------------------------
  1 | import org.bytedeco.javacv.Java2DFrameConverter;
  2 | import org.bytedeco.javacv.OpenCVFrameConverter;
  3 | import org.bytedeco.opencv.opencv_core.Mat;
  4 | import org.bytedeco.opencv.opencv_videoio.VideoCapture;
  5 | import org.datavec.image.loader.NativeImageLoader;
  6 | import org.nd4j.linalg.api.ndarray.INDArray;
  7 | 
  8 | import java.io.IOException;
  9 | 
 10 | import static org.bytedeco.opencv.global.opencv_videoio.CAP_PROP_FRAME_WIDTH;
 11 | import static org.opencv.highgui.HighGui.destroyAllWindows;
 12 | import static org.opencv.videoio.Videoio.*;
 13 | 
 14 | //Special thanks for yquemener for writing the original camera stream class
 15 | public class CameraStreamer extends Thread {
 16 | 
 17 |     public Runnable camStreamRunnable;
 18 |     private int videoCaptureIndex = 0;
 19 | //    private String videoCaptureIndex = "C:\\Users\\ChooWilson\\Desktop\\koreacrowd.mp4";
 20 | 
 21 |     private int videoCaptureFPS = 30;
 22 |     private VideoCapture cap = null;
 23 |     private NativeImageLoader nil = null;
 24 |     private final int videoCaptureWidth = 1280;
 25 |     private final int videoCaptureHeight = 720;
 26 |     private final int outputWidth = 416;
 27 |     private final int outputHeight = 416;
 28 | 
 29 |     private OpenCVFrameConverter.ToMat converterMatToFrame;
 30 |     private Java2DFrameConverter converterFrameToBufferedImage;
 31 | 
 32 | 
 33 |     public void startCapture() {
 34 | 
 35 | //        converterFrameToBufferedImage = new Java2DFrameConverter();
 36 | //        converterMatToFrame = new OpenCVFrameConverter.ToMat();
 37 | 
 38 |         if (this.cap == null) {
 39 |             this.cap = new VideoCapture();
 40 |         }
 41 |         if (this.nil == null) {
 42 |             this.nil = new NativeImageLoader(
 43 |                     this.outputHeight,
 44 |                     this.outputWidth,
 45 |                     3);
 46 |         }
 47 |         this.cap.open(this.videoCaptureIndex);
 48 |         this.cap.set(CAP_PROP_FRAME_WIDTH, this.videoCaptureWidth);
 49 |         this.cap.set(CAP_PROP_FRAME_HEIGHT, this.videoCaptureHeight);
 50 |         this.cap.set(CAP_PROP_FPS, this.videoCaptureFPS);
 51 | 
 52 |         // Disable autofocus
 53 |         // If the picture is blurry for closeup objects, you may want to change this
 54 |         this.cap.set(CAP_PROP_AUTOFOCUS, 0);
 55 |         this.cap.set(CAP_PROP_FOCUS, 0);
 56 |         this.start();
 57 |     }
 58 | 
 59 |     public Mat getCVFrame() {
 60 |         Mat cvFrame = new Mat();
 61 |         // For some reasons, OpenCV capture needs that to prevent memory leaks, explicitly calling the garbage collector
 62 |         System.gc();
 63 |         synchronized (cap) {
 64 |             cap.retrieve(cvFrame);
 65 |         }
 66 |         assert (this.outputHeight <= this.videoCaptureHeight);
 67 |         assert (this.outputWidth <= this.videoCaptureWidth);
 68 |         assert (cvFrame != null);
 69 |         assert (!cvFrame.empty());
 70 |         return cvFrame;
 71 |     }
 72 | 
 73 |     public INDArray getNDArray(Mat frame) throws IOException {
 74 |         return nil.asMatrix(frame).div(255.0);
 75 |     }
 76 | 
 77 |     //    Destroy the VideoCapture properly
 78 |     public void stopCapture() {
 79 |         if (!cap.isNull())
 80 |             this.cap.release();
 81 |         destroyAllWindows();
 82 |     }
 83 | 
 84 |     //    Run VideoCapture as an independent thread
 85 |     @Override
 86 |     public void run() {
 87 |         while (isAlive()) {
 88 |             synchronized (cap) {
 89 |                 this.cap.grab();
 90 |             }
 91 |             try {
 92 |                 camStreamRunnable.run();
 93 |             } catch (Exception e) {
 94 |                 System.err.println("Camera Stream Thread threw an exception. Error message: " + e.toString());
 95 |             }
 96 |             try {
 97 |                 Thread.sleep(10);
 98 |             } catch (InterruptedException e) {
 99 |                 e.printStackTrace();
100 |                 stopCapture();
101 |             }
102 |         }
103 |         stopCapture();
104 |     }
105 | 
106 |     //    public BufferedImage getBufferedImage() throws IOException {
107 |     //        Frame f = converterMatToFrame.convert(getCVFrame());
108 |     //        return converterFrameToBufferedImage.convert(f);
109 |     //    }
110 | }
111 | 


--------------------------------------------------------------------------------
/SocialDistancingChecker/src/main/java/SocialDistanceChecker.java:
--------------------------------------------------------------------------------
  1 | import org.bytedeco.opencv.opencv_core.Mat;
  2 | import org.bytedeco.opencv.opencv_core.Point;
  3 | import org.bytedeco.opencv.opencv_core.Scalar;
  4 | import org.datavec.image.loader.NativeImageLoader;
  5 | import org.deeplearning4j.nn.graph.ComputationGraph;
  6 | import org.deeplearning4j.nn.layers.objdetect.DetectedObject;
  7 | import org.deeplearning4j.nn.layers.objdetect.YoloUtils;
  8 | import org.deeplearning4j.nn.modelimport.keras.KerasModelImport;
  9 | import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException;
 10 | import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfigurationException;
 11 | import org.nd4j.linalg.api.ndarray.INDArray;
 12 | import org.nd4j.linalg.factory.Nd4j;
 13 | import org.nd4j.linalg.indexing.NDArrayIndex;
 14 | 
 15 | import java.io.IOException;
 16 | import java.util.ArrayList;
 17 | import java.util.HashSet;
 18 | import java.util.List;
 19 | import java.util.Set;
 20 | 
 21 | import static org.bytedeco.opencv.global.opencv_highgui.*;
 22 | import static org.bytedeco.opencv.global.opencv_imgcodecs.imread;
 23 | import static org.bytedeco.opencv.global.opencv_imgproc.*;
 24 | import static org.nd4j.common.util.MathUtils.sigmoid;
 25 | import static org.nd4j.linalg.indexing.NDArrayIndex.point;
 26 | import static org.nd4j.linalg.ops.transforms.Transforms.euclideanDistance;
 27 | 
 28 | //  Author: Choo Wilson(yptheangel)
 29 | //  Special thanks to yquemener for writing yolov3 output interpreter
 30 | //  This example shows a Social Distancing Monitor running on only an image
 31 | 
 32 | public class SocialDistanceChecker {
 33 | 
 34 |     private static ComputationGraph model;
 35 |     //        COCO has 80 classes
 36 |     private static int numClass = 80;
 37 |     private static int[][] anchors = {{10, 13}, {16, 30}, {33, 23}, {30, 61}, {62, 45}, {59, 119}, {116, 90}, {156, 198}, {373, 326}};
 38 |     private static int yolowidth = 416;
 39 |     private static int yoloheight = 416;
 40 | 
 41 |     public static void main(String[] args) throws InvalidKerasConfigurationException, IOException, UnsupportedKerasConfigurationException {
 42 | 
 43 |         int safeDistance = 80;
 44 |         String modelPATH = "C:\\Users\\ChooWilson\\Downloads\\yolov3_416_fixed.h5";
 45 |         model = KerasModelImport.importKerasModelAndWeights(modelPATH);
 46 |         model.init();
 47 |         System.out.println(model.summary());
 48 | 
 49 |         String testImagePATH = "C:\\Users\\ChooWilson\\Desktop\\Social-Distancing-Monitor-Java\\crowd_topview_gray_1280x720.jpg";
 50 |         Mat opencvMat = imread(testImagePATH);
 51 |         NativeImageLoader nil = new NativeImageLoader(yolowidth, yoloheight, 3);
 52 |         INDArray input = nil.asMatrix(testImagePATH).div(255);
 53 | //        DL4j defaults to NCHW, need to convert to NHWC(channel last)
 54 |         input = input.permute(0, 2, 3, 1);
 55 |         List<DetectedObject> objs = getPredictedObjects(input);
 56 |         YoloUtils.nms(objs, 0.4);
 57 | 
 58 |         int w = opencvMat.cols();
 59 |         int h = opencvMat.rows();
 60 | 
 61 |         List<INDArray> centers = new ArrayList<>();
 62 |         List<INDArray> people = new ArrayList<>();
 63 |         Set violators = new HashSet<INDArray>();
 64 | 
 65 |         int centerX;
 66 |         int centerY;
 67 | 
 68 |         for (DetectedObject obj : objs) {
 69 | //             0 is the index of "person" in COCO dataset list of objects
 70 |             if (obj.getPredictedClass() == 0) {
 71 |                 //            Scale the coordinates back to full size
 72 |                 centerX = (int) obj.getCenterX() * w / yolowidth;
 73 |                 centerY = (int) obj.getCenterY() * h / yoloheight;
 74 | 
 75 |                 circle(opencvMat, new Point(centerX, centerY), 2, new Scalar(0, 255, 0, 0), 2, 0, 0);
 76 |                 //            Draw bounding boxes on opencv mat
 77 |                 double[] xy1 = obj.getTopLeftXY();
 78 |                 double[] xy2 = obj.getBottomRightXY();
 79 |                 //            Scale the coordinates back to full size
 80 |                 xy1[0] = xy1[0] * w / yolowidth;
 81 |                 xy1[1] = xy1[1] * h / yoloheight;
 82 |                 xy2[0] = xy2[0] * w / yolowidth;
 83 |                 xy2[1] = xy2[1] * h / yoloheight;
 84 | 
 85 |                 //Draw bounding box
 86 |                 rectangle(opencvMat, new Point((int) xy1[0], (int) xy1[1]), new Point((int) xy2[0], (int) xy2[1]), new Scalar(0, 255, 0, 0), 2, LINE_8, 0);
 87 |                 centers.add(Nd4j.create(new float[]{(float) centerX, (float) centerY}));
 88 |                 people.add(Nd4j.create(new float[]{(float) xy1[0], (float) xy1[1], (float) xy2[0], (float) xy2[1]}));
 89 |             }
 90 |         }
 91 | 
 92 |         //        Calculate the euclidean distance between all pairs of center points
 93 |         for (int i = 0; i < centers.size(); i++) {
 94 |             for (int j = 0; j < centers.size(); j++) {
 95 |                 double distance = euclideanDistance(centers.get(i), centers.get(j));
 96 |                 if (distance < safeDistance && distance > 0) {
 97 |                     line(opencvMat, new Point(centers.get(i).getInt(0), centers.get(i).getInt(1)),
 98 |                             new Point(centers.get(j).getInt(0), centers.get(j).getInt(1)), Scalar.RED, 2, 1, 0);
 99 | 
100 |                     violators.add(centers.get(i));
101 |                     violators.add(centers.get(j));
102 | 
103 |                     int xmin = people.get(i).getInt(0);
104 |                     int ymin = people.get(i).getInt(1);
105 |                     int xmax = people.get(i).getInt(2);
106 |                     int ymax = people.get(i).getInt(3);
107 | 
108 |                     rectangle(opencvMat, new Point(xmin, ymin), new Point(xmax, ymax), Scalar.RED, 2, LINE_8, 0);
109 |                     circle(opencvMat, new Point(centers.get(i).getInt(0), centers.get(i).getInt(1)), 3, Scalar.RED, -1, 0, 0);
110 |                 }
111 |             }
112 |         }
113 |         putText(opencvMat, String.format("Number of people: %d", people.size()), new Point(10, 30), 4, 1.0, new Scalar(0, 255, 0, 0), 2, LINE_8, false);
114 |         putText(opencvMat, String.format("Number of violators: %d", violators.size()), new Point(10, 60), 4, 1.0, new Scalar(0, 0, 255, 0), 2, LINE_8, false);
115 |         imshow("Social Distancing Monitor", opencvMat);
116 | 
117 |         if (waitKey(0) == 27) {
118 |             destroyAllWindows();
119 |         }
120 |     }
121 | 
122 |     public static List<DetectedObject> getPredictedObjects(INDArray input) {
123 | 
124 |         INDArray[] outputs = model.output(input);
125 | 
126 |         List<DetectedObject> out = new ArrayList();
127 |         float detectionThreshold = 0.6f;
128 |         // Each cell had information for 3 boxes
129 |         // box info starts from indices {0,85,170}
130 |         int[] boxOffsets = {0, numClass + 5, (numClass + 5) * 2};
131 |         int exampleNum_in_batch = 0;
132 | 
133 | 
134 |         for (int layerNum = 0; layerNum < 3; layerNum++) {
135 |             long gridWidth = outputs[layerNum].shape()[1];
136 |             long gridHeight = outputs[layerNum].shape()[2];
137 |             float cellWidth = yolowidth / gridWidth;
138 |             float cellHeight = yoloheight / gridHeight;
139 | 
140 |             for (int i = 0; i < gridHeight; i++) {
141 |                 for (int j = 0; j < gridWidth; j++) {
142 |                     float centerX;
143 |                     float centerY;
144 |                     float width;
145 |                     float height;
146 |                     int anchorInd;
147 | 
148 |                     for (int k = 0; k < 3; k++) {
149 | //                        exampleNum_in_batch is 0 because there is only 1 example in the batch
150 | //                        getFloat(),get() has 4 arguments because there are 4 indices we can use to get the single float value we want, in the order NHWC
151 |                         float prob = outputs[layerNum].getFloat(new int[]{exampleNum_in_batch, i, j, boxOffsets[k] + 4});
152 |                         if (prob > detectionThreshold) {
153 | //                            TODO: class probabilities does not make sense
154 |                             INDArray classes_scores = outputs[layerNum].get(
155 |                                     point(exampleNum_in_batch),
156 |                                     point(i),
157 |                                     point(j),
158 |                                     NDArrayIndex.interval(boxOffsets[k] + 5, boxOffsets[k] + numClass + 5));
159 | 
160 |                             centerX = outputs[layerNum].getFloat(new int[]{exampleNum_in_batch, i, j, boxOffsets[k] + 0});
161 |                             centerY = outputs[layerNum].getFloat(new int[]{exampleNum_in_batch, i, j, boxOffsets[k] + 1});
162 |                             width = outputs[layerNum].getFloat(new int[]{exampleNum_in_batch, i, j, boxOffsets[k] + 2});
163 |                             height = outputs[layerNum].getFloat(new int[]{exampleNum_in_batch, i, j, boxOffsets[k] + 3});
164 | 
165 |                             anchorInd = (2 - layerNum) * 3 + k;
166 | 
167 |                             centerX = (float) ((sigmoid(centerX) + j) * cellWidth);
168 |                             centerY = (float) ((sigmoid(centerY) + i) * cellHeight);
169 | 
170 |                             width = (float) (Math.exp(width)) * anchors[anchorInd][0];
171 |                             height = (float) (Math.exp(height)) * anchors[anchorInd][1];
172 | 
173 |                             out.add(new DetectedObject(k, centerX, centerY, width, height, classes_scores, prob));
174 |                         }
175 |                     }
176 |                 }
177 |             }
178 |         }
179 |         return out;
180 |     }
181 | }
182 | 


--------------------------------------------------------------------------------
/SocialDistancingChecker/src/main/java/SocialDistanceCheckerVideo.java:
--------------------------------------------------------------------------------
  1 | import org.bytedeco.ffmpeg.global.avcodec;
  2 | import org.bytedeco.javacv.FFmpegFrameGrabber;
  3 | import org.bytedeco.javacv.FFmpegFrameRecorder;
  4 | import org.bytedeco.javacv.Frame;
  5 | import org.bytedeco.javacv.OpenCVFrameConverter;
  6 | import org.bytedeco.opencv.opencv_core.Mat;
  7 | import org.bytedeco.opencv.opencv_core.Point;
  8 | import org.bytedeco.opencv.opencv_core.Scalar;
  9 | import org.datavec.image.loader.NativeImageLoader;
 10 | import org.deeplearning4j.nn.graph.ComputationGraph;
 11 | import org.deeplearning4j.nn.layers.objdetect.DetectedObject;
 12 | import org.deeplearning4j.nn.layers.objdetect.YoloUtils;
 13 | import org.deeplearning4j.nn.modelimport.keras.KerasModelImport;
 14 | import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException;
 15 | import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfigurationException;
 16 | import org.nd4j.linalg.api.ndarray.INDArray;
 17 | import org.nd4j.linalg.factory.Nd4j;
 18 | import org.nd4j.linalg.indexing.NDArrayIndex;
 19 | 
 20 | import java.io.IOException;
 21 | import java.util.ArrayList;
 22 | import java.util.HashSet;
 23 | import java.util.List;
 24 | import java.util.Set;
 25 | 
 26 | import static org.bytedeco.opencv.global.opencv_highgui.*;
 27 | import static org.bytedeco.opencv.global.opencv_imgproc.*;
 28 | import static org.nd4j.common.util.MathUtils.sigmoid;
 29 | import static org.nd4j.linalg.indexing.NDArrayIndex.point;
 30 | import static org.nd4j.linalg.ops.transforms.Transforms.euclideanDistance;
 31 | 
 32 | //  Author: Choo Wilson(yptheangel)
 33 | //  Special thanks to yquemener for writing yolov3 output interpreter
 34 | //  This example shows a Social Distancing Monitor running on video
 35 | 
 36 | public class SocialDistanceCheckerVideo {
 37 | 
 38 |     private static ComputationGraph model;
 39 |     //        COCO has 80 classes
 40 |     private static int numClass = 80;
 41 |     private static int[][] anchors = {{10, 13}, {16, 30}, {33, 23}, {30, 61}, {62, 45}, {59, 119}, {116, 90}, {156, 198}, {373, 326}};
 42 |     private static int yolowidth = 416;
 43 |     private static int yoloheight = 416;
 44 |     private static boolean save2video = false;
 45 |     private static FFmpegFrameRecorder recorder = new FFmpegFrameRecorder("output.mp4", 1280, 720, 0);
 46 | 
 47 | 
 48 |     public static void main(String[] args) throws InvalidKerasConfigurationException, IOException, UnsupportedKerasConfigurationException {
 49 | 
 50 |         int safeDistance = 80;
 51 |         String modelPATH = "C:\\Users\\ChooWilson\\Downloads\\yolov3_416_fixed.h5";
 52 |         model = KerasModelImport.importKerasModelAndWeights(modelPATH);
 53 |         model.init();
 54 |         System.out.println(model.summary());
 55 | 
 56 | //        String videoPath = "C:\\Users\\ChooWilson\\Desktop\\topdown.mp4";
 57 |         String videoPath = "C:\\Users\\ChooWilson\\Desktop\\koreacrowd.mp4";
 58 |         FFmpegFrameGrabber grabber = new FFmpegFrameGrabber(videoPath);
 59 |         grabber.setFormat("mp4");
 60 |         grabber.start();
 61 | 
 62 |         if (save2video) {
 63 |             recorder.setVideoCodec(avcodec.AV_CODEC_ID_MPEG4);
 64 |             recorder.setVideoBitrate(9000);
 65 |             recorder.setFormat("mp4");
 66 |             recorder.setVideoQuality(0); // maximum quality
 67 |             recorder.setFrameRate(15);
 68 |             recorder.start();
 69 |         }
 70 | 
 71 |         OpenCVFrameConverter.ToMat frame2Mat = new OpenCVFrameConverter.ToMat();
 72 | 
 73 |         while (grabber.grab() != null) {
 74 |             Frame frame = grabber.grabImage();
 75 |             if (frame != null) {
 76 |                 Mat opencvMat = frame2Mat.convert(frame);
 77 |                 NativeImageLoader nil = new NativeImageLoader(yolowidth, yoloheight, 3);
 78 |                 INDArray input = nil.asMatrix(opencvMat).div(255);
 79 |                 input = input.permute(0, 2, 3, 1);
 80 | 
 81 |                 List<DetectedObject> objs = getPredictedObjects(input);
 82 |                 YoloUtils.nms(objs, 0.4);
 83 | 
 84 |                 int w = opencvMat.cols();
 85 |                 int h = opencvMat.rows();
 86 |                 List<INDArray> centers = new ArrayList<>();
 87 |                 List<INDArray> people = new ArrayList<>();
 88 |                 Set violators = new HashSet<INDArray>();
 89 | 
 90 |                 int centerX;
 91 |                 int centerY;
 92 | 
 93 |                 for (DetectedObject obj : objs) {
 94 |                     if (obj.getPredictedClass() == 0) {
 95 |                         //            Scale the coordinates back to full size
 96 |                         centerX = (int) obj.getCenterX() * w / yolowidth;
 97 |                         centerY = (int) obj.getCenterY() * h / yoloheight;
 98 | 
 99 |                         circle(opencvMat, new Point(centerX, centerY), 2, new Scalar(0, 255, 0, 0), -1, 0, 0);
100 |                         //            Draw bounding boxes on opencv mat
101 |                         double[] xy1 = obj.getTopLeftXY();
102 |                         double[] xy2 = obj.getBottomRightXY();
103 |                         //            Scale the coordinates back to full size
104 |                         xy1[0] = xy1[0] * w / yolowidth;
105 |                         xy1[1] = xy1[1] * h / yoloheight;
106 |                         xy2[0] = xy2[0] * w / yolowidth;
107 |                         xy2[1] = xy2[1] * h / yoloheight;
108 | 
109 |                         //Draw bounding box
110 |                         rectangle(opencvMat, new Point((int) xy1[0], (int) xy1[1]), new Point((int) xy2[0], (int) xy2[1]), new Scalar(0, 255, 0, 0), 2, LINE_8, 0);
111 |                         centers.add(Nd4j.create(new float[]{(float) centerX, (float) centerY}));
112 |                         people.add(Nd4j.create(new float[]{(float) xy1[0], (float) xy1[1], (float) xy2[0], (float) xy2[1]}));
113 |                     }
114 |                 }
115 |                 //        Calculate the euclidean distance between all pairs of center points
116 |                 for (int i = 0; i < centers.size(); i++) {
117 |                     for (int j = 0; j < centers.size(); j++) {
118 |                         double distance = euclideanDistance(centers.get(i), centers.get(j));
119 |                         if (distance < safeDistance && distance > 0) {
120 |                             line(opencvMat, new Point(centers.get(i).getInt(0), centers.get(i).getInt(1)),
121 |                                     new Point(centers.get(j).getInt(0), centers.get(j).getInt(1)), Scalar.RED, 2, 1, 0);
122 | 
123 |                             violators.add(centers.get(i));
124 |                             violators.add(centers.get(j));
125 | 
126 |                             int xmin = people.get(i).getInt(0);
127 |                             int ymin = people.get(i).getInt(1);
128 |                             int xmax = people.get(i).getInt(2);
129 |                             int ymax = people.get(i).getInt(3);
130 | 
131 |                             rectangle(opencvMat, new Point(xmin, ymin), new Point(xmax, ymax), Scalar.RED, 2, LINE_8, 0);
132 |                             circle(opencvMat, new Point(centers.get(i).getInt(0), centers.get(i).getInt(1)), 3, Scalar.RED, -1, 0, 0);
133 |                         }
134 |                     }
135 |                 }
136 | 
137 |                 putText(opencvMat, String.format("Number of people: %d", people.size()), new Point(10, 30), 4, 1.0, new Scalar(0, 255, 0, 0), 2, LINE_8, false);
138 |                 putText(opencvMat, String.format("Number of violators: %d", violators.size()), new Point(10, 60), 4, 1.0, new Scalar(0, 0, 255, 0), 2, LINE_8, false);
139 | 
140 |                 if (save2video) {
141 |                     recorder.record(frame2Mat.convert(opencvMat));
142 |                 }
143 | 
144 |                 imshow("Social Distancing Monitor", opencvMat);
145 |                 //    Press Esc key to quit
146 |                 if (waitKey(33) == 27) {
147 |                     if (save2video) {
148 |                         recorder.stop();
149 |                     }
150 |                     destroyAllWindows();
151 |                     break;
152 |                 }
153 |             }
154 |         }
155 |         if (save2video) {
156 |             recorder.stop();
157 |         }
158 |     }
159 | 
160 |     public static List<DetectedObject> getPredictedObjects(INDArray input) {
161 | 
162 |         INDArray[] outputs = model.output(input);
163 | 
164 |         List<DetectedObject> out = new ArrayList();
165 |         float detectionThreshold = 0.6f;
166 |         // Each cell had information for 3 boxes
167 |         int[] boxOffsets = {0, numClass + 5, (numClass + 5) * 2};
168 |         int exampleNum_in_batch = 0;
169 | 
170 |         for (int layerNum = 0; layerNum < 3; layerNum++) {
171 |             long gridWidth = outputs[layerNum].shape()[1];
172 |             long gridHeight = outputs[layerNum].shape()[2];
173 |             float cellWidth = yolowidth / gridWidth;
174 |             float cellHeight = yoloheight / gridHeight;
175 | 
176 |             for (int i = 0; i < gridHeight; i++) {
177 |                 for (int j = 0; j < gridWidth; j++) {
178 |                     float centerX;
179 |                     float centerY;
180 |                     float width;
181 |                     float height;
182 |                     int anchorInd;
183 | 
184 |                     for (int k = 0; k < 3; k++) {
185 |                         float prob = outputs[layerNum].getFloat(new int[]{exampleNum_in_batch, i, j, boxOffsets[k] + 4});
186 |                         if (prob > detectionThreshold) {
187 | //                            TODO: class probabilities does not make sense
188 |                             INDArray classes_scores = outputs[layerNum].get(
189 |                                     point(exampleNum_in_batch),
190 |                                     point(i),
191 |                                     point(j),
192 |                                     NDArrayIndex.interval(boxOffsets[k] + 5, boxOffsets[k] + numClass + 5));
193 | 
194 |                             centerX = outputs[layerNum].getFloat(new int[]{exampleNum_in_batch, i, j, boxOffsets[k] + 0});
195 |                             centerY = outputs[layerNum].getFloat(new int[]{exampleNum_in_batch, i, j, boxOffsets[k] + 1});
196 |                             width = outputs[layerNum].getFloat(new int[]{exampleNum_in_batch, i, j, boxOffsets[k] + 2});
197 |                             height = outputs[layerNum].getFloat(new int[]{exampleNum_in_batch, i, j, boxOffsets[k] + 3});
198 |                             anchorInd = (2 - layerNum) * 3 + k;
199 | 
200 |                             centerX = (float) ((sigmoid(centerX) + j) * cellWidth);
201 |                             centerY = (float) ((sigmoid(centerY) + i) * cellHeight);
202 |                             width = (float) (Math.exp(width)) * anchors[anchorInd][0];
203 |                             height = (float) (Math.exp(height)) * anchors[anchorInd][1];
204 | 
205 |                             out.add(new DetectedObject(k, centerX, centerY, width, height, classes_scores, prob));
206 |                         }
207 |                     }
208 |                 }
209 |             }
210 |         }
211 |         return out;
212 |     }
213 | }
214 | 


--------------------------------------------------------------------------------
/SocialDistancingChecker/src/main/java/SocialDistanceCheckerWebcam.java:
--------------------------------------------------------------------------------
  1 | import org.bytedeco.opencv.opencv_core.Mat;
  2 | import org.bytedeco.opencv.opencv_core.Point;
  3 | import org.bytedeco.opencv.opencv_core.Scalar;
  4 | import org.deeplearning4j.nn.graph.ComputationGraph;
  5 | import org.deeplearning4j.nn.layers.objdetect.DetectedObject;
  6 | import org.deeplearning4j.nn.layers.objdetect.YoloUtils;
  7 | import org.deeplearning4j.nn.modelimport.keras.KerasModelImport;
  8 | import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException;
  9 | import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfigurationException;
 10 | import org.nd4j.linalg.api.ndarray.INDArray;
 11 | import org.nd4j.linalg.factory.Nd4j;
 12 | import org.nd4j.linalg.indexing.NDArrayIndex;
 13 | 
 14 | import java.io.IOException;
 15 | import java.util.ArrayList;
 16 | import java.util.HashSet;
 17 | import java.util.List;
 18 | import java.util.Set;
 19 | 
 20 | import static org.bytedeco.opencv.global.opencv_highgui.*;
 21 | import static org.bytedeco.opencv.global.opencv_imgproc.*;
 22 | import static org.nd4j.common.util.MathUtils.sigmoid;
 23 | import static org.nd4j.linalg.indexing.NDArrayIndex.point;
 24 | import static org.nd4j.linalg.ops.transforms.Transforms.euclideanDistance;
 25 | 
 26 | //  Author: Choo Wilson(yptheangel)
 27 | //  Special thanks to yquemener for writing yolov3 output interpreter
 28 | //  This example shows a Social Distancing Monitor running on your webcam
 29 | //  Set camera config on CameraStreamer.java
 30 | //  This code is still experimental and the performance is questionable.
 31 | //  Frame streaming happens on another thread while main thread reads a frame from the frame streaming thread and run inference
 32 | //  Due to the slow inference, the bounding box is not synced with the latest frame
 33 | // TODO: Try using FFMpegGrabber instead of OpenCV videocapture
 34 | 
 35 | public class SocialDistanceCheckerWebcam {
 36 | 
 37 |     private static CameraStreamer camStreamer = new CameraStreamer();
 38 |     private static ComputationGraph model;
 39 |     //        COCO has 80 classes
 40 |     private static int numClass = 80;
 41 |     private static int[][] anchors = {{10, 13}, {16, 30}, {33, 23}, {30, 61}, {62, 45}, {59, 119}, {116, 90}, {156, 198}, {373, 326}};
 42 |     private static int yolowidth = 416;
 43 |     private static int yoloheight = 416;
 44 |     private static INDArray input = null;
 45 |     private static Mat cvFrame = new Mat();
 46 | 
 47 | 
 48 |     public static void main(String[] args) throws InvalidKerasConfigurationException, IOException, UnsupportedKerasConfigurationException, InterruptedException {
 49 |         int safeDistance = 80;
 50 | 
 51 |         String modelPATH = "C:\\Users\\ChooWilson\\Downloads\\yolov3_416_fixed.h5";
 52 |         model = KerasModelImport.importKerasModelAndWeights(modelPATH);
 53 |         Thread modelInitThread = new Thread(() -> model.init());
 54 |         modelInitThread.start();
 55 | 
 56 |         // Start camera streamer thread
 57 |         camStreamer.startCapture();
 58 |         camStreamer.camStreamRunnable = () -> {
 59 |             try {
 60 |                 cvFrame = camStreamer.getCVFrame();
 61 |             } catch (Exception e) {
 62 |                 System.err.println("Error capturing frame. Error message: " + e.toString());
 63 |             }
 64 |         };
 65 | 
 66 | //        TODO:: Remove relying on this delay in order to make sure the camera stream is running before calling imshow
 67 |         Thread.sleep(1000);
 68 |         //TODO: Why is it when cvFrame is not empty OR VideoCap object is opened, the model init thread is already dead?
 69 |         while (!cvFrame.empty()) {
 70 | //            System.out.println(modelInitThread.isAlive());
 71 |             if (modelInitThread.isAlive()) {
 72 |                 imshow("Social Distancing Monitor", cvFrame);
 73 |             } else {
 74 |                 input = camStreamer.getNDArray(cvFrame);
 75 |                 if (input != null) {
 76 |                     input = input.permute(0, 2, 3, 1);
 77 | 
 78 |                     List<DetectedObject> objs = getPredictedObjects(input);
 79 |                     YoloUtils.nms(objs, 0.4);
 80 | 
 81 |                     int w = cvFrame.cols();
 82 |                     int h = cvFrame.rows();
 83 |                     List<INDArray> centers = new ArrayList<>();
 84 |                     List<INDArray> people = new ArrayList<>();
 85 |                     Set violators = new HashSet<INDArray>();
 86 | 
 87 |                     int centerX;
 88 |                     int centerY;
 89 | 
 90 |                     for (DetectedObject obj : objs) {
 91 |                         if (obj.getPredictedClass() == 0) {
 92 |                             //            Scale the coordinates back to full size
 93 |                             centerX = (int) obj.getCenterX() * w / yolowidth;
 94 |                             centerY = (int) obj.getCenterY() * h / yoloheight;
 95 | 
 96 |                             circle(cvFrame, new Point(centerX, centerY), 2, new Scalar(0, 255, 0, 0), -1, 0, 0);
 97 |                             //            Draw bounding boxes on opencv mat
 98 |                             double[] xy1 = obj.getTopLeftXY();
 99 |                             double[] xy2 = obj.getBottomRightXY();
100 |                             //            Scale the coordinates back to full size
101 |                             xy1[0] = xy1[0] * w / yolowidth;
102 |                             xy1[1] = xy1[1] * h / yoloheight;
103 |                             xy2[0] = xy2[0] * w / yolowidth;
104 |                             xy2[1] = xy2[1] * h / yoloheight;
105 | 
106 |                             //Draw bounding box
107 |                             rectangle(cvFrame, new Point((int) xy1[0], (int) xy1[1]), new Point((int) xy2[0], (int) xy2[1]), new Scalar(0, 255, 0, 0), 2, LINE_8, 0);
108 |                             centers.add(Nd4j.create(new float[]{(float) centerX, (float) centerY}));
109 |                             people.add(Nd4j.create(new float[]{(float) xy1[0], (float) xy1[1], (float) xy2[0], (float) xy2[1]}));
110 |                         }
111 |                     }
112 |                     //        Calculate the euclidean distance between all pairs of center points
113 |                     for (int i = 0; i < centers.size(); i++) {
114 |                         for (int j = 0; j < centers.size(); j++) {
115 |                             double distance = euclideanDistance(centers.get(i), centers.get(j));
116 |                             if (distance < safeDistance && distance > 0) {
117 |                                 line(cvFrame, new Point(centers.get(i).getInt(0), centers.get(i).getInt(1)),
118 |                                         new Point(centers.get(j).getInt(0), centers.get(j).getInt(1)), Scalar.RED, 2, 1, 0);
119 | 
120 |                                 violators.add(centers.get(i));
121 |                                 violators.add(centers.get(j));
122 | 
123 |                                 int xmin = people.get(i).getInt(0);
124 |                                 int ymin = people.get(i).getInt(1);
125 |                                 int xmax = people.get(i).getInt(2);
126 |                                 int ymax = people.get(i).getInt(3);
127 | 
128 |                                 rectangle(cvFrame, new Point(xmin, ymin), new Point(xmax, ymax), Scalar.RED, 2, LINE_8, 0);
129 |                                 circle(cvFrame, new Point(centers.get(i).getInt(0), centers.get(i).getInt(1)), 3, Scalar.RED, -1, 0, 0);
130 |                             }
131 |                         }
132 |                     }
133 |                     putText(cvFrame, String.format("Number of people: %d", people.size()), new Point(10, 30), 4, 1.0, new Scalar(0, 255, 0, 0), 2, LINE_8, false);
134 |                     putText(cvFrame, String.format("Number of violators: %d", violators.size()), new Point(10, 60), 4, 1.0, new Scalar(0, 0, 255, 0), 2, LINE_8, false);
135 |                 }
136 |                 imshow("Social Distancing Monitor", cvFrame);
137 |             }
138 | //            imshow("Social Distancing Monitor", cvFrame);
139 | 
140 |             //    Press Esc key to quit
141 |             if (waitKey(33) == 27) {
142 |                 destroyAllWindows();
143 |                 break;
144 |             }
145 |         }
146 |     }
147 | 
148 |     public static List<DetectedObject> getPredictedObjects(INDArray input) {
149 | 
150 |         INDArray[] outputs = model.output(input);
151 |         List<DetectedObject> out = new ArrayList();
152 |         float detectionThreshold = 0.6f;
153 |         // Each cell had information for 3 boxes
154 |         int[] boxOffsets = {0, numClass + 5, (numClass + 5) * 2};
155 |         int exampleNum_in_batch = 0;
156 | 
157 |         for (int layerNum = 0; layerNum < 3; layerNum++) {
158 |             long gridWidth = outputs[layerNum].shape()[1];
159 |             long gridHeight = outputs[layerNum].shape()[2];
160 |             float cellWidth = yolowidth / gridWidth;
161 |             float cellHeight = yoloheight / gridHeight;
162 | 
163 |             for (int i = 0; i < gridHeight; i++) {
164 |                 for (int j = 0; j < gridWidth; j++) {
165 |                     float centerX;
166 |                     float centerY;
167 |                     float width;
168 |                     float height;
169 |                     int anchorInd;
170 | 
171 |                     for (int k = 0; k < 3; k++) {
172 |                         float prob = outputs[layerNum].getFloat(new int[]{exampleNum_in_batch, i, j, boxOffsets[k] + 4});
173 |                         if (prob > detectionThreshold) {
174 | //                            TODO: class probabilities does not make sense
175 |                             INDArray classes_scores = outputs[layerNum].get(
176 |                                     point(exampleNum_in_batch),
177 |                                     point(i),
178 |                                     point(j),
179 |                                     NDArrayIndex.interval(boxOffsets[k] + 5, boxOffsets[k] + numClass + 5));
180 | 
181 |                             centerX = outputs[layerNum].getFloat(new int[]{exampleNum_in_batch, i, j, boxOffsets[k] + 0});
182 |                             centerY = outputs[layerNum].getFloat(new int[]{exampleNum_in_batch, i, j, boxOffsets[k] + 1});
183 |                             width = outputs[layerNum].getFloat(new int[]{exampleNum_in_batch, i, j, boxOffsets[k] + 2});
184 |                             height = outputs[layerNum].getFloat(new int[]{exampleNum_in_batch, i, j, boxOffsets[k] + 3});
185 |                             anchorInd = (2 - layerNum) * 3 + k;
186 | 
187 |                             centerX = (float) ((sigmoid(centerX) + j) * cellWidth);
188 |                             centerY = (float) ((sigmoid(centerY) + i) * cellHeight);
189 |                             width = (float) (Math.exp(width)) * anchors[anchorInd][0];
190 |                             height = (float) (Math.exp(height)) * anchors[anchorInd][1];
191 | 
192 |                             out.add(new DetectedObject(k, centerX, centerY, width, height, classes_scores, prob));
193 |                         }
194 |                     }
195 |                 }
196 |             }
197 |         }
198 |         return out;
199 |     }
200 | }
201 | 


--------------------------------------------------------------------------------
/SocialDistancingChecker/src/main/resources/log4j.properties:
--------------------------------------------------------------------------------
1 | # Root logger option
2 | log4j.rootLogger=INFO, stdout
3 | 
4 | # Direct log messages to stdout
5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender
6 | log4j.appender.stdout.Target=System.out
7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n


--------------------------------------------------------------------------------
/crowd_topview_gray_1280x720.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/yptheangel/Social-Distancing-Monitor-Java/9ca972e4e84f0f99e2ff5e6fcc13953f037e95ac/crowd_topview_gray_1280x720.jpg


--------------------------------------------------------------------------------
/poster.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/yptheangel/Social-Distancing-Monitor-Java/9ca972e4e84f0f99e2ff5e6fcc13953f037e95ac/poster.png


--------------------------------------------------------------------------------