├── settings.gradle ├── whosthat.jpg ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── src └── main │ ├── resources │ ├── pedestrians.jpg │ └── application.properties │ └── java │ └── hr │ └── kn │ └── whosthat │ ├── camera │ ├── detection │ │ ├── PeopleDetector.java │ │ ├── PeopleDetectionResult.java │ │ ├── PeopleDetectorHog.java │ │ ├── PeopleDetectorMotion.java │ │ └── PeopleDetectorYOLO.java │ ├── CameraCommunicator.java │ └── cropper │ │ └── PhotoCropper.java │ ├── WhosThatApplication.java │ ├── notification │ └── TelegramService.java │ ├── DetectionScheduler.java │ └── config │ └── HttpClientConfig.java ├── devEnv └── opencv_tips.md ├── whosthat.env ├── .gitignore ├── Dockerfile ├── gradlew.bat ├── README.md └── gradlew /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'whosthat' 2 | -------------------------------------------------------------------------------- /whosthat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karlonovak/whosthat/HEAD/whosthat.jpg -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karlonovak/whosthat/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /src/main/resources/pedestrians.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karlonovak/whosthat/HEAD/src/main/resources/pedestrians.jpg -------------------------------------------------------------------------------- /src/main/java/hr/kn/whosthat/camera/detection/PeopleDetector.java: -------------------------------------------------------------------------------- 1 | package hr.kn.whosthat.camera.detection; 2 | 3 | public interface PeopleDetector { 4 | 5 | PeopleDetectionResult detectPeople(byte[] photo); 6 | 7 | } 8 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /devEnv/opencv_tips.md: -------------------------------------------------------------------------------- 1 | # Read image from file: 2 | var img = Imgcodecs.imread(imagePath); 3 | 4 | # Crop image: 5 | var roi = new Rect(207, 15, 728, 926); 6 | var cropped = new Mat(img, roi); 7 | 8 | # Write image to file: 9 | Imgcodecs.imwrite("/home/knovak/Pictures/opencv/cropped.jpg", cropped); 10 | 11 | # Byte array to Mat: 12 | var mat = Imgcodecs.imdecode(new MatOfByte(photo), Imgcodecs.IMREAD_COLOR); 13 | 14 | -------------------------------------------------------------------------------- /whosthat.env: -------------------------------------------------------------------------------- 1 | LOG_LEVEL=DEBUG 2 | 3 | CAM_ADDRESS=http://192.168.1.50:6500/ISAPI/Streaming/channels/101/picture 4 | CAM_USER= 5 | CAM_PASS= 6 | CAM_FREQUENCY_IN_S=5 7 | CAM_CROP_RECTANGLES= 8 | 9 | YOLO_WEIGHTS=/opt/darknet/yolov3.weights 10 | YOLO_CONFIG=/opt/darknet/cfg/yolov3.cfg 11 | 12 | TELEGRAM_TOKEN=1785646214:ABHEd4GeiRNFRd_0dsvMKIgGmMVsDIBbosP 13 | TELEGRAM_CHAT_ID=-527125351 14 | TELEGRAM_NOTIFICATION_THRESHOLD_IN_S=60 -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | logging.level.hr.kn=${LOG_LEVEL:#{INFO}} 2 | 3 | camera.username=${CAM_USER} 4 | camera.password=${CAM_PASS} 5 | camera.address=${CAM_ADDRESS} 6 | camera.frequency=${CAM_FREQUENCY_IN_S:#{5}} 7 | camera.crop.rectangles=${CAM_CROP_RECTANGLES:#{null}} 8 | 9 | yolo.weights=${YOLO_WEIGHTS} 10 | yolo.config=${YOLO_CONFIG} 11 | 12 | telegram.token=${TELEGRAM_TOKEN} 13 | telegram.chatId=${TELEGRAM_CHAT_ID} 14 | telegram.notification.threshold=${TELEGRAM_NOTIFICATION_THRESHOLD_IN_S:#{60}} 15 | -------------------------------------------------------------------------------- /src/main/java/hr/kn/whosthat/WhosThatApplication.java: -------------------------------------------------------------------------------- 1 | package hr.kn.whosthat; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.scheduling.annotation.EnableScheduling; 6 | import origami.Origami; 7 | 8 | @SpringBootApplication 9 | @EnableScheduling 10 | public class WhosThatApplication { 11 | 12 | public static void main(String[] args) { 13 | Origami.init(); 14 | SpringApplication.run(WhosThatApplication.class); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | .gradle 3 | build/ 4 | !gradle/wrapper/gradle-wrapper.jar 5 | !**/src/main/**/build/ 6 | !**/src/test/**/build/ 7 | 8 | ### STS ### 9 | .apt_generated 10 | .classpath 11 | .factorypath 12 | .project 13 | .settings 14 | .springBeans 15 | .sts4-cache 16 | bin/ 17 | !**/src/main/**/bin/ 18 | !**/src/test/**/bin/ 19 | 20 | ### IntelliJ IDEA ### 21 | .idea 22 | *.iws 23 | *.iml 24 | *.ipr 25 | out/ 26 | !**/src/main/**/out/ 27 | !**/src/test/**/out/ 28 | 29 | ### NetBeans ### 30 | /nbproject/private/ 31 | /nbbuild/ 32 | /dist/ 33 | /nbdist/ 34 | /.nb-gradle/ 35 | 36 | ### VS Code ### 37 | .vscode/ 38 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:11.0.2-jre-slim-stretch 2 | 3 | WORKDIR /opt 4 | VOLUME /tmp 5 | 6 | ARG JAR_FILE 7 | ADD ${JAR_FILE} whosthat.jar 8 | 9 | RUN mkdir -p work 10 | 11 | RUN pwd 12 | 13 | RUN apt-get update && apt-get install -y git 14 | RUN apt-get update && apt-get install -y wget 15 | 16 | RUN git clone https://github.com/pjreddie/darknet.git 17 | RUN cd darknet && wget https://pjreddie.com/media/files/yolov3.weights 18 | 19 | ENTRYPOINT java $JAVA_OPTS \ 20 | -Djava.net.preferIPv4Stack=true \ 21 | -Dcom.sun.management.jmxremote \ 22 | -Dcom.sun.management.jmxremote.port=9000 \ 23 | -Dcom.sun.management.jmxremote.authenticate=false \ 24 | -Dcom.sun.management.jmxremote.ssl=false \ 25 | -Dcom.sun.management.jmxremote.rmi.port=9000 \ 26 | -Dcom.sun.management.jmxremote.local.only=false \ 27 | -Djava.security.egd=file:/dev/./urandom \ 28 | -jar whosthat.jar 29 | -------------------------------------------------------------------------------- /src/main/java/hr/kn/whosthat/camera/detection/PeopleDetectionResult.java: -------------------------------------------------------------------------------- 1 | package hr.kn.whosthat.camera.detection; 2 | 3 | public class PeopleDetectionResult { 4 | 5 | private boolean peopleDetected; 6 | private byte[] image; 7 | private float confidence; 8 | 9 | private PeopleDetectionResult() { 10 | } 11 | 12 | public static PeopleDetectionResult detected(byte[] image, float confidence) { 13 | var result = new PeopleDetectionResult(); 14 | result.peopleDetected = true; 15 | result.image = image; 16 | result.confidence = confidence; 17 | return result; 18 | } 19 | 20 | public static PeopleDetectionResult notDetected() { 21 | var result = new PeopleDetectionResult(); 22 | result.peopleDetected = false; 23 | return result; 24 | } 25 | 26 | public boolean arePeopleDetected() { 27 | return peopleDetected; 28 | } 29 | 30 | public byte[] getImage() { 31 | return image; 32 | } 33 | 34 | public float getConfidence() { 35 | return confidence; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/hr/kn/whosthat/camera/CameraCommunicator.java: -------------------------------------------------------------------------------- 1 | package hr.kn.whosthat.camera; 2 | 3 | import org.springframework.beans.factory.annotation.Value; 4 | import org.springframework.http.MediaType; 5 | import org.springframework.stereotype.Service; 6 | import org.springframework.web.reactive.function.client.WebClient; 7 | import reactor.core.publisher.Flux; 8 | import reactor.core.publisher.Mono; 9 | 10 | @Service 11 | public class CameraCommunicator { 12 | 13 | private final String cameraAddress; 14 | private final WebClient cameraHttpClient; 15 | private static final String CAMERA_MOTION = "http://192.168.6.20:65002/ISAPI/Event/notification/alertStream"; 16 | 17 | public CameraCommunicator(WebClient cameraHttpClient, @Value("${camera.address}") String cameraAddress) { 18 | this.cameraAddress = cameraAddress; 19 | this.cameraHttpClient = cameraHttpClient; 20 | } 21 | 22 | public Mono acquireCameraPhoto() { 23 | return cameraHttpClient.get() 24 | .uri(cameraAddress) 25 | .retrieve() 26 | .bodyToMono(byte[].class); 27 | } 28 | 29 | public Flux acquireCameraMotions() { 30 | return cameraHttpClient.get() 31 | .uri(CAMERA_MOTION) 32 | .accept(MediaType.TEXT_EVENT_STREAM) 33 | .retrieve() 34 | .bodyToFlux(String.class); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/hr/kn/whosthat/camera/detection/PeopleDetectorHog.java: -------------------------------------------------------------------------------- 1 | package hr.kn.whosthat.camera.detection; 2 | 3 | import org.opencv.core.*; 4 | import org.opencv.imgcodecs.Imgcodecs; 5 | import org.opencv.imgproc.Imgproc; 6 | import org.opencv.objdetect.HOGDescriptor; 7 | import org.springframework.stereotype.Service; 8 | 9 | import java.util.Arrays; 10 | 11 | @Service 12 | public class PeopleDetectorHog implements PeopleDetector { 13 | 14 | private final HOGDescriptor hog = new HOGDescriptor(); 15 | 16 | public PeopleDetectorHog() { 17 | hog.setSVMDetector(HOGDescriptor.getDefaultPeopleDetector()); 18 | } 19 | 20 | @Override 21 | public PeopleDetectionResult detectPeople(byte[] photo) { 22 | var mat = Imgcodecs.imdecode(new MatOfByte(photo), Imgcodecs.IMREAD_COLOR); 23 | var locations = new MatOfRect(); 24 | var weights = new MatOfDouble(); 25 | 26 | hog.detectMultiScale(mat, locations, weights); 27 | 28 | if (locations.rows() > 0 && preciseWeightFound(weights)) { 29 | var locationsArray = locations.toArray(); 30 | for (Rect rect : locationsArray) { 31 | Imgproc.rectangle(mat, rect.tl(), rect.br(), new Scalar(0, 255, 0, 255), 3); 32 | } 33 | return PeopleDetectionResult.detected(photo, 0.0f); 34 | } else { 35 | return PeopleDetectionResult.notDetected(); 36 | } 37 | } 38 | 39 | private boolean preciseWeightFound(MatOfDouble weights) { 40 | return Arrays.stream(weights.toArray()) 41 | .filter(weight -> weight > 0.8) 42 | .findAny() 43 | .isPresent(); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/hr/kn/whosthat/notification/TelegramService.java: -------------------------------------------------------------------------------- 1 | package hr.kn.whosthat.notification; 2 | 3 | import com.pengrad.telegrambot.TelegramBot; 4 | import com.pengrad.telegrambot.request.SendPhoto; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.beans.factory.annotation.Value; 8 | import org.springframework.stereotype.Service; 9 | 10 | @Service 11 | public class TelegramService { 12 | 13 | private final Logger logger = LoggerFactory.getLogger(TelegramService.class); 14 | 15 | private final Integer chatId; 16 | private final TelegramBot telegramBot; 17 | private final long notificationThreshold; 18 | 19 | private long lastNotificationTime = 0; 20 | 21 | public TelegramService(@Value("${telegram.token}") String telegramToken, 22 | @Value("${telegram.chatId}") Integer chatId, 23 | @Value("${telegram.notification.threshold}") Integer notificationThreshold) { 24 | this.telegramBot = new TelegramBot(telegramToken); 25 | this.chatId = chatId; 26 | this.notificationThreshold = notificationThreshold * 1000; // convert to millis 27 | } 28 | 29 | public synchronized void sendPhoto(byte[] photo, String caption) { 30 | if (minutePassedSinceLastNotification()) { 31 | logger.info("Notifying user via Telegram"); 32 | var sendPhotoRequest = new SendPhoto(chatId, photo).caption(caption); 33 | telegramBot.execute(sendPhotoRequest); 34 | lastNotificationTime = System.currentTimeMillis(); 35 | } 36 | } 37 | 38 | // Don't send notification more than once a minute. 39 | private boolean minutePassedSinceLastNotification() { 40 | return System.currentTimeMillis() - lastNotificationTime > notificationThreshold; 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/hr/kn/whosthat/DetectionScheduler.java: -------------------------------------------------------------------------------- 1 | package hr.kn.whosthat; 2 | 3 | import hr.kn.whosthat.camera.CameraCommunicator; 4 | import hr.kn.whosthat.camera.cropper.PhotoCropper; 5 | import hr.kn.whosthat.camera.detection.PeopleDetector; 6 | import hr.kn.whosthat.notification.TelegramService; 7 | import org.springframework.beans.factory.annotation.Value; 8 | import org.springframework.stereotype.Component; 9 | 10 | import java.util.concurrent.ScheduledThreadPoolExecutor; 11 | import java.util.concurrent.TimeUnit; 12 | 13 | @Component 14 | public class DetectionScheduler { 15 | 16 | private final CameraCommunicator cameraCommunicator; 17 | private final TelegramService telegramService; 18 | private final PeopleDetector peopleDetector; 19 | private final PhotoCropper photoCropper; 20 | 21 | public DetectionScheduler(CameraCommunicator cameraCommunicator, 22 | TelegramService telegramService, 23 | PeopleDetector peopleDetector, 24 | PhotoCropper photoCropper, 25 | @Value("${camera.frequency}") Integer camFrequency) { 26 | this.cameraCommunicator = cameraCommunicator; 27 | this.telegramService = telegramService; 28 | this.peopleDetector = peopleDetector; 29 | this.photoCropper = photoCropper; 30 | startDetector(camFrequency); 31 | } 32 | 33 | public void startDetector(Integer camFrequency) { 34 | new ScheduledThreadPoolExecutor(1).scheduleAtFixedRate( 35 | () -> cameraCommunicator 36 | .acquireCameraPhoto() 37 | .subscribe(camSnap -> { 38 | var croppedSnap = photoCropper.removePartsOfImage(camSnap); 39 | var detection = peopleDetector.detectPeople(croppedSnap); 40 | if (detection.arePeopleDetected()) { 41 | var message = String.format("A person! %.2f%% sure.", detection.getConfidence() * 100); 42 | telegramService.sendPhoto(camSnap, message); 43 | } 44 | }), 45 | 0, camFrequency, TimeUnit.SECONDS); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/hr/kn/whosthat/camera/cropper/PhotoCropper.java: -------------------------------------------------------------------------------- 1 | package hr.kn.whosthat.camera.cropper; 2 | 3 | import org.opencv.core.*; 4 | import org.opencv.imgcodecs.Imgcodecs; 5 | import org.opencv.imgproc.Imgproc; 6 | import org.springframework.beans.factory.annotation.Value; 7 | import org.springframework.stereotype.Service; 8 | 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | 12 | @Service 13 | public class PhotoCropper { 14 | 15 | private final List cropRectangles; 16 | 17 | public PhotoCropper(@Value("${camera.crop.rectangles}") String cropRectangles) { 18 | this.cropRectangles = generateCropRectangles(cropRectangles); 19 | } 20 | 21 | private List generateCropRectangles(String cropRectangles) { 22 | var rects = new ArrayList(); 23 | if (cropRectangles != null) { 24 | var rectangleCoordinates = cropRectangles.split("_"); 25 | for (String rectangleCoordinate : rectangleCoordinates) { 26 | var firstCoordinate = rectangleCoordinate.split(",")[0]; 27 | var secondCoordinate = rectangleCoordinate.split(",")[1]; 28 | 29 | var firstX = firstCoordinate.split("x")[0]; 30 | var firstY = firstCoordinate.split("x")[1]; 31 | var secondX = secondCoordinate.split("x")[0]; 32 | var secondY = secondCoordinate.split("x")[1]; 33 | 34 | rects.add(new Rect( 35 | new Point(Integer.parseInt(firstX), Integer.parseInt(firstY)), 36 | new Point(Integer.parseInt(secondX), Integer.parseInt(secondY))) 37 | ); 38 | } 39 | } 40 | return rects; 41 | } 42 | 43 | public byte[] removePartsOfImage(byte[] image) { 44 | var mat = Imgcodecs.imdecode(new MatOfByte(image), Imgcodecs.IMREAD_COLOR); 45 | for (Rect rectangle : cropRectangles) { 46 | Imgproc.rectangle(mat, rectangle.br(), rectangle.tl(), new Scalar(0, 0, 0), -1); 47 | } 48 | return matToByte(mat); 49 | } 50 | 51 | private byte[] matToByte(Mat mat) { 52 | int length = (int) (mat.total() * mat.elemSize()); 53 | byte[] buffer = new byte[length]; 54 | mat.get(0, 0, buffer); 55 | MatOfByte mem = new MatOfByte(); 56 | Imgcodecs.imencode(".jpg", mat, mem); 57 | return mem.toArray(); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/hr/kn/whosthat/camera/detection/PeopleDetectorMotion.java: -------------------------------------------------------------------------------- 1 | package hr.kn.whosthat.camera.detection; 2 | 3 | import org.opencv.core.*; 4 | import org.opencv.imgcodecs.Imgcodecs; 5 | import org.opencv.imgproc.Imgproc; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.stereotype.Service; 9 | 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | 13 | @Service 14 | public class PeopleDetectorMotion implements PeopleDetector { 15 | 16 | private final Logger logger = LoggerFactory.getLogger(PeopleDetectorMotion.class); 17 | 18 | private Mat frame = new Mat(); 19 | private Mat lastFrame = new Mat(); 20 | private Mat gray = new Mat(); 21 | private Mat frameDelta = new Mat(); 22 | private Mat thresh = new Mat(); 23 | 24 | private Integer curr = 0; 25 | 26 | @Override 27 | public PeopleDetectionResult detectPeople(byte[] photo) { 28 | frame = Imgcodecs.imdecode(new MatOfByte(photo), Imgcodecs.IMREAD_COLOR); 29 | var roi = new Rect(1000, 15, 920, 1500); 30 | var cropped = new Mat(frame, roi); 31 | 32 | if (curr == 0) { 33 | Imgproc.cvtColor(cropped, lastFrame, Imgproc.COLOR_BGR2GRAY); 34 | Imgproc.GaussianBlur(lastFrame, lastFrame, new Size(21, 21), 0); 35 | curr++; 36 | return PeopleDetectionResult.notDetected(); 37 | } 38 | 39 | Imgproc.cvtColor(cropped, gray, Imgproc.COLOR_BGR2GRAY); 40 | Imgproc.GaussianBlur(gray, gray, new Size(21, 21), 0); 41 | 42 | //compute difference between first frame and current frame 43 | Core.absdiff(lastFrame, gray, frameDelta); 44 | Imgproc.threshold(frameDelta, thresh, 25, 255, Imgproc.THRESH_BINARY); 45 | 46 | List cnts = new ArrayList<>(); 47 | Imgproc.dilate(thresh, thresh, new Mat(), new Point(-1, -1), 2); 48 | Imgproc.findContours(thresh, cnts, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE); 49 | 50 | for (MatOfPoint cnt : cnts) { 51 | var contourArea = Imgproc.contourArea(cnt); 52 | logger.debug("Motion detected with contourArea {}", contourArea); 53 | if (contourArea < 100_000) { 54 | continue; 55 | } 56 | 57 | curr++; 58 | Imgproc.cvtColor(cropped, lastFrame, Imgproc.COLOR_BGR2GRAY); 59 | Imgproc.GaussianBlur(lastFrame, lastFrame, new Size(21, 21), 0); 60 | return PeopleDetectionResult.detected(photo, 0.0f); 61 | } 62 | 63 | return PeopleDetectionResult.notDetected(); 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /src/main/java/hr/kn/whosthat/config/HttpClientConfig.java: -------------------------------------------------------------------------------- 1 | package hr.kn.whosthat.config; 2 | 3 | import org.apache.hc.client5.http.auth.AuthScope; 4 | import org.apache.hc.client5.http.auth.CredentialsProvider; 5 | import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; 6 | import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; 7 | import org.apache.hc.client5.http.impl.async.HttpAsyncClients; 8 | import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; 9 | import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; 10 | import org.apache.hc.core5.pool.PoolReusePolicy; 11 | import org.apache.hc.core5.util.TimeValue; 12 | import org.springframework.beans.factory.annotation.Value; 13 | import org.springframework.context.annotation.Bean; 14 | import org.springframework.context.annotation.Configuration; 15 | import org.springframework.http.client.reactive.HttpComponentsClientHttpConnector; 16 | import org.springframework.web.reactive.function.client.ExchangeStrategies; 17 | import org.springframework.web.reactive.function.client.WebClient; 18 | 19 | import java.net.MalformedURLException; 20 | import java.net.URL; 21 | 22 | @Configuration 23 | public class HttpClientConfig { 24 | 25 | @Bean 26 | public WebClient cameraHttpClient(@Value("${camera.username}") String camUser, 27 | @Value("${camera.password}") String camPassword, 28 | @Value("${camera.address}") String camAddress) throws MalformedURLException { 29 | var httpClient = createHttpClientForCamera(camUser, camPassword, camAddress); 30 | var clientConnector = new HttpComponentsClientHttpConnector(httpClient); 31 | 32 | return WebClient.builder() 33 | .clientConnector(clientConnector) 34 | .exchangeStrategies(ExchangeStrategies.builder() 35 | .codecs(configurer -> configurer 36 | .defaultCodecs() 37 | .maxInMemorySize(16 * 1024 * 1024)) 38 | .build()) 39 | .build(); 40 | } 41 | 42 | private CloseableHttpAsyncClient createHttpClientForCamera(String camUser, String camPassword, String camAddress) throws MalformedURLException { 43 | var provider = createCredentialsProvider(camUser, camPassword, camAddress); 44 | 45 | var connectionManager = PoolingAsyncClientConnectionManagerBuilder.create() 46 | .setConnPoolPolicy(PoolReusePolicy.LIFO) 47 | .setMaxConnTotal(100) 48 | .setMaxConnPerRoute(100) 49 | .setConnectionTimeToLive(TimeValue.ofDays(10_000L)) 50 | .build(); 51 | 52 | return HttpAsyncClients.custom() 53 | .setConnectionManager(connectionManager) 54 | .setDefaultCredentialsProvider(provider) 55 | .build(); 56 | } 57 | 58 | private CredentialsProvider createCredentialsProvider(String camUser, String camPassword, String camAddress) throws MalformedURLException { 59 | var url = new URL(camAddress); 60 | var provider = new BasicCredentialsProvider(); 61 | var credentials = new UsernamePasswordCredentials(camUser, camPassword.toCharArray()); 62 | provider.setCredentials(new AuthScope(url.getHost(), url.getPort()), credentials); 63 | return provider; 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## About The Project 2 | whosthat is app for detecting people on CCTV/cameras and sending notifications with photo snapshots using Telegram bot. 3 | 4 | ## whosthat in action 5 | This is what it looks like on my phone in action. 6 |
7 | Never misses a single person, not even in infrared mode (during night time) :) 8 |
9 | (blurred for privacy reasons) 10 |
11 | ![alt text](whosthat.jpg "whosthat in action on my phone") 12 | ### Built With 13 | * Java 14 14 | * Spring Boot 2.4.2 15 | * OpenCV 4.2.0-0 16 | * darknet/yolo 17 | 18 | ### Tested with 19 | * Docker 20.10.5 20 | * Gradle 6.7.1 21 | * HikVision VR DS-7604NI-K1 / 4P 22 | 23 | ## Getting Started 24 | 25 | ### Prerequisites 26 | 27 | To be able to run whosthat, working Docker installation and gradle should be available on your machine: 28 | * check if Docker is working 29 | ```sh 30 | docker -v 31 | ``` 32 | If version not printed, make sure to install Docker first. 33 | * check if gradle is working 34 | ```sh 35 | gradle -v 36 | ``` 37 | If version not printed, make sure to install gradle first. 38 | ### Installation 39 | 40 | 1. Clone the repo 41 | ```sh 42 | git clone https://github.com/karlonovak/whosthat 43 | cd whosthat 44 | ``` 45 | 2. Build app and Docker image (might take a while to download darknet model) 46 | ```sh 47 | gradle docker 48 | ``` 49 | 3. Check if Docker image created 50 | ```sh 51 | docker images | grep whosthat 52 | ``` 53 | whosthat image should be printed out. 54 | 4. Fill in whosthat.env file with your data (see instructions below!) 55 | 5. ```sh 56 | docker run --env-file whosthat.env whosthat 57 | ``` 58 | 6. If everything went well, success message should be shown: 59 | ```sh 60 | Started WhosThatApplication in x.xx seconds (JVM running for y.yyy) 61 | ``` 62 | 63 | ## Environment file 64 | 65 | whosthat comes with provided example env file, whosthat.env file inside project root. 66 | Environment file should be filled in with your camera's data. Available variables are: 67 | 1. [Optional] configurable log level for app, best left default 68 | ```sh 69 | LOG_LEVEL=DEBUG 70 | ``` 71 | 2. [Mandatory] this param points to an HTTP address of a JPEG image of your camera. 72 | One such example (for my type of Hikvision camera) is shown below. 73 | ```sh 74 | CAM_ADDRESS=http://192.168.1.50:6500/ISAPI/Streaming/channels/101/picture 75 | ``` 76 | 3. [Optional] BASIC/DIGEST auth username if needed 77 | ```sh 78 | CAM_USER=admin 79 | ``` 80 | 4. [Optional] BASIC/DIGEST auth password if needed 81 | ```sh 82 | CAM_PASS=admin1234 83 | ``` 84 | 5. [Optional] frequency at which snapshot is taken from the camera 85 | ```sh 86 | CAM_FREQUENCY_IN_S=5 87 | ``` 88 | 6. [Optional] list of rectangles to exclude from detection if camera snapshots contains parts that 89 | don't need to be included for detection, example with two rectangles (delimited by '_'): 1980x0,2688x480_2400x0,2688x1520 90 | ```sh 91 | CAM_CROP_RECTANGLES= 92 | ``` 93 | 7. [Optional] Path to darknet weights, default is the one suited for Docker container (automatically downloaded on image building) 94 | ```sh 95 | YOLO_WEIGHTS=/opt/darknet/yolov3.weights 96 | ``` 97 | 8. [Optional] Path to darknet config, default is the one suited for Docker container (automatically downloaded on image building) 98 | ```sh 99 | YOLO_CONFIG=/opt/darknet/cfg/yolov3.cfg 100 | ``` 101 | 9. [Mandatory] Telegram token (https://core.telegram.org/bots#6-botfather) 102 | ```sh 103 | TELEGRAM_TOKEN=1785646214:ABHEd4GeiRNFRd_0dsvMKIgGmMVsDIBbosP 104 | ``` 105 | 10. [Mandatory] Chat ID of a group that bot should use to send detected images to 106 | (https://stackoverflow.com/questions/32423837/telegram-bot-how-to-get-a-group-chat-id) 107 | ```sh 108 | TELEGRAM_CHAT_ID=-527125351 109 | ``` 110 | 11. [Optional] To avoid spam, declare a threshold for Telegram notifications 111 | ```sh 112 | TELEGRAM_NOTIFICATION_THRESHOLD_IN_S=60 113 | ``` 114 | 115 | 116 | -------------------------------------------------------------------------------- /src/main/java/hr/kn/whosthat/camera/detection/PeopleDetectorYOLO.java: -------------------------------------------------------------------------------- 1 | package hr.kn.whosthat.camera.detection; 2 | 3 | import org.opencv.core.*; 4 | import org.opencv.dnn.Dnn; 5 | import org.opencv.dnn.Net; 6 | import org.opencv.imgcodecs.Imgcodecs; 7 | import org.opencv.imgproc.Imgproc; 8 | import org.opencv.utils.Converters; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | import org.springframework.beans.factory.annotation.Value; 12 | import org.springframework.context.annotation.Primary; 13 | import org.springframework.stereotype.Service; 14 | 15 | import java.util.ArrayList; 16 | import java.util.Collections; 17 | import java.util.List; 18 | 19 | @Primary 20 | @Service 21 | public class PeopleDetectorYOLO implements PeopleDetector { 22 | 23 | private final Logger logger = LoggerFactory.getLogger(PeopleDetectorYOLO.class); 24 | 25 | private final Net net; 26 | private final List outBlobNames; 27 | 28 | public PeopleDetectorYOLO(@Value("${yolo.weights}") String yoloWeights, 29 | @Value("${yolo.config}") String yoloConfig) { 30 | this.net = Dnn.readNetFromDarknet(yoloConfig, yoloWeights); 31 | this.outBlobNames = getOutputNames(net); 32 | } 33 | 34 | private static List getOutputNames(Net net) { 35 | List outLayers = net.getUnconnectedOutLayers().toList(); 36 | List layersNames = net.getLayerNames(); 37 | 38 | List names = new ArrayList<>(); 39 | outLayers.forEach((item) -> names.add(layersNames.get(item - 1))); // unfold and create R-CNN layers from the loaded YOLO model 40 | return names; 41 | } 42 | 43 | @Override 44 | public PeopleDetectionResult detectPeople(byte[] photo) { 45 | var frame = Imgcodecs.imdecode(new MatOfByte(photo), Imgcodecs.IMREAD_COLOR); 46 | var blob = Dnn.blobFromImage(frame, 0.00392, new Size(608, 608), new Scalar(0), true, false); 47 | 48 | List result = new ArrayList<>(); 49 | net.setInput(blob); 50 | net.forward(result, outBlobNames); 51 | 52 | float confThreshold = 0.85f; 53 | List classIds = new ArrayList<>(); 54 | List confidences = new ArrayList<>(); 55 | List rectangles = new ArrayList<>(); 56 | for (Mat level : result) { 57 | // each row is a candidate detection, the 1st 4 numbers are 58 | // [center_x, center_y, width, height], followed by (N-4) class probabilities 59 | for (int j = 0; j < level.rows(); ++j) { 60 | Mat row = level.row(j); 61 | Mat scores = row.colRange(5, level.cols()); 62 | Core.MinMaxLocResult mm = Core.minMaxLoc(scores); 63 | float confidence = (float) mm.maxVal; 64 | Point classIdPoint = mm.maxLoc; 65 | if (confidence > confThreshold) { 66 | int centerX = (int) (row.get(0, 0)[0] * frame.cols()); //scaling for drawing the bounding boxes 67 | int centerY = (int) (row.get(0, 1)[0] * frame.rows()); 68 | int width = (int) (row.get(0, 2)[0] * frame.cols()); 69 | int height = (int) (row.get(0, 3)[0] * frame.rows()); 70 | int left = centerX - width / 2; 71 | int top = centerY - height / 2; 72 | 73 | classIds.add((int) classIdPoint.x); 74 | confidences.add(confidence); 75 | rectangles.add(new Rect(left, top, width, height)); 76 | } 77 | } 78 | } 79 | 80 | if (classIds.contains(YOLOObjectClass.PERSON.classId)) { 81 | logger.info("Person detected!"); 82 | float nmsThresh = 0.85f; 83 | MatOfFloat confs = new MatOfFloat(Converters.vector_float_to_Mat(confidences)); 84 | Rect[] boxesArray = rectangles.toArray(new Rect[0]); 85 | MatOfRect boxes = new MatOfRect(boxesArray); 86 | MatOfInt indices = new MatOfInt(); 87 | Dnn.NMSBoxes(boxes, confs, confThreshold, nmsThresh, indices); // We draw the bounding boxes for objects here 88 | 89 | int[] ind = indices.toArray(); 90 | for (int i = 0; i < ind.length; ++i) { 91 | logger.debug("Confidence is " + confidences.get(i) + " and class is " + classIds.get(i)); 92 | int idx = ind[i]; 93 | Rect box = boxesArray[idx]; 94 | Imgproc.rectangle(frame, box.tl(), box.br(), new Scalar(0, 0, 255), 2); 95 | 96 | } 97 | return PeopleDetectionResult.detected(photo, Collections.max(confidences)); 98 | } 99 | 100 | return PeopleDetectionResult.notDetected(); 101 | } 102 | 103 | enum YOLOObjectClass { 104 | PERSON(0); 105 | 106 | private final Integer classId; 107 | 108 | YOLOObjectClass(Integer classId) { 109 | this.classId = classId; 110 | } 111 | } 112 | 113 | } 114 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # 4 | # Copyright 2015 the original author or authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | ## 21 | ## Gradle start up script for UN*X 22 | ## 23 | ############################################################################## 24 | 25 | # Attempt to set APP_HOME 26 | # Resolve links: $0 may be a link 27 | PRG="$0" 28 | # Need this for relative symlinks. 29 | while [ -h "$PRG" ] ; do 30 | ls=`ls -ld "$PRG"` 31 | link=`expr "$ls" : '.*-> \(.*\)$'` 32 | if expr "$link" : '/.*' > /dev/null; then 33 | PRG="$link" 34 | else 35 | PRG=`dirname "$PRG"`"/$link" 36 | fi 37 | done 38 | SAVED="`pwd`" 39 | cd "`dirname \"$PRG\"`/" >/dev/null 40 | APP_HOME="`pwd -P`" 41 | cd "$SAVED" >/dev/null 42 | 43 | APP_NAME="Gradle" 44 | APP_BASE_NAME=`basename "$0"` 45 | 46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 48 | 49 | # Use the maximum available, or set MAX_FD != -1 to use that value. 50 | MAX_FD="maximum" 51 | 52 | warn () { 53 | echo "$*" 54 | } 55 | 56 | die () { 57 | echo 58 | echo "$*" 59 | echo 60 | exit 1 61 | } 62 | 63 | # OS specific support (must be 'true' or 'false'). 64 | cygwin=false 65 | msys=false 66 | darwin=false 67 | nonstop=false 68 | case "`uname`" in 69 | CYGWIN* ) 70 | cygwin=true 71 | ;; 72 | Darwin* ) 73 | darwin=true 74 | ;; 75 | MINGW* ) 76 | msys=true 77 | ;; 78 | NONSTOP* ) 79 | nonstop=true 80 | ;; 81 | esac 82 | 83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 84 | 85 | 86 | # Determine the Java command to use to start the JVM. 87 | if [ -n "$JAVA_HOME" ] ; then 88 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 89 | # IBM's JDK on AIX uses strange locations for the executables 90 | JAVACMD="$JAVA_HOME/jre/sh/java" 91 | else 92 | JAVACMD="$JAVA_HOME/bin/java" 93 | fi 94 | if [ ! -x "$JAVACMD" ] ; then 95 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 96 | 97 | Please set the JAVA_HOME variable in your environment to match the 98 | location of your Java installation." 99 | fi 100 | else 101 | JAVACMD="java" 102 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 103 | 104 | Please set the JAVA_HOME variable in your environment to match the 105 | location of your Java installation." 106 | fi 107 | 108 | # Increase the maximum file descriptors if we can. 109 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 110 | MAX_FD_LIMIT=`ulimit -H -n` 111 | if [ $? -eq 0 ] ; then 112 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 113 | MAX_FD="$MAX_FD_LIMIT" 114 | fi 115 | ulimit -n $MAX_FD 116 | if [ $? -ne 0 ] ; then 117 | warn "Could not set maximum file descriptor limit: $MAX_FD" 118 | fi 119 | else 120 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 121 | fi 122 | fi 123 | 124 | # For Darwin, add options to specify how the application appears in the dock 125 | if $darwin; then 126 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 127 | fi 128 | 129 | # For Cygwin or MSYS, switch paths to Windows format before running java 130 | if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then 131 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 132 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 133 | 134 | JAVACMD=`cygpath --unix "$JAVACMD"` 135 | 136 | # We build the pattern for arguments to be converted via cygpath 137 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 138 | SEP="" 139 | for dir in $ROOTDIRSRAW ; do 140 | ROOTDIRS="$ROOTDIRS$SEP$dir" 141 | SEP="|" 142 | done 143 | OURCYGPATTERN="(^($ROOTDIRS))" 144 | # Add a user-defined pattern to the cygpath arguments 145 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 146 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 147 | fi 148 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 149 | i=0 150 | for arg in "$@" ; do 151 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 152 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 153 | 154 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 155 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 156 | else 157 | eval `echo args$i`="\"$arg\"" 158 | fi 159 | i=`expr $i + 1` 160 | done 161 | case $i in 162 | 0) set -- ;; 163 | 1) set -- "$args0" ;; 164 | 2) set -- "$args0" "$args1" ;; 165 | 3) set -- "$args0" "$args1" "$args2" ;; 166 | 4) set -- "$args0" "$args1" "$args2" "$args3" ;; 167 | 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 168 | 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 169 | 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 170 | 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 171 | 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 172 | esac 173 | fi 174 | 175 | # Escape application args 176 | save () { 177 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 178 | echo " " 179 | } 180 | APP_ARGS=`save "$@"` 181 | 182 | # Collect all arguments for the java command, following the shell quoting and substitution rules 183 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 184 | 185 | exec "$JAVACMD" "$@" 186 | --------------------------------------------------------------------------------