├── .gitignore ├── docs └── images │ ├── arch3.png │ └── object-size-detector.png ├── resources └── config.json ├── cmake ├── pahomqtt.cmake ├── FindDemoOpenVINO.cmake └── osdetect.cmake ├── LICENSE ├── setup.sh ├── CMakeLists.txt ├── application ├── include │ └── mqtt.h └── src │ ├── mqtt.cpp │ └── main.cpp └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant/ 2 | Vagrantfile 3 | build/ 4 | *.mp4 5 | json 6 | -------------------------------------------------------------------------------- /docs/images/arch3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel-iot-devkit/object-size-detector-cpp/HEAD/docs/images/arch3.png -------------------------------------------------------------------------------- /docs/images/object-size-detector.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel-iot-devkit/object-size-detector-cpp/HEAD/docs/images/object-size-detector.png -------------------------------------------------------------------------------- /resources/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "inputs":[ 3 | { 4 | "video":"../resources/bolt-multi-size-detection.mp4" 5 | } 6 | ] 7 | } 8 | 9 | -------------------------------------------------------------------------------- /cmake/pahomqtt.cmake: -------------------------------------------------------------------------------- 1 | # This module installs paho mqtt as an external dependency 2 | # Depends on openSSL 3 | 4 | include(ExternalProject) 5 | find_package(Git REQUIRED) 6 | find_package(PkgConfig REQUIRED) 7 | pkg_search_module(OPENSSL REQUIRED openssl) 8 | 9 | ExternalProject_add( 10 | pahomqtt 11 | GIT_REPOSITORY "https://github.com/eclipse/paho.mqtt.c.git" 12 | GIT_TAG "v1.3.0" 13 | UPDATE_COMMAND "" 14 | PATCH_COMMAND "" 15 | SOURCE_DIR "${CMAKE_BINARY_DIR}/paho-src" 16 | CMAKE_ARGS -DPAHO_WITH_SSL=TRUE -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/paho-build 17 | ) 18 | 19 | include_directories(${CMAKE_BINARY_DIR}/paho-build/include) 20 | link_directories(${CMAKE_BINARY_DIR}/paho-build/lib) 21 | -------------------------------------------------------------------------------- /cmake/FindDemoOpenVINO.cmake: -------------------------------------------------------------------------------- 1 | # Use system variables set by CV SDK 2 | if(NOT DEFINED OpenCV_DIR AND DEFINED ENV{OpenCV_DIR}) 3 | set(OpenCV_DIR "$ENV{OpenCV_DIR}") 4 | endif() 5 | 6 | if(NOT DEFINED OpenCV_DIR) 7 | if(EXISTS "${INTEL_CVSDK_DIR}/opencv/share/OpenCV" ) 8 | set(OpenCV_DIR "${INTEL_CVSDK_DIR}/opencv/share/OpenCV") 9 | elseif(EXISTS "$ENV{INTEL_CVSDK_DIR}/opencv/share/OpenCV") 10 | set(OpenCV_DIR "$ENV{INTEL_CVSDK_DIR}/opencv/share/OpenCV") 11 | elseif(EXISTS "/opt/intel/computer_vision_sdk/opencv/share/OpenCV") 12 | set(OpenCV_DIR "/opt/intel/computer_vision_sdk/opencv/share/OpenCV") 13 | endif() 14 | endif() 15 | 16 | # Avoid system installed OpenCV 17 | find_package(OpenCV PATHS ${OpenCV_DIR} QUIET NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH) 18 | 19 | if(OpenCV_FOUND) 20 | message(STATUS "Intel OpenVINO was found") 21 | message(STATUS "OpenCV_INCLUDE_DIRS=${OpenCV_INCLUDE_DIRS}") 22 | message(STATUS "OpenCV_LIBS=${OpenCV_LIBS}") 23 | endif() -------------------------------------------------------------------------------- /cmake/osdetect.cmake: -------------------------------------------------------------------------------- 1 | # Detects current OS 2 | 3 | find_program(LSB_RELEASE lsb_release) 4 | if(LSB_RELEASE-NOTFOUND) 5 | message(FATAL_ERROR "${Red}Unsupported OS! This demo is for ${REQUIRED_OS_ID} ${REQUIRED_OS_VERSION}.${CR}") 6 | endif() 7 | execute_process(COMMAND ${LSB_RELEASE} -is 8 | OUTPUT_VARIABLE LSB_RELEASE_ID 9 | OUTPUT_STRIP_TRAILING_WHITESPACE 10 | ) 11 | if(NOT LSB_RELEASE_ID STREQUAL ${REQUIRED_OS_ID}) 12 | message(STATUS "OS found: ${LSB_RELEASE_ID}") 13 | message(FATAL_ERROR "${Red}Unsupported OS! This demo is for ${REQUIRED_OS_ID} ${REQUIRED_OS_VERSION}.${CR}") 14 | endif() 15 | execute_process(COMMAND ${LSB_RELEASE} -rs 16 | OUTPUT_VARIABLE LSB_RELEASE_VERSION 17 | OUTPUT_STRIP_TRAILING_WHITESPACE 18 | ) 19 | if(LSB_RELEASE_VERSION VERSION_LESS ${REQUIRED_OS_VERSION}) 20 | message(FATAL_ERROR "${Red}Unsupported OS version! This demo is for ${REQUIRED_OS_ID} ${REQUIRED_OS_VERSION}.${CR}") 21 | elseif(LSB_RELEASE_VERSION VERSION_GREATER ${REQUIRED_OS_VERSION}) 22 | message(WARNING "${Red}This demo has not been tested on ${REQUIRED_OS_ID} ${LSB_RELEASE_VERSION}.${CR}") 23 | endif() 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2021, Intel Corporation 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- 1 | : ' 2 | * Copyright (c) 2018 Intel Corporation. 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining 5 | * a copy of this software and associated documentation files (the 6 | * "Software"), to deal in the Software without restriction, including 7 | * without limitation the rights to use, copy, modify, merge, publish, 8 | * distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so, subject to 10 | * the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be 13 | * included in all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | ' 23 | sudo apt-get update 24 | sudo apt-get install mosquitto mosquitto-clients # install mosquitto 25 | sudo apt-get install pkg-config 26 | 27 | if [ -d = "json" ] 28 | then 29 | sudo rm -r json 30 | fi 31 | 32 | git clone https://github.com/nlohmann/json # Cloning json parser from github 33 | 34 | 35 | cd resources 36 | wget -O bolt-multi-size-detection.mp4 https://github.com/intel-iot-devkit/sample-videos/raw/master/bolt-multi-size-detection.mp4 # Downloading the video required for the RI -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.11) 2 | project(FAC) 3 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake") 4 | 5 | # Fancy colorized output messages 6 | string(ASCII 27 Esc) 7 | set(CR "${Esc}[m") 8 | set(Red "${Esc}[1;31m") 9 | set(Green "${Esc}[1;32m") 10 | set(Blue "${Esc}[1;34m") 11 | 12 | # Allow users to manually set OpenCV path if desired 13 | set(OpenCV_DIR CACHE PATH "Path to Intel OpenCV library") 14 | 15 | message(STATUS "${Blue}Assembly Line Measurements${CR}") 16 | 17 | # Check OS 18 | message(STATUS "${Blue}Checking OS version...${CR}") 19 | 20 | set(REQUIRED_OS_ID "Ubuntu") 21 | set(REQUIRED_OS_VERSION "18.04") 22 | include(osdetect) 23 | 24 | # Check for Intel OpenVINO 25 | message(STATUS "${Blue}Checking prerequisites...${CR}") 26 | 27 | find_package(DemoOpenVINO) 28 | if(NOT OpenCV_FOUND) 29 | message(FATAL_ERROR "${Red}Intel OpenVINO was not found!\n${Blue}Have you sourced the ${Green}setupvars.sh${Blue} script provided by the Intel OpenVINO?${CR}") 30 | endif( ) 31 | 32 | include_directories(${OpenCV_INCLUDE_DIRS}) 33 | include_directories(application/include) 34 | include_directories(json/single_include) 35 | 36 | # Install paho MQTT dependency 37 | include(pahomqtt) 38 | 39 | # Application executables 40 | set(MONITOR monitor) 41 | set(DSOURCES application/src/main.cpp application/src/mqtt.cpp) 42 | add_executable(${MONITOR} ${DSOURCES}) 43 | add_dependencies(${MONITOR} pahomqtt) 44 | set_target_properties(${MONITOR} ${TRAINER} PROPERTIES COMPILE_FLAGS "-pthread -std=c++11") 45 | target_link_libraries (${MONITOR} ${OpenCV_LIBS} pthread paho-mqtt3cs) 46 | 47 | # Install 48 | install(TARGETS ${MONITOR} DESTINATION bin) 49 | -------------------------------------------------------------------------------- /application/include/mqtt.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015 - 2018 Intel Corporation. 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining 5 | * a copy of this software and associated documentation files (the 6 | * "Software"), to deal in the Software without restriction, including 7 | * without limitation the rights to use, copy, modify, merge, publish, 8 | * distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so, subject to 10 | * the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be 13 | * included in all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | */ 23 | 24 | #ifndef MQTT_H_INCLUDED 25 | #define MQTT_H_INCLUDED 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | extern "C" { 36 | #include "MQTTClient.h" 37 | #include "MQTTClientPersistence.h" 38 | } 39 | 40 | #define QOS 1 41 | #define TIMEOUT 1000L 42 | 43 | struct mqtt_service_config 44 | { 45 | std::string server; 46 | std::string client_id; 47 | std::string topic; 48 | std::string username; 49 | std::string password; 50 | std::string cert; 51 | std::string cert_key; 52 | std::string ca_root; 53 | }; 54 | 55 | std::string std_getenv(const std::string &name); 56 | std::pair get_mqtt_config(); 57 | int mqtt_start(MQTTClient_messageArrived* msgrcv); 58 | void mqtt_close(); 59 | void mqtt_connect(); 60 | void mqtt_disconnect(); 61 | int mqtt_publish(std::string const &topic, std::string const &message); 62 | void mqtt_subscribe(std::string const &topic); 63 | 64 | #endif 65 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DISCONTINUATION OF PROJECT # 2 | This project will no longer be maintained by Intel. 3 | Intel has ceased development and contributions including, but not limited to, maintenance, bug fixes, new releases, or updates, to this project. 4 | Intel no longer accepts patches to this project. 5 | # Object Size Detector 6 | 7 | | Details | | 8 | |-----------------------|---------------| 9 | | Target OS: | Ubuntu\* 18.04 LTS | 10 | | Programming Language: | C++\* | 11 | | Time to Complete: | 45 min | 12 | 13 | ![app image](./docs/images/object-size-detector.png) 14 | 15 | ## What It Does 16 | 17 | This application demonstrates how to use CV to detect and measure the approximate size of assembly line parts. It is designed to work with an assembly line camera mounted above the assembly line belt. The application monitors mechanical parts as they are moving down the assembly line and raises an alert if it detects a part on the belt outside a specified size range. 18 | 19 | ## Requirements 20 | 21 | ### Hardware 22 | * 6th to 8th Generation Intel® Core™ processors with Intel® Iris® Pro graphics and Intel® HD Graphics 23 | 24 | ### Software 25 | * [Ubuntu\* 18.04 LTS](http://releases.ubuntu.com/18.04/) 26 | *Note*: Use kernel versions 4.14+ with this software. 27 | 28 | Determine the kernel version with the uname command. In a terminal window type on the command line: 29 | ``` 30 | uname -a 31 | ``` 32 | * Intel® Distribution of OpenVINO™ toolkit 2020 R3 Release 33 | 34 | ## How It Works 35 | 36 | This object size detector works with a video source, such as a camera. The application captures video frames and processes the frame data with OpenCV* algorithms. It detects objects on the assembly line and calculates the area (length x width) the objects occupy. If the calculated area is not within a predefined range, as specified via command line parameters, the application raises an alert to notify the assembly line operator. Optionally, the application sends data to a message queuing telemetry transport (MQTT) machine, or machine messaging server, as part of an assembly line data analytics system. 37 | 38 | ![Code organization](./docs/images/arch3.png) 39 | 40 | The program creates three threads for concurrency: 41 | 42 | - A main thread that performs the video I/O 43 | - A worker thread that processes video frames using the deep neural networks 44 | - A worker thread that publishes MQTT messages 45 | 46 | ## Setup 47 | 48 | ### Get the code 49 | 50 | Clone the reference implementation 51 | ``` 52 | sudo apt-get update && sudo apt-get install git 53 | git clone https://github.com/intel-iot-devkit/object-size-detector-cpp.git 54 | ``` 55 | 56 | ### Install OpenVINO 57 | 58 | Refer to [Install Intel® Distribution of OpenVINO™ toolkit for Linux*](https://software.intel.com/en-us/articles/OpenVINO-Install-Linux) to learn how to install and configure the toolkit. 59 | 60 | ### Other dependencies 61 | 62 | **Mosquitto**
63 | 64 | Mosquitto is an open source message broker that implements the MQTT protocol. The MQTT protocol provides a lightweight method of carrying out messaging using a publish/subscribe model. 65 | 66 | ### Install the dependencies 67 | 68 | To download the sample video and install the dependencies of the application, run the below command in the `object-size-detector-cpp` directory: 69 | ``` 70 | ./setup.sh 71 | ``` 72 | ### The Config File 73 | 74 | The _resources/config.json_ contains the path of videos that will be used by the application as input. Each block represents one video file. 75 | 76 | For example: 77 | ``` 78 | { 79 | "inputs": [ 80 | { 81 | "video":"path_to_video/video1.mp4" 82 | } 83 | ] 84 | } 85 | ``` 86 | 87 | The `path/to/video` is the path to an input video file. 88 | 89 | ### Which Input Video to use 90 | 91 | We recommend using the [bolt-multi-size-detection](https://github.com/intel-iot-devkit/sample-videos/blob/master/bolt-multi-size-detection.mp4) video. For example: 92 | 93 | ``` 94 | { 95 | "inputs":[ 96 | { 97 | "video":"sample-videos/bolt-multi-size-detection.mp4" 98 | } 99 | ] 100 | } 101 | ``` 102 | 103 | ### Using the Camera Stream instead of video 104 | 105 | Replace `path/to/video` with the camera ID in the **config.json** file, where the ID is taken from the video device (the number **X** in /dev/video**X**). 106 | 107 | On Ubuntu, to list all available video devices use the following command: 108 | 109 | ``` 110 | ls /dev/video* 111 | ``` 112 | 113 | For example, if the output of above command is __/dev/video0__, then config.json would be: 114 | 115 | ``` 116 | { 117 | "inputs": [ 118 | { 119 | "video":"0" 120 | } 121 | ] 122 | } 123 | ``` 124 | 125 | 126 | ### Setup the Environment 127 | 128 | Configure the environment to use the Intel® Distribution of OpenVINO™ toolkit by exporting environment variables: 129 | 130 | ``` 131 | source /opt/intel/openvino/bin/setupvars.sh 132 | ``` 133 | 134 | __Note__: This command needs to be executed only once in the terminal where the application will be executed. If the terminal is closed, the command needs to be executed again. 135 | 136 | ### Build the Application 137 | 138 | To build, go to the `object-size-detector-cpp` and run the following commands: 139 | 140 | ``` 141 | mkdir -p build && cd build 142 | cmake .. 143 | make 144 | ``` 145 | 146 | ## Run the Application 147 | 148 | To see a list of the various options: 149 | ``` 150 | ./monitor -help 151 | ``` 152 | 153 | To run the application with the necessary settings, use the min and max parameters: 154 | ``` 155 | ./monitor -min=10000 -max=30000 156 | ``` 157 | 158 | The min and max parameters set the values for the minimum and maximum sizes of the part area. If a part’s calculated area in pixels is not within this range, the application issues an alert. 159 | 160 | ### Machine to Machine Messaging with MQTT 161 | 162 | If you wish to use a MQTT server to publish data, you should set the following environment variables before running the program: 163 | ``` 164 | export MQTT_SERVER=localhost:1883 165 | export MQTT_CLIENT_ID=cvservice 166 | ``` 167 | 168 | Change the `MQTT_SERVER` to a value that matches the MQTT server you are connecting to. 169 | 170 | You should change the `MQTT_CLIENT_ID` to a unique value for each monitoring station, so you can track the data for individual locations. For example: 171 | 172 | ``` 173 | export MQTT_CLIENT_ID=assemblyline1337 174 | ``` 175 | 176 | If you want to monitor the MQTT messages sent to your local server, and you have the `mosquitto` client utilities installed, you can run the following command on a new terminal while the application is running: 177 | ``` 178 | mosquitto_sub -t 'defects/counter' 179 | ``` 180 | -------------------------------------------------------------------------------- /application/src/mqtt.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015 - 2018 Intel Corporation. 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining 5 | * a copy of this software and associated documentation files (the 6 | * "Software"), to deal in the Software without restriction, including 7 | * without limitation the rights to use, copy, modify, merge, publish, 8 | * distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so, subject to 10 | * the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be 13 | * included in all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | */ 23 | 24 | #include "mqtt.h" 25 | 26 | bool mqtt_initialized = false; 27 | MQTTClient client; 28 | MQTTClient_connectOptions conn_opts = MQTTClient_connectOptions_initializer; 29 | MQTTClient_message pubmsg = MQTTClient_message_initializer; 30 | MQTTClient_deliveryToken token; 31 | MQTTClient_SSLOptions sslOptions = MQTTClient_SSLOptions_initializer; 32 | 33 | std::string std_getenv(const std::string &name) 34 | { 35 | auto value = getenv(name.c_str()); 36 | return value != nullptr ? std::string(value) : std::string(); 37 | } 38 | 39 | void mqtt_init(mqtt_service_config const &config) 40 | { 41 | if (mqtt_initialized) 42 | { 43 | return; 44 | } 45 | 46 | std::vector server_c( 47 | config.server.c_str(), 48 | config.server.c_str() + config.server.size() + 1 49 | ); 50 | 51 | std::vector client_id_c( 52 | config.client_id.c_str(), 53 | config.client_id.c_str() + config.client_id.size() + 1 54 | ); 55 | 56 | MQTTClient_create(&client, 57 | &server_c[0], 58 | &client_id_c[0], 59 | MQTTCLIENT_PERSISTENCE_NONE, 60 | NULL); 61 | 62 | // connection options 63 | conn_opts.keepAliveInterval = 20; 64 | conn_opts.cleansession = 1; 65 | 66 | if (!config.username.empty()) 67 | { 68 | std::vector username_c( 69 | config.username.c_str(), 70 | config.username.c_str() + config.username.size() + 1 71 | ); 72 | 73 | conn_opts.username = &username_c[0]; 74 | } 75 | 76 | if (!config.password.empty()) 77 | { 78 | std::vector password_c( 79 | config.password.c_str(), 80 | config.password.c_str() + config.password.size() + 1 81 | ); 82 | 83 | conn_opts.password = &password_c[0]; 84 | } 85 | 86 | // ssl options 87 | if (!config.cert.empty() && !config.cert_key.empty() && !config.ca_root.empty()) 88 | { 89 | std::vector cert_c( 90 | config.cert.c_str(), 91 | config.cert.c_str() + config.cert.size() + 1 92 | ); 93 | 94 | std::vector cert_key_c( 95 | config.cert_key.c_str(), 96 | config.cert_key.c_str() + config.cert_key.size() + 1 97 | ); 98 | 99 | std::vector ca_root_c( 100 | config.ca_root.c_str(), 101 | config.ca_root.c_str() + config.ca_root.size() + 1 102 | ); 103 | 104 | sslOptions.keyStore = &cert_c[0]; 105 | sslOptions.privateKey = &cert_key_c[0]; 106 | sslOptions.trustStore = &ca_root_c[0]; 107 | } 108 | else 109 | { 110 | sslOptions.enableServerCertAuth = false; 111 | }; 112 | conn_opts.ssl = &sslOptions; 113 | 114 | mqtt_initialized = true; 115 | }; 116 | 117 | int mqtt_start(MQTTClient_messageArrived* msgrcv) 118 | { 119 | auto mqtt_config_result = get_mqtt_config(); 120 | 121 | mqtt_service_config mqtt_config; 122 | bool mqtt_config_valid; 123 | 124 | std::tie(mqtt_config, mqtt_config_valid) = mqtt_config_result; 125 | 126 | if (!mqtt_config_valid) 127 | { 128 | return 1; 129 | } 130 | 131 | mqtt_init(mqtt_config); 132 | MQTTClient_setCallbacks(client, NULL, NULL, msgrcv, NULL); 133 | return 0; 134 | } 135 | 136 | void mqtt_close() 137 | { 138 | if (mqtt_initialized) 139 | { 140 | //std::cout << "Closing MQTT..." << std::endl; 141 | MQTTClient_destroy(&client); 142 | } 143 | }; 144 | 145 | void mqtt_connect() 146 | { 147 | if (mqtt_initialized) 148 | { 149 | int rc; 150 | if ((rc = MQTTClient_connect(client, &conn_opts)) != MQTTCLIENT_SUCCESS) 151 | { 152 | //std::cout << "Failed to connect to MQTT server, return code:" << rc << std::endl; 153 | return; 154 | } 155 | } 156 | } 157 | 158 | void mqtt_disconnect() 159 | { 160 | if (mqtt_initialized) 161 | { 162 | MQTTClient_disconnect(client, 10000); 163 | } 164 | } 165 | 166 | int mqtt_publish(std::string const &topic, std::string const &message) 167 | { 168 | if (!mqtt_initialized) { 169 | return -1; 170 | } 171 | 172 | std::vector topic_c( 173 | topic.c_str(), 174 | topic.c_str() + topic.size() + 1 175 | ); 176 | 177 | std::vector message_c( 178 | message.c_str(), 179 | message.c_str() + message.size() + 1 180 | ); 181 | 182 | pubmsg.payload = &message_c[0]; 183 | pubmsg.payloadlen = strlen(&message_c[0]); 184 | pubmsg.qos = QOS; 185 | pubmsg.retained = 0; 186 | int result = MQTTClient_publishMessage(client, &topic_c[0], &pubmsg, &token); 187 | if (result != 0) { // MQTTCLIENT_SUCCESS = 0 188 | return result; 189 | } 190 | return MQTTClient_waitForCompletion(client, token, TIMEOUT); 191 | } 192 | 193 | void mqtt_subscribe(std::string const &topic) 194 | { 195 | if (!mqtt_initialized) { 196 | return; 197 | } 198 | 199 | MQTTClient_subscribe(client, topic.c_str(), 1); 200 | } 201 | 202 | std::pair get_mqtt_config() 203 | { 204 | const auto mqtt_server = std_getenv("MQTT_SERVER"); 205 | const auto mqtt_client_id = std_getenv("MQTT_CLIENT_ID"); 206 | const auto mqtt_username = std_getenv("MQTT_USERNAME"); 207 | const auto mqtt_password = std_getenv("MQTT_PASSWORD"); 208 | const auto mqtt_cert = std_getenv("MQTT_CERT"); 209 | const auto mqtt_cert_key = std_getenv("MQTT_CERT_KEY"); 210 | const auto mqtt_ca_root = std_getenv("MQTT_CA_ROOT"); 211 | 212 | auto config_valid = false; 213 | 214 | mqtt_service_config config = { 215 | mqtt_server, 216 | mqtt_client_id, 217 | mqtt_username, 218 | mqtt_password, 219 | mqtt_cert, 220 | mqtt_cert_key, 221 | mqtt_ca_root}; 222 | 223 | if (!mqtt_server.empty() && !mqtt_client_id.empty()) 224 | { 225 | config_valid = true; 226 | } 227 | 228 | std::pair result = { 229 | config, 230 | config_valid 231 | }; 232 | 233 | return result; 234 | } 235 | -------------------------------------------------------------------------------- /application/src/main.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Intel Corporation. 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining 5 | * a copy of this software and associated documentation files (the 6 | * "Software"), to deal in the Software without restriction, including 7 | * without limitation the rights to use, copy, modify, merge, publish, 8 | * distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so, subject to 10 | * the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be 13 | * included in all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | */ 23 | 24 | // std includes 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | 38 | // OpenCV includes 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | 46 | // MQTT 47 | #include "mqtt.h" 48 | 49 | using namespace std; 50 | using namespace cv; 51 | using namespace dnn; 52 | using json = nlohmann::json; 53 | json jsonobj; 54 | 55 | 56 | // OpenCV-related variables 57 | Mat frame, displayFrame; 58 | VideoCapture cap; 59 | int delay = 5; 60 | int rate; 61 | // nextImage provides queue for captured video frames 62 | queue nextImage; 63 | 64 | // flag to control background threads 65 | atomic keepRunning(true); 66 | 67 | // flag to handle UNIX signals 68 | static volatile sig_atomic_t sig_caught = 0; 69 | 70 | // mqtt parameters 71 | const string topic = "defects/counter"; 72 | 73 | // assembly part and defect areas 74 | int min_area; 75 | int max_area; 76 | int total_parts = 0; 77 | int total_defects = 0; 78 | bool prev_seen = false; 79 | bool prev_defect = false; 80 | int frame_defect_count = 0; 81 | int frame_ok_count = 0; 82 | 83 | // AssemblyInfo contains information about assembly line defects 84 | struct AssemblyInfo 85 | { 86 | bool inc_total; 87 | bool defect; 88 | int area; 89 | bool show; 90 | Rect rect; 91 | }; 92 | 93 | // currentInfo contains the latest AssemblyInfo as tracked by the application 94 | AssemblyInfo currentInfo = {false}; 95 | 96 | mutex m, m1, m2; 97 | 98 | const char* keys = 99 | "{ help h | | Print help message. }" 100 | "{ minarea min | 20000 | Minimum part area of assembly object. }" 101 | "{ maxarea max | 30000 | Maximum part area of assembly object. }" 102 | "{ rate r | 1 | number of seconds between data updates to MQTT server. }"; 103 | 104 | // nextImageAvailable returns the next image from the queue in a thread-safe way 105 | Mat nextImageAvailable() { 106 | Mat rtn; 107 | m.lock(); 108 | if (!nextImage.empty()) { 109 | rtn = nextImage.front(); 110 | nextImage.pop(); 111 | } 112 | m.unlock(); 113 | 114 | return rtn; 115 | } 116 | 117 | // addImage adds an image to the queue in a thread-safe way 118 | void addImage(Mat img) { 119 | m.lock(); 120 | if (nextImage.empty()) { 121 | nextImage.push(img); 122 | } 123 | m.unlock(); 124 | } 125 | 126 | // getCurrentInfo returns the most-recent AssemblyInfo for the application. 127 | AssemblyInfo getCurrentInfo() { 128 | m2.lock(); 129 | AssemblyInfo info; 130 | info = currentInfo; 131 | m2.unlock(); 132 | 133 | return info; 134 | } 135 | 136 | // updateInfo uppdates the current AssemblyInfo for the application to the latest detected values 137 | void updateInfo(AssemblyInfo info) { 138 | m2.lock(); 139 | currentInfo.defect = info.defect; 140 | currentInfo.show = info.show; 141 | currentInfo.area = info.area; 142 | currentInfo.rect = info.rect; 143 | if (info.inc_total) { 144 | total_parts++; 145 | } 146 | if (info.defect) { 147 | total_defects++; 148 | } 149 | m2.unlock(); 150 | } 151 | 152 | // resetInfo resets the current AssemblyInfo for the application. 153 | void resetInfo() { 154 | m2.lock(); 155 | currentInfo.defect = false; 156 | currentInfo.area = 0; 157 | currentInfo.inc_total = false; 158 | currentInfo.rect = Rect(0,0,0,0); 159 | m2.unlock(); 160 | } 161 | 162 | // publish MQTT message with a JSON payload 163 | void publishMQTTMessage(const string& topic, const AssemblyInfo& info) 164 | { 165 | ostringstream s; 166 | s << "{\"Defect\": \"" << info.defect << "\"}"; 167 | string payload = s.str(); 168 | 169 | mqtt_publish(topic, payload); 170 | 171 | string msg = "MQTT message published to topic: " + topic; 172 | syslog(LOG_INFO, "%s", msg.c_str()); 173 | syslog(LOG_INFO, "%s", payload.c_str()); 174 | } 175 | 176 | // message handler for the MQTT subscription for the any desired control channel topic 177 | int handleMQTTControlMessages(void *context, char *topicName, int topicLen, MQTTClient_message *message) 178 | { 179 | string topic = topicName; 180 | string msg = "MQTT message received: " + topic; 181 | syslog(LOG_INFO, "%s", msg.c_str()); 182 | 183 | return 1; 184 | } 185 | 186 | // Function called by worker thread to process the next available video frame. 187 | void frameRunner() { 188 | while (keepRunning.load()) { 189 | Mat next = nextImageAvailable(); 190 | if (!next.empty()) { 191 | Mat img; 192 | Rect max_rect; 193 | int max_blob_area = 0; 194 | int part_area = 0; 195 | bool defect = false; 196 | bool frame_defect = false; 197 | bool inc_total = false; 198 | Size size(3,3); 199 | vector hierarchy; 200 | vector > contours; 201 | 202 | cvtColor(frame, img, COLOR_RGB2GRAY); 203 | // Blur the image to smooth it before easier preprocessing 204 | GaussianBlur(img, img, size, 0, 0 ); 205 | 206 | // Morphology: OPEN -> CLOSE -> OPEN 207 | // MORPH_OPEN removes the noise and closes the "holes" in the background 208 | // MORPH_CLOSE remove the noise and closes the "holes" in the foreground 209 | morphologyEx(img, img, MORPH_OPEN, getStructuringElement(MORPH_ELLIPSE, size)); 210 | morphologyEx(img, img, MORPH_CLOSE, getStructuringElement(MORPH_ELLIPSE, size)); 211 | morphologyEx(img, img, MORPH_OPEN, getStructuringElement(MORPH_ELLIPSE, size)); 212 | 213 | // threshold the image to emphasize assembly part 214 | threshold(img, img, 200, 255, THRESH_BINARY); 215 | // find the contours of assembly part 216 | findContours(img, contours, hierarchy, RETR_EXTERNAL, CHAIN_APPROX_NONE); 217 | 218 | // we will pick detected objects with largest size 219 | for (size_t i = 0; i < contours.size(); i++) 220 | { 221 | Rect rect = boundingRect(contours[i]); 222 | part_area = rect.width * rect.height; 223 | // is large enough, and completely within the camera with no overlapping edge. 224 | if (part_area > max_blob_area && rect.x > 0 && rect.x + rect.width < img.cols && rect.width > 30) 225 | { 226 | max_blob_area = part_area; 227 | max_rect = rect; 228 | } 229 | } 230 | part_area = max_blob_area; 231 | 232 | // if no object is detected we dont do anything 233 | if (part_area != 0) { 234 | // increment ok or defect counts 235 | if (part_area > max_area || part_area < min_area) 236 | { 237 | frame_defect = true; 238 | frame_defect_count++; 239 | } else { 240 | frame_ok_count++; 241 | } 242 | 243 | // if the part wasn't seen before it's a new part 244 | if (!prev_seen) { 245 | prev_seen = true; 246 | inc_total = true; 247 | } else { 248 | // if the previously seen object has no defect detected in 10 previous consecutive frames 249 | if (!frame_defect && frame_ok_count > 10) { 250 | frame_defect_count = 0; 251 | } 252 | // if previously seen object has a defect detected in 10 previous consecutive frames 253 | if (frame_defect && frame_defect_count > 10) { 254 | if (!prev_defect) { 255 | prev_defect = true; 256 | defect = true; 257 | } 258 | frame_ok_count = 0; 259 | } 260 | } 261 | } else { 262 | // no part detected -- we are looking at empty belt. reset values. 263 | prev_seen = false; 264 | prev_defect = false; 265 | frame_defect_count = 0; 266 | frame_ok_count = 0; 267 | } 268 | 269 | AssemblyInfo info; 270 | info.defect = defect; 271 | info.show = prev_defect; 272 | info.area = part_area; 273 | info.rect = max_rect; 274 | info.inc_total = inc_total; 275 | 276 | updateInfo(info); 277 | } 278 | } 279 | 280 | cout << "Video processing thread stopped" << endl; 281 | } 282 | 283 | // Function called by worker thread to handle MQTT updates. Pauses for rate second(s) between updates. 284 | void messageRunner() { 285 | while (keepRunning.load()) { 286 | AssemblyInfo info = getCurrentInfo(); 287 | publishMQTTMessage(topic, info); 288 | this_thread::sleep_for(chrono::seconds(rate)); 289 | } 290 | 291 | cout << "MQTT sender thread stopped" << endl; 292 | } 293 | 294 | // signal handler for the main thread 295 | void handle_sigterm(int signum) 296 | { 297 | /* we only handle SIGTERM and SIGKILL here */ 298 | if (signum == SIGTERM) { 299 | cout << "Interrupt signal (" << signum << ") received" << endl; 300 | sig_caught = 1; 301 | } 302 | } 303 | 304 | int main(int argc, char** argv) 305 | { 306 | // parse command parameters 307 | CommandLineParser parser(argc, argv, keys); 308 | String input; 309 | std::string conf_file = "../resources/config.json"; 310 | std::ifstream confFile(conf_file); 311 | confFile>>jsonobj; 312 | 313 | parser.about("Use this script to using OpenVINO."); 314 | if (argc == 1 || parser.has("help")) 315 | { 316 | parser.printMessage(); 317 | 318 | return 0; 319 | } 320 | 321 | min_area = parser.get("minarea"); 322 | max_area = parser.get("maxarea"); 323 | rate = parser.get("rate"); 324 | 325 | auto obj = jsonobj["inputs"]; 326 | input = obj[0]["video"]; 327 | 328 | if (input.size() == 1 && *(input.c_str()) >= '0' && *(input.c_str()) <= '9') 329 | cap.open(std::stoi(input)); 330 | else 331 | cap.open(input); 332 | 333 | if (!cap.isOpened()) 334 | { 335 | cerr << "ERROR! Unable to open video source\n"; 336 | return -1; 337 | } 338 | 339 | // Also adjust delay so video playback matches the number of FPS in the file 340 | double fps = cap.get(CAP_PROP_FPS); 341 | delay = 1000 / fps; 342 | 343 | // connect MQTT messaging 344 | int result = mqtt_start(handleMQTTControlMessages); 345 | if (result == 0) { 346 | syslog(LOG_INFO, "MQTT started."); 347 | } else { 348 | syslog(LOG_INFO, "MQTT NOT started: have you set the ENV varables?"); 349 | } 350 | 351 | mqtt_connect(); 352 | 353 | // register SIGTERM signal handler 354 | signal(SIGTERM, handle_sigterm); 355 | 356 | // start worker threads 357 | thread t1(frameRunner); 358 | thread t2(messageRunner); 359 | 360 | string label; 361 | // read video input data 362 | for (;;) { 363 | cap.read(frame); 364 | 365 | if (frame.empty()) { 366 | keepRunning = false; 367 | cerr << "ERROR! blank frame grabbed\n"; 368 | break; 369 | } 370 | 371 | resize(frame, frame, Size(960, 540)); 372 | displayFrame = frame.clone(); 373 | addImage(frame); 374 | 375 | AssemblyInfo info = getCurrentInfo(); 376 | label = format("Measurement: %d Expected range: [%d - %d] Defect: %s", 377 | info.area, min_area, max_area, info.defect? "TRUE" : "FALSE"); 378 | putText(displayFrame, label, Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); 379 | 380 | label = format("Total parts: %d Total Defects: %d", total_parts, total_defects); 381 | putText(displayFrame, label, Point(0, 40), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); 382 | 383 | if (info.show) { 384 | rectangle(displayFrame, info.rect, Scalar(255, 0, 0), 1); 385 | } else { 386 | rectangle(displayFrame, info.rect, Scalar(0, 255, 0), 1); 387 | } 388 | 389 | imshow("Object Size Detector", displayFrame); 390 | 391 | if (waitKey(delay) >= 0 || sig_caught) { 392 | cout << "Attempting to stop background threads" << endl; 393 | keepRunning = false; 394 | break; 395 | } 396 | } 397 | 398 | // wait for the threads to finish 399 | t1.join(); 400 | t2.join(); 401 | cap.release(); 402 | 403 | // disconnect MQTT messaging 404 | mqtt_disconnect(); 405 | mqtt_close(); 406 | 407 | return 0; 408 | } 409 | --------------------------------------------------------------------------------