├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── Common ├── AsyncRecurringTask.hxx ├── Constants.cxx ├── Constants.hxx ├── Logger.cxx ├── Logger.hxx └── Utils.hxx ├── LICENSE ├── README.md ├── Transformations ├── kafkaBoolTrans.cxx ├── kafkaBoolTrans.hxx ├── kafkaFloatTrans.cxx ├── kafkaFloatTrans.hxx ├── kafkaInt32Trans.cxx ├── kafkaInt32Trans.hxx ├── kafkaInt64Trans.cxx ├── kafkaInt64Trans.hxx ├── kafkaStringTrans.cxx ├── kafkaStringTrans.hxx ├── kafkaTimeTrans.cxx ├── kafkaTimeTrans.hxx ├── kafkaUint8Trans.cxx └── kafkaUint8Trans.hxx ├── config.h.in ├── doc ├── REMUS_RealTime_Evolution_-_KAFKA.pptx ├── kafkaActivity.svg ├── kafkaActivity.uml └── kafkaDemoProjectDPEs.gif ├── kafkaConsumerFacade.cxx ├── kafkaConsumerFacade.hxx ├── kafkaDrv.cxx ├── kafkaDrv.hxx ├── kafkaHWMapper.cxx ├── kafkaHWMapper.hxx ├── kafkaHWService.cxx ├── kafkaHWService.hxx ├── kafkaMain.cxx ├── kafkaProducerFacade.cxx ├── kafkaProducerFacade.hxx ├── kafkaResources.cxx ├── kafkaResources.hxx ├── winccoa ├── config │ ├── config.kafka │ └── user.keytab ├── dplist │ └── kafka_driver_config.dpl ├── panels │ └── para │ │ └── address_kafka.pnl └── scripts │ ├── libs │ └── kafka_dpe_addressing.ctl │ ├── userDrivers.ctl │ └── userPara.ctl └── winccoa316_demo_project └── kafka_producer_consumer_demo.zip /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | *.o 3 | *.gcno 4 | *.gcda 5 | **/*.log 6 | build 7 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cern-hse-computing/WCCOAkafkaDrv/a37d333d8148a4df5ac4c33d01de0605b6ca5e8a/.gitmodules -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.17) 2 | 3 | project( 4 | WCCOAkafkaDrv 5 | DESCRIPTION "KAFKA driver for WinCC OA" 6 | LANGUAGES CXX 7 | ) 8 | 9 | set(PROJECT_VERSION 1.1.0) 10 | 11 | configure_file(config.h.in configured/config.h) 12 | include_directories(${CMAKE_CURRENT_BINARY_DIR}/configured) 13 | set(DRV_VERSION ${PROJECT_VERSION}) 14 | 15 | set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) 16 | set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) 17 | 18 | set(CMAKE_CXX_STANDARD 11) 19 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 20 | 21 | if(NOT CMAKE_BUILD_TYPE) 22 | set(CMAKE_BUILD_TYPE Release) 23 | else() 24 | # if Coverage is selected, look for required tools 25 | if(CMAKE_BUILD_TYPE STREQUAL "Coverage") 26 | find_program(LCOV_PATH lcov) 27 | if(NOT LCOV_PATH) 28 | message(FATAL_ERROR "lcov not found! Cannot build coverage.") 29 | endif() 30 | endif() 31 | # Code coverage report target 32 | add_custom_target(coverage 33 | COMMAND lcov --capture --directory . --output-file coverage.info 34 | COMMAND lcov --remove coverage.info '/usr/*' '*/test/*' '*/CMakeFiles/*' --output-file coverage.info.cleaned 35 | COMMAND genhtml coverage.info.cleaned --output-directory coverage 36 | COMMAND xdg-open coverage/index.html 37 | WORKING_DIRECTORY ${CMAKE_BINARY_DIR} 38 | COMMENT "Generating code coverage report..." 39 | USES_TERMINAL 40 | ) 41 | endif() 42 | 43 | set(CMAKE_CXX_FLAGS "-rdynamic") 44 | set(CMAKE_CXX_FLAGS_DEBUG "-O0 -ggdb") 45 | set(CMAKE_CXX_FLAGS_COVERAGE "-O0 -g --coverage -fPIC") 46 | set(CMAKE_CXX_FLAGS_RELEASE "-O3") 47 | 48 | 49 | # Define target 50 | set(TARGET ${PROJECT_NAME}) 51 | 52 | # Include WinCC_OA API 53 | set(API_ROOT "$ENV{API_ROOT}" CACHE FILEPATH "directory of the WinCC_OA API installation") 54 | include(${API_ROOT}/CMakeDefines.txt) 55 | 56 | 57 | file(GLOB KAFKA_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/*.cxx)# Collect sources 58 | file(GLOB KAFKA_COMMON ${CMAKE_CURRENT_SOURCE_DIR}/Common/*.cxx) 59 | file(GLOB KAFKA_TRANSFORMATIONS ${CMAKE_CURRENT_SOURCE_DIR}/Transformations/*.cxx) 60 | set(SOURCES ${KAFKA_SOURCES} ${KAFKA_COMMON} ${KAFKA_TRANSFORMATIONS}) 61 | 62 | # Add driver 63 | add_driver(${TARGET} ${SOURCES}) 64 | 65 | # Link pthread 66 | find_package( Threads ) 67 | target_link_libraries( ${TARGET} ${CMAKE_THREAD_LIBS_INIT} ) 68 | 69 | # ExternalProject_Add librdkafka and cppkafka 70 | include(ExternalProject) 71 | ExternalProject_Add(librdkafka-external 72 | GIT_REPOSITORY git@github.com:confluentinc/librdkafka.git 73 | GIT_TAG v1.5.0 74 | GIT_SHALLOW ON 75 | GIT_PROGRESS ON 76 | SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/librdkafka 77 | CONFIGURE_COMMAND ${CMAKE_CURRENT_BINARY_DIR}/librdkafka/configure --source-deps-only --enable-sasl --enable-gssapi --prefix=${CMAKE_CURRENT_BINARY_DIR} 78 | BUILD_COMMAND make 79 | INSTALL_COMMAND make install 80 | UPDATE_COMMAND "" 81 | BUILD_IN_SOURCE 1 82 | ) 83 | 84 | ExternalProject_Add(cppkafka-external 85 | GIT_REPOSITORY git@github.com:mfontanini/cppkafka.git 86 | GIT_TAG v0.4.1 87 | GIT_PROGRESS ON 88 | GIT_SUBMODULES "" 89 | SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cppkafka 90 | CMAKE_ARGS 91 | -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR} 92 | -DCPPKAFKA_DISABLE_TESTS=ON 93 | -DCMAKE_INSTALL_LIBDIR=lib 94 | BUILD_IN_SOURCE 1 95 | UPDATE_COMMAND "" 96 | DEPENDS librdkafka-external 97 | ) 98 | include_directories(${CMAKE_CURRENT_BINARY_DIR}/include GLOBAL) 99 | 100 | add_library(librdkafka SHARED IMPORTED GLOBAL) 101 | add_library(libcppkafka SHARED IMPORTED GLOBAL) 102 | add_dependencies(librdkafka librdkafka-external) 103 | add_dependencies(libcppkafka cppkafka-external) 104 | set_target_properties(librdkafka PROPERTIES IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/lib/librdkafka.so) 105 | set_target_properties(libcppkafka PROPERTIES IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/lib/libcppkafka.so) 106 | 107 | target_link_libraries(${TARGET} libcppkafka librdkafka) 108 | set_target_properties(${TARGET} PROPERTIES INSTALL_RPATH "$ORIGIN/lib") 109 | 110 | # PVSS_PROJ_PATH Install 111 | # Check if PVSS_PROJ_PATH is set 112 | if(NOT DEFINED ENV{PVSS_PROJ_PATH}) 113 | message(WARNING "PVSS_PROJ_PATH environment variable is not set. Commodity targets will not be available (install, run, valgrind).") 114 | else() 115 | # Install driver to PVSS_PROJ_PATH/bin 116 | install(TARGETS ${TARGET} DESTINATION $ENV{PVSS_PROJ_PATH}/bin) 117 | 118 | # install lib folder 119 | install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib DESTINATION $ENV{PVSS_PROJ_PATH}/bin) 120 | 121 | # PVSS_PROJ_PATH contains the project name , so we can extract it 122 | string(REGEX MATCH "([^/]+)/?$" PVSS_PROJECT_NAME $ENV{PVSS_PROJ_PATH}) 123 | string(REGEX REPLACE "/$" "" PVSS_PROJECT_NAME ${PVSS_PROJECT_NAME}) 124 | message(STATUS "PVSS project name extracted from $PVSS_PROJ_PATH ($ENV{PVSS_PROJ_PATH}): ${PVSS_PROJECT_NAME}") 125 | 126 | # Try to parse config/progs file to extract driver number and its config file 127 | # i.e. WCCOAkafkaDrv | always | 30 | 3 | 1 |-num 32 +config config.kafka 128 | if(NOT EXISTS $ENV{PVSS_PROJ_PATH}/config/progs) 129 | message(WARNING "Cannot find $PVSS_PROJ_PATH/config/progs. Setting driver number to 999.") 130 | set(DRIVER_NUMBER 999) 131 | else() 132 | # Extract driver number and config file from config/progs 133 | if(EXISTS $ENV{PVSS_PROJ_PATH}/config/progs) 134 | file(STRINGS "$ENV{PVSS_PROJ_PATH}/config/progs" PROGS) 135 | foreach(PROG ${PROGS}) 136 | string(REGEX MATCHALL "${TARGET}.*-num ([0-9]+) \\+config (config\\..*)" DRIVER_MATCH ${PROG}) 137 | if(DRIVER_MATCH) 138 | set(DRIVER_NUMBER ${CMAKE_MATCH_1}) 139 | set(DRIVER_CONFIG_FILE ${CMAKE_MATCH_2}) 140 | message(STATUS "Driver number extracted from $ENV{PVSS_PROJ_PATH}/config/progs: ${DRIVER_NUMBER}") 141 | message(STATUS "Driver config file extracted from $ENV{PVSS_PROJ_PATH}/config/progs: ${DRIVER_CONFIG_FILE}") 142 | break() 143 | endif() 144 | endforeach() 145 | endif() 146 | if(NOT DEFINED DRIVER_NUMBER) 147 | message(WARNING "Driver number could not be extracted from $PVSS_PROJ_PATH/config/progs. Setting it to 999.") 148 | set(DRIVER_NUMBER 999) 149 | endif() 150 | endif() 151 | 152 | # Run local build target (useful for debugging, coverage, valgrind, etc.) 153 | set(RUNBUILD_ARGS "-num ${DRIVER_NUMBER} -proj ${PVSS_PROJECT_NAME} +config $ENV{PVSS_PROJ_PATH}/config/${DRIVER_CONFIG_FILE} &") 154 | separate_arguments(RUNBUILD_ARGS) 155 | add_custom_target(run 156 | COMMAND ${TARGET} ${RUNBUILD_ARGS} 157 | WORKING_DIRECTORY ${CMAKE_BINARY_DIR} 158 | COMMENT "Launching: ${TARGET} ${RUNBUILD_ARGS}" 159 | USES_TERMINAL 160 | COMMAND_EXPAND_LISTS 161 | ) 162 | add_dependencies(run ${TARGET}) 163 | 164 | # valgrind target 165 | find_program(VALGRIND_PATH valgrind) 166 | if(VALGRIND_PATH) 167 | set(VALGRIND_CMD ${VALGRIND_PATH} "--leak-check=full" "--show-leak-kinds=all" "--track-origins=yes" "--verbose" "--log-file=valgrind.log" "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}" ${RUNBUILD_ARGS}) 168 | add_custom_target(valgrind 169 | COMMAND ${VALGRIND_CMD} 170 | WORKING_DIRECTORY ${CMAKE_BINARY_DIR} 171 | COMMENT "Launching: ${VALGRIND_CMD} \n\t Log file: ${CMAKE_BINARY_DIR}/valgrind.log. \n\t Use `kill` target to stop the process." 172 | USES_TERMINAL 173 | COMMAND_EXPAND_LISTS 174 | ) 175 | add_dependencies(valgrind ${TARGET}) 176 | endif() 177 | 178 | endif() 179 | 180 | # Send SIGTERM to the driver or valgrind process 181 | add_custom_target(kill 182 | COMMAND pkill -SIGTERM -u "$ENV{USER}" -f "${TARGET}" -e 183 | COMMENT "Sending SIGTERM for ${TARGET}. WinCC OA will restart it automatically if configured to `always`." 184 | USES_TERMINAL 185 | ) 186 | 187 | # Clean and update driver 188 | add_custom_target(update 189 | COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target clean 190 | COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target install 191 | COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target kill 192 | ) 193 | 194 | 195 | 196 | # Config summary 197 | message(STATUS "") 198 | message(STATUS "---------------+-----------------------------------------------------------------------------------------------") 199 | message(STATUS "Configured ${TARGET} ${DRV_VERSION}") 200 | message(STATUS "---------------+-----------------------------------------------------------------------------------------------") 201 | message(STATUS " Target | Description") 202 | message(STATUS "---------------+-----------------------------------------------------------------------------------------------") 203 | message(STATUS " kill | Kills ${TARGET}") 204 | message(STATUS " update | Calls following targets: clean -> install -> kill") 205 | if(DEFINED ENV{PVSS_PROJ_PATH}) 206 | message(STATUS " install | Will install ${TARGET} ${PROJECT_VERSION} in: $ENV{PVSS_PROJ_PATH}/bin") 207 | message(STATUS " | with librdkafka and cppkafka libs in $ENV{PVSS_PROJ_PATH}/bin/lib") 208 | message(STATUS " run | Runs ${TARGET} from ${CMAKE_BINARY_DIR} with:") 209 | message(STATUS " | ${TARGET} -num -proj +config ") 210 | message(STATUS " | pvss_project_name = ${PVSS_PROJECT_NAME}") 211 | message(STATUS " | driver_number = ${DRIVER_NUMBER}") 212 | message(STATUS " | driver_config_file = $ENV{PVSS_PROJ_PATH}/config/${DRIVER_CONFIG_FILE}") 213 | else() 214 | message(STATUS " install | PVSS_PROJ_PATH environment variable was not set. Cannot install from CMake.") 215 | message(STATUS " run | PVSS_PROJ_PATH environment variable was not set. Cannot run local build from CMake.") 216 | endif() 217 | if(VALGRIND_PATH) 218 | message(STATUS " valgrind | Runs ${TARGET} from ${CMAKE_BINARY_DIR} with valgrind.") 219 | message(STATUS " | (similar to run target)") 220 | else() 221 | message(STATUS " valgrind | Not available. valgrind or PVSS_PROJ_PATH not found.") 222 | endif() 223 | if(CMAKE_BUILD_TYPE STREQUAL "Coverage") 224 | message(STATUS " coverage | Generates code coverage report (requires lcov)") 225 | else() 226 | message(STATUS " coverage | Not available. Build type is not Coverage -> -DCMAKE_BUILD_TYPE=Coverage. lcov required.") 227 | endif() 228 | message(STATUS "---------------+-----------------------------------------------------------------------------------------------") 229 | -------------------------------------------------------------------------------- /Common/AsyncRecurringTask.hxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #ifndef AsyncRecurringTask_HXX 16 | #define AsyncRecurringTask_HXX 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | 26 | /** 27 | * @brief An asynchrounous recurring task wrapper 28 | */ 29 | 30 | 31 | namespace Common 32 | { 33 | template 34 | class AsyncRecurringTask 35 | { 36 | public: 37 | AsyncRecurringTask(Task&& action, const uint32_t& timeoutMs) 38 | : _action(std::move(action)), _timeoutMs(timeoutMs) 39 | { 40 | _future = std::async(std::launch::async, [this]() 41 | { 42 | while(!this->_done) 43 | { 44 | this->_action(); 45 | this->sleep(); 46 | } 47 | }); 48 | } 49 | 50 | AsyncRecurringTask(const AsyncRecurringTask& ) = delete; 51 | AsyncRecurringTask& operator=(const AsyncRecurringTask& ) = delete; 52 | 53 | ~AsyncRecurringTask() 54 | { 55 | // This will break the sleep cycle if the task is destroyed 56 | _done = true; 57 | _cv.notify_one(); 58 | } 59 | 60 | 61 | private: 62 | void sleep() 63 | { 64 | std::unique_lock lock(_mutex); 65 | _cv.wait_for(lock, std::chrono::milliseconds(_timeoutMs)); 66 | } 67 | 68 | Task _action; 69 | std::atomic _done{false}; 70 | std::future _future; 71 | std::condition_variable _cv; 72 | std::mutex _mutex; 73 | const uint32_t& _timeoutMs; 74 | }; 75 | } 76 | #endif // AsyncRecurringTask_HXX 77 | -------------------------------------------------------------------------------- /Common/Constants.cxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #include "Constants.hxx" 16 | #include "Logger.hxx" 17 | #include "Utils.hxx" 18 | 19 | namespace Common { 20 | 21 | // Common 22 | std::string Constants::drv_name = "kafka"; 23 | uint32_t Constants::DRV_NO = 0; // Read from PVSS on driver startup 24 | 25 | // Producer 26 | uint32_t Constants::PRODUCER_KAFKA_STATS_INTERVAL = 60; // Read from PVSS on driver startup 27 | uint32_t Constants::DEBOUNCING_THREAD_INTERVAL = 50; // Read from PVSS on driver startup 28 | 29 | // Consumer 30 | size_t Constants::CONSUMER_MAX_POLL_RECORDS = 1000; // Read from PVSS on driver startup 31 | uint32_t Constants::CONSUMER_KAFKA_STATS_INTERVAL = 60; // Read from PVSS on driver startup 32 | 33 | 34 | bool Constants::slave_mode = false; // Slave mode will accept all packages and speak only when spoken to 35 | 36 | std::map Constants::producer_config; 37 | std::map Constants::consumer_config; 38 | 39 | // The map can be used to map a callback to a HwObject address 40 | std::map> Constants::parse_map = 41 | { 42 | { "DEBUGLVL", 43 | [](const char* data) 44 | { 45 | Common::Logger::globalInfo(Common::Logger::L1, "setLogLvl:", CharString(data)); 46 | Common::Logger::setLogLvl((int)std::atoi(data)); 47 | } 48 | }, 49 | { "DEBOUNCINGTHREADINTERVAL", 50 | [](const char* data) 51 | { 52 | Common::Logger::globalInfo(Common::Logger::L1, "setDebouncingThreadInterval:", CharString(data)); 53 | Common::Constants::setDebouncingThreadInterval((int)std::atoi(data)); 54 | } 55 | }, 56 | { "MAXPOLLRECORDS", 57 | [](const char* data) 58 | { 59 | Common::Logger::globalInfo(Common::Logger::L1, "setConsumerMaxPollRecords:", CharString(data)); 60 | Common::Constants::setConsumerMaxPollRecords((size_t)std::atoi(data)); 61 | } 62 | } 63 | 64 | }; 65 | } 66 | -------------------------------------------------------------------------------- /Common/Constants.hxx: -------------------------------------------------------------------------------- 1 | /*! 2 | * \file Constants.hxx 3 | * \author Kacper "szkudi" Szkudlarek 4 | * \brief File contain definitions of some constants used in driver 5 | * 6 | * Changelog: 7 | * 8 | */ 9 | 10 | 11 | #ifndef CONSTANTS_HXX_ 12 | #define CONSTANTS_HXX_ 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | /** © Copyright 2019 CERN 24 | * 25 | * This software is distributed under the terms of the 26 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 27 | * copied verbatim in the file “LICENSE” 28 | * 29 | * In applying this licence, CERN does not waive the privileges 30 | * and immunities granted to it by virtue of its status as an 31 | * Intergovernmental Organization or submit itself to any jurisdiction. 32 | * 33 | * Author: Alexandru Savulescu (HSE-CEN-CO) 34 | * 35 | **/ 36 | 37 | #include 38 | #include 39 | #include 40 | 41 | namespace Common{ 42 | 43 | /*! 44 | * \class Constants 45 | * \brief Class containing constant values used in driver 46 | */ 47 | class Constants{ 48 | public: 49 | 50 | //producer 51 | constexpr static const char* PRODUCER_CONFIG_KEYWORD {"PRODUCER.CONFIG."}; 52 | constexpr static const char* CONFIG_STATISTICS_INTERVAL_MS {"statistics.interval.ms"}; 53 | constexpr static const int CONFIG_STATISTICS_INTERVAL_MS_DEFAULT = 60 * 1000; // 60 seconds 54 | 55 | //consumer 56 | constexpr static const char* CONSUMER_CONFIG_KEYWORD {"CONSUMER.CONFIG."}; 57 | 58 | 59 | constexpr static const char* GROUP_ID_KEYWORD {"group.id"}; 60 | 61 | 62 | static void setLogpath(std::string lp); 63 | 64 | static std::string& getLogpath(); 65 | 66 | static void setDrvName(std::string lp); 67 | 68 | static std::string& getDrvName(); 69 | 70 | // called in driver init to set the driver number dynamically 71 | static void setDrvNo(uint32_t no); 72 | // subsequentally called when writing buffers etc. 73 | static uint32_t getDrvNo(); 74 | 75 | 76 | static void setDebouncingThreadInterval(uint32_t no); 77 | static const uint32_t& getDebouncingThreadInterval(); 78 | 79 | static void setProducerKafkaStatsInterval(); 80 | static const uint32_t& getProducerKafkaStatsInterval(); 81 | 82 | static void setConsumerKafkaStatsInterval(); 83 | static const uint32_t& getConsumerKafkaStatsInterval(); 84 | 85 | static void setConsumerMaxPollRecords(size_t max); 86 | static const size_t& getConsumerMaxPollRecords(); 87 | 88 | // slave mode functions (listens but does not speak) 89 | static bool isSlaveMode() { 90 | return slave_mode; 91 | } 92 | 93 | static void setSlaveMode(bool slaveMode) { 94 | slave_mode = slaveMode; 95 | } 96 | 97 | static const std::map& GetProducerConfig(); 98 | static const std::map>& GetParseMap(); 99 | static void setProducerConfig(std::string& key, std::string& value); 100 | static bool hasProducerConfig(); 101 | 102 | static const std::map& GetConsumerConfig(); 103 | static void setConsumerConfig(std::string& key, std::string& value); 104 | static bool hasConsumerConfig(); 105 | 106 | private: 107 | static std::string drv_name; 108 | 109 | static uint32_t DRV_NO; // WinCC OA manager number 110 | static uint32_t DEBOUNCING_THREAD_INTERVAL; 111 | static uint32_t PRODUCER_KAFKA_STATS_INTERVAL; 112 | static uint32_t CONSUMER_KAFKA_STATS_INTERVAL; 113 | static size_t CONSUMER_MAX_POLL_RECORDS; 114 | 115 | static bool slave_mode; 116 | static std::map producer_config; 117 | static std::map consumer_config; 118 | 119 | static std::map> parse_map; 120 | }; 121 | 122 | inline const std::map& Constants::GetProducerConfig() 123 | { 124 | return producer_config; 125 | } 126 | 127 | inline const std::map& Constants::GetConsumerConfig() 128 | { 129 | return consumer_config; 130 | } 131 | 132 | 133 | 134 | inline const std::map>& Constants::GetParseMap() 135 | { 136 | return parse_map; 137 | } 138 | 139 | 140 | 141 | inline void Constants::setProducerConfig(std::string& key, std::string& value) 142 | { 143 | producer_config[key] = value; 144 | } 145 | 146 | 147 | inline void Constants::setConsumerConfig(std::string& key, std::string& value) 148 | { 149 | consumer_config[key] = value; 150 | } 151 | 152 | inline bool Constants::hasProducerConfig() 153 | { 154 | return producer_config.size() > 0; 155 | } 156 | 157 | 158 | inline bool Constants::hasConsumerConfig() 159 | { 160 | return consumer_config.size() > 0; 161 | } 162 | 163 | 164 | inline void Constants::setDrvName(std::string dname){ 165 | drv_name = dname; 166 | } 167 | 168 | inline std::string& Constants::getDrvName(){ 169 | return drv_name; 170 | } 171 | 172 | inline void Constants::setDrvNo(uint32_t no){ 173 | DRV_NO = no; 174 | } 175 | 176 | inline uint32_t Constants::getDrvNo(){ 177 | return DRV_NO; 178 | } 179 | 180 | inline void Constants::setDebouncingThreadInterval(uint32_t no){ 181 | DEBOUNCING_THREAD_INTERVAL = no; 182 | } 183 | 184 | inline const uint32_t& Constants::getDebouncingThreadInterval(){ 185 | return DEBOUNCING_THREAD_INTERVAL; 186 | } 187 | 188 | inline void Constants::setConsumerMaxPollRecords(size_t max) 189 | { 190 | CONSUMER_MAX_POLL_RECORDS = max; 191 | } 192 | 193 | inline const size_t& Constants::getConsumerMaxPollRecords() 194 | { 195 | return CONSUMER_MAX_POLL_RECORDS; 196 | } 197 | 198 | inline void Constants::setProducerKafkaStatsInterval(){ 199 | int milis; 200 | try 201 | { 202 | if(!Common::Utils::convertToInt(Common::Constants::GetProducerConfig().at(Common::Constants::CONFIG_STATISTICS_INTERVAL_MS), milis)) 203 | { 204 | milis = Common::Constants::CONFIG_STATISTICS_INTERVAL_MS_DEFAULT; // DEFAULT 205 | } 206 | } 207 | catch(std::exception& e) 208 | { 209 | Common::Logger::globalWarning(__PRETTY_FUNCTION__," Constants ProducerConfig does not have ", Common::Constants::CONFIG_STATISTICS_INTERVAL_MS); 210 | milis = Common::Constants::CONFIG_STATISTICS_INTERVAL_MS_DEFAULT; // DEFAULT 211 | } 212 | PRODUCER_KAFKA_STATS_INTERVAL = milis; 213 | } 214 | 215 | inline const uint32_t& Constants::getProducerKafkaStatsInterval(){ 216 | return PRODUCER_KAFKA_STATS_INTERVAL; 217 | } 218 | 219 | inline void Constants::setConsumerKafkaStatsInterval(){ 220 | int milis; 221 | try 222 | { 223 | if(!Common::Utils::convertToInt(Common::Constants::GetConsumerConfig().at(Common::Constants::CONFIG_STATISTICS_INTERVAL_MS), milis)) 224 | { 225 | milis = Common::Constants::CONFIG_STATISTICS_INTERVAL_MS_DEFAULT; // DEFAULT 226 | } 227 | } 228 | catch(std::exception& e) 229 | { 230 | Common::Logger::globalWarning(__PRETTY_FUNCTION__," Constants ConsumerConfig does not have ", Common::Constants::CONFIG_STATISTICS_INTERVAL_MS); 231 | milis = Common::Constants::CONFIG_STATISTICS_INTERVAL_MS_DEFAULT; // DEFAULT 232 | } 233 | CONSUMER_KAFKA_STATS_INTERVAL = milis; 234 | } 235 | 236 | inline const uint32_t& Constants::getConsumerKafkaStatsInterval(){ 237 | return CONSUMER_KAFKA_STATS_INTERVAL; 238 | } 239 | 240 | 241 | }//namespace 242 | #endif /* CONSTANTS_HXX_ */ 243 | -------------------------------------------------------------------------------- /Common/Logger.cxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #include "Logger.hxx" 16 | #include "Common/Constants.hxx" 17 | #include 18 | 19 | namespace Common { 20 | 21 | int Logger::loggingLevel = 1; 22 | const char * Logger::timestrformat = "%a, %d.%m.%Y %H:%M:%S"; 23 | 24 | void Logger::globalInfo(int lvl, const char *note1, const char* note2, const char* note3){ 25 | if(loggingLevel > 0 && loggingLevel >= lvl){ 26 | ErrHdl::error( 27 | ErrClass::PRIO_INFO, 28 | ErrClass::ERR_CONTROL, 29 | ErrClass::NOERR, 30 | note1, 31 | note2, 32 | note3); 33 | } 34 | } 35 | 36 | void Logger::globalWarning(const char *note1, const char* note2, const char* note3){ 37 | if(loggingLevel > L0){ 38 | ErrHdl::error( 39 | ErrClass::PRIO_WARNING, 40 | ErrClass::ERR_CONTROL, 41 | ErrClass::NOERR, 42 | note1, 43 | note2, 44 | note3); 45 | } 46 | } 47 | 48 | void Logger::globalError(const char *note1, const char* note2, const char* note3){ 49 | if(loggingLevel > L0){ 50 | ErrHdl::error( 51 | ErrClass::PRIO_FATAL, 52 | ErrClass::ERR_CONTROL, 53 | ErrClass::NOERR, 54 | note1, 55 | note2, 56 | note3); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /Common/Logger.hxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #ifndef DEBUGMETHODS_HXX 16 | #define DEBUGMETHODS_HXX 17 | 18 | #include 19 | #include 20 | 21 | #include 22 | #include 23 | 24 | #include 25 | 26 | using std::mutex; 27 | using std::lock_guard; 28 | 29 | namespace Common { 30 | 31 | /*! 32 | * \class Debug 33 | * \brief Class is handling writing to starndard output debuge informations provided by user. 34 | * Class is singleton and has overloaded operator << (). 35 | * 36 | * \see operator <<() 37 | */ 38 | class Logger{ 39 | public: 40 | Logger() : creationTime(0), logNum(0), devNum(0), prefix(""){ 41 | }; 42 | 43 | Logger(int devNum):creationTime(0), logNum(0), devNum(devNum){ 44 | updatePrefix(); 45 | }; 46 | 47 | ~Logger(); 48 | 49 | /*! 50 | * Change of global logger info level 51 | * \param new logging level 52 | */ 53 | static void setLogLvl(int lvl); 54 | 55 | /*! 56 | * Setting number of device to which logger is binded 57 | * \param device number 58 | */ 59 | void setDevNum(int num); 60 | 61 | static void globalInfo(int lvl, const char *note1 = NULL, const char* note2 = NULL, const char* note3 = NULL); 62 | 63 | static void globalWarning(const char *note1 = NULL, const char* note2 = NULL, const char* note3 = NULL); 64 | 65 | static void globalError(const char *note1 = NULL, const char* note2 = NULL, const char* note3 = NULL); 66 | 67 | static const int L0 = 0; 68 | static const int L1 = 1; 69 | static const int L2 = 2; 70 | static const int L3 = 3; 71 | static const int L4 = 4; 72 | 73 | static const int& getLogLevel(); 74 | private: 75 | 76 | std::fstream& getStream(); 77 | 78 | void closeStream(); 79 | 80 | void updatePrefix(); 81 | 82 | static int loggingLevel; 83 | 84 | std::fstream f; 85 | long creationTime; 86 | int logNum; 87 | 88 | int devNum; 89 | 90 | std::string prefix; 91 | 92 | static const char * timestrformat; 93 | 94 | mutex localVariableDataAccess; 95 | }; 96 | 97 | 98 | inline void Logger::setLogLvl(int lvl){ 99 | loggingLevel = lvl; 100 | } 101 | 102 | 103 | inline const int& Logger::getLogLevel(){ 104 | return loggingLevel; 105 | } 106 | 107 | inline void Logger::setDevNum(int num){ 108 | lock_guard raiiLock(localVariableDataAccess); 109 | devNum = num; 110 | updatePrefix(); 111 | } 112 | 113 | inline void Logger::updatePrefix(){ 114 | prefix = devNum? (char *)("[MS " + CharString(devNum) + "] "): ""; 115 | } 116 | 117 | }//namespace 118 | #endif /* DEBUGMETHODS_HXX_ */ 119 | -------------------------------------------------------------------------------- /Common/Utils.hxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #ifndef UTILS_HXX 16 | #define UTILS_HXX 17 | 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | template 28 | std::ostream& operator << (std::ostream& os, const std::vector& iterable) 29 | { 30 | for (auto & i : iterable) 31 | os << i << " "; 32 | return os; 33 | } 34 | 35 | 36 | namespace Common { 37 | 38 | using std::future; 39 | using std::future_status; 40 | using std::cout; 41 | using std::exception; 42 | using std::endl; 43 | 44 | class Utils 45 | { 46 | public: 47 | template 48 | static bool future_is_ready(const future& fut) 49 | { 50 | bool ret = false; 51 | try 52 | { 53 | future_status status; 54 | status = fut.wait_for(std::chrono::milliseconds(0)); 55 | if (status == future_status::deferred) { 56 | cout << "deferred\n"; 57 | } else if (status == future_status::timeout) { 58 | cout << "timeout\n"; 59 | } else if (status == future_status::ready) { 60 | cout << "ready!\n"; 61 | ret = true; 62 | } 63 | else 64 | { 65 | cout << " Ha!"; 66 | } 67 | } 68 | catch(exception& e) 69 | { 70 | ret = true; 71 | cout << "No state" << endl; 72 | } 73 | return ret; 74 | } 75 | 76 | 77 | static auto split(const std::string& str, char&& delimiter = '$') -> std::vector 78 | { 79 | std::vector result; 80 | std::size_t previous = 0; 81 | std::size_t current = 0; 82 | 83 | while ((current = str.find(delimiter, previous))!= std::string::npos) { 84 | result.emplace_back(str.substr(previous, current - previous)); 85 | previous = current + 1; 86 | } 87 | 88 | result.emplace_back(str.substr(previous)); 89 | 90 | return result; 91 | } 92 | 93 | static bool convertToInt(const std::string& str, int& out) 94 | { 95 | bool ret = false; 96 | try 97 | { 98 | out = std::stoi(str.c_str()); 99 | ret = true; 100 | } 101 | catch (std::exception& e) 102 | { 103 | std::cout << "Exception : " << e.what() << std::endl; 104 | ret = false; 105 | } 106 | 107 | return ret; 108 | } 109 | }; 110 | 111 | 112 | class DataRAII 113 | { 114 | 115 | }; 116 | 117 | } 118 | 119 | #endif // UTILS_HXX 120 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | CERN HSE Computing (HSE-TS-CS) 2 | ================================================== 3 | 4 | Contact email: hse-ts-cs@cern.ch 5 | 6 | WinCC OA Kafka Driver 7 | ================================================== 8 | 9 | 10 | # Table of contents # 11 | 1. [Description](#toc1) 12 | 2. [Libraries](#toc2) 13 | 14 | 2.1 [Requirements for external libraries](#toc2.1) 15 | 16 | 3. [Compilation](#toc3) 17 | 18 | 3.1. [To build the driver](#toc3.1) 19 | 20 | 3.2. [Build options](#toc3.2) 21 | 22 | 3.3. [To install and update](#toc3.3) 23 | 24 | 3.4. [Run](#toc3.4) 25 | 26 | 4. [Config file](#toc4) 27 | 28 | 4.1 [Keytab (CERN Only)](#toc4.1) 29 | 30 | 5. [WinCC OA Installation](#toc5) 31 | 32 | 6. [Ready-to-launch project (CERN only)](#toc6) 33 | 34 | 6.1. [Project demo set-up](#toc6.1) 35 | * 6.1.1 [Download & unzip](#toc6.1.1) 36 | * 6.1.2 [Kafka Topic](#toc6.1.2) 37 | * 6.1.3 [Keytab](#toc6.1.3) 38 | * 6.1.4 [Config](#toc6.1.4) 39 | * 6.1.5 [Install libraries](#toc6.1.5) 40 | * 6.1.6 [Run the project](#toc6.1.6) 41 | 42 | 7. [Kafka Driver Technical Documentation](#toc7) 43 | 44 | 7.1. [Main entry points](#toc7.1) 45 | 46 | 7.2. [Addressing DPEs with the Kafka driverl](#toc7.2) 47 | 48 | * 7.2.1 [Data types](#toc7.2.1) 49 | * 7.2.2 [For streaming](#toc7.2.2) 50 | * 7.2.3 [For data ingestion](#toc7.2.3) 51 | * 7.2.4 [Adding a new type](#toc7.2.4) 52 | 53 | 7.3 [Driver configuration](#toc7.3) 54 | 55 | 7.4 [Activity diagram](#toc7.4) 56 | 57 | 58 | 59 | 60 | # 1. Description # 61 | 62 | This is a generic WinCC OA driver used to stream and/or ingest data via Kafka. 63 | The user has the following possibilities: 64 | 65 | * Run a producer 66 | * Run one or several consumers (one consumer per topic) 67 | * Run both a producer and/or consumer(s) 68 | 69 | 70 | 71 | # 2. Libraries # 72 | 73 | * C++11 STL 74 | * WinCC OA API libraries 75 | * [cppkafka](https://github.com/mfontanini/cppkafka) under BSD2 (check CMakeLists for release version) 76 | * [librdkafka](https://github.com/confluentinc/librdkafka) under BSD2 (check CMakeLists for release version) 77 | 78 | 79 | 80 | 81 | ## 2.1 Requirements for external libraries ## 82 | `librdkafka` and `cppkafka` are open source libraries, available on Github. They are both built from source via CMake. 83 | 84 | You need to make sure to install the following libs on your system: 85 | * cyrus-sasl-gssapi 86 | * boost 87 | * cmake 88 | * openssl-dev 89 | 90 | 91 | 92 | # 3. Compilation: 93 | The project is CMake enabled. 94 | Note that if you set the `PVSS_PROJ_PATH` environment variable beforehand, you can install and update the driver via the `install` and `update` targets. 95 | 96 | 97 | 98 | ## 3.1 To build the driver 99 | 100 | export PVSS_PROJ_PATH= 101 | mkdir build 102 | cd build 103 | cmake .. 104 | make -j 105 | 106 | 107 | 108 | ## 3.2 Build options 109 | 110 | The default CMake build type is `Release`. The follow build types are taken into account: 111 | 112 | | Build Type | Details | 113 | |------------|----------------------------------| 114 | | Release | Default, 03 optimization | 115 | | Debug | gdb enabled, no optimization | 116 | | Coverage | gcov enabled, no optimization | 117 | ------------------------------------------------- 118 | 119 | To enable a specific build type, run: 120 | 121 | cmake -DCMAKE_BUILD_TYPE= .. 122 | 123 | For Code Coverage make sure you have `lcov` installed. At the end of your tests, perform the following to get the html coverage report: 124 | 125 | make coverage 126 | 127 | If `valgrind` is installed, you can also run the tests with valgrind: 128 | 129 | make valgrind 130 | Once you are done with the tests, you can stop the valgrind process with: 131 | 132 | make kill 133 | 134 | 135 | 136 | 137 | ## 3.3 To install and update 138 | 139 | In order to install the driver in the `PVSS_PROJ_PATH/bin`, run: 140 | 141 | make install 142 | 143 | Making changes to the driver and updating it are simplified by using the `update` target, triggering a `clean`, `install` (and `kill` the driver): 144 | 145 | make update 146 | 147 | The `kill` target will simply terminate the running process. Combined with the `always` setting of the driver in WinCCOA, this will ensure that the driver is always up to date. 148 | 149 | 150 | 151 | ## 3.3 Run 152 | 153 | It can be run from the WinCCOA Console or from command line and it will require the `config.kafka` driver_config_file: 154 | 155 | ./WINCCOAkafkaDrv -num -proj +config 156 | 157 | If you want to run the driver from the `build` folder (i.e. debug, coverage or valgrind mode) you can call: 158 | 159 | make run 160 | 161 | CMake will extract/parse the following information: 162 | 163 | | Variable | Details | 164 | |-----------------------|-------------------------------------------------------------------------------| 165 | | project_name | The PVSS project name from PVSS_PROJ_PATH environment variable | 166 | | driver_number | The driver number from the $PVSS_PROJ_PATH/config/progs file. Defaults to 999.| 167 | | driver_config_file | The driver config file from the $PVSS_PROJ_PATH/config/progs file. | 168 | 169 | 170 | # 4. Config file # 171 | 172 | The `config.kafka` file has to be present under the WinCCOA project folder `config`. 173 | 174 | Here, producer and consumer configs can be specified (based on your own needs). These will be parsed by the driver at startup thanks to the `PRODUCER.CONFIG.` keyword for producer and the `CONSUMER.CONFIG.` keyword for the consumer(s). 175 | 176 | Here is an example config file: 177 | ``` 178 | [kafka] 179 | PRODUCER.CONFIG.metadata.broker.list = dbnile-kafka-a-8.cern.ch:9093,dbnile-kafka-b-8.cern.ch:9093,dbnile-kafka-c-8.cern.ch:9093 180 | PRODUCER.CONFIG.security.protocol = SASL_SSL 181 | PRODUCER.CONFIG.sasl.mechanism = GSSAPI 182 | PRODUCER.CONFIG.sasl.kerberos.service.name = kafka 183 | PRODUCER.CONFIG.sasl.kerberos.principal = asavules 184 | PRODUCER.CONFIG.sasl.kerberos.keytab = /home/asavules/Workspace/as.keytab 185 | PRODUCER.CONFIG.group.id = test-consumer-group 186 | PRODUCER.CONFIG.statistics.interval.ms = 60000 187 | CONSUMER.CONFIG.metadata.broker.list = dbnile-kafka-a-8.cern.ch:9093,dbnile-kafka-b-8.cern.ch:9093,dbnile-kafka-c-8.cern.ch:9093 188 | CONSUMER.CONFIG.security.protocol = SASL_SSL 189 | CONSUMER.CONFIG.sasl.mechanism = GSSAPI 190 | CONSUMER.CONFIG.sasl.kerberos.service.name = kafka 191 | CONSUMER.CONFIG.sasl.kerberos.principal = asavules 192 | CONSUMER.CONFIG.sasl.kerberos.keytab = /home/asavules/Workspace/as.keytab 193 | CONSUMER.CONFIG.statistics.interval.ms = 60000 194 | CONSUMER.CONFIG.group.id = remustest_meas_consumer 195 | CONSUMER.CONFIG.enable.auto.commit = true 196 | CONSUMER.CONFIG.auto.offset.reset = latest 197 | 198 | ``` 199 | In this config file you can add other producer or consumer configuration entries(see [Kafka Official Documentation](https://kafka.apache.org/documentation)). 200 | 201 | 202 | 203 | ## 4.1 Keytab (CERN Only) ## 204 | 205 | To be able to access kafka, you will need to create a keytab. 206 | 207 | Please refer to official user guide, section `How to generate a keytab file`: 208 | [CERN Kafka Official User Guide](https://nile-user-guide.web.cern.ch/nile-user-guide/kafka/security.html) 209 | 210 | For example with NICE user `jdoe`: 211 | ``` 212 | cern-get-keytab --user --keytab jdoe.keytab --login jdoe 213 | ``` 214 | 215 | Edit the `config/config.kafka` file in your WinCC OA project and update the `CONSUMER.CONFIG.sasl.kerberos.keytab` and/or `PRODUCER.CONFIG.sasl.kerberos.keytab` entry to point to your keytab file + the `CONSUMER.CONFIG.sasl.kerberos.principal` and/or `PRODUCER.CONFIG.sasl.kerberos.principal` to your NICE username. 216 | 217 | For example: 218 | 219 | [kafka] 220 | PRODUCER.CONFIG.metadata.broker.list = dbnile-kafka-a-8.cern.ch:9093,dbnile-kafka-b-8.cern.ch:9093,dbnile-kafka-c-8.cern.ch:9093 221 | PRODUCER.CONFIG.security.protocol = SASL_SSL 222 | PRODUCER.CONFIG.sasl.mechanism = GSSAPI 223 | PRODUCER.CONFIG.sasl.kerberos.service.name = kafka 224 | PRODUCER.CONFIG.sasl.kerberos.principal = jdoe 225 | PRODUCER.CONFIG.sasl.kerberos.keytab = /home/jdoe/Workspace/jdoe.keytab 226 | PRODUCER.CONFIG.group.id = test-producer-group-demo 227 | PRODUCER.CONFIG.statistics.interval.ms = 60000 228 | 229 | CONSUMER.CONFIG.metadata.broker.list = dbnile-kafka-a-8.cern.ch:9093,dbnile-kafka-b-8.cern.ch:9093,dbnile-kafka-c-8.cern.ch:9093 230 | CONSUMER.CONFIG.security.protocol = SASL_SSL 231 | CONSUMER.CONFIG.sasl.mechanism = GSSAPI 232 | CONSUMER.CONFIG.sasl.kerberos.service.name = kafka 233 | CONSUMER.CONFIG.sasl.kerberos.principal = jdoe 234 | CONSUMER.CONFIG.sasl.kerberos.keytab = /home/jdoe/Workspace/jdoe.keytab 235 | CONSUMER.CONFIG.statistics.interval.ms = 60000 236 | CONSUMER.CONFIG.group.id = remustest_demo_consumer 237 | CONSUMER.CONFIG.enable.auto.commit = true 238 | 239 | 240 | 241 | 242 | # 5. WinCC OA Installation # 243 | 244 | Under the [winccoa folder](./winccoa/) you will find the following files that you need to copy to your pre-existing project in the corresponding paths: 245 | 246 | * [dplist/kafka_driver_config.dpl](./winccoa/dplist/kafka_driver_config.dpl) : it contains `CONFIG_KAFKA DPEs`. It is used to set the driver options such as log levels and to retreive consumer and producer statistics. Once you've successfully launched the driver in the WinCC project management, you can import it via the ASCII Manager(refer to the official WinCC OA Documentation). 247 | 248 | Notes: 249 | 250 | * The internal driver number in the dump is 2. If it's unavailable to you, try to modify the dump file directly. 251 | 252 | * [dplist/panels/para/address_kafka.pnl](./winccoa/panels/para/address_kafka.pnl) : a panel that you can use in para for kafka addressing. If you install this panel, then you will also need the WinCC OA scripts that go along: 253 | 254 | * [scripts/userDrivers.ctl](./winccoa/scripts/userDrivers.ctl) 255 | * [scripts/userPara.ctl](./winccoa/scripts/userPara.ctl) 256 | 257 | In order to set up the addressing of kafka DPE form a control script, you may use the folowing library: 258 | * [scripts/libs/kafka_dpe_addressing.ctl](./winccoa/scripts/libs/kafka_dpe_addressing.ctl). 259 | Usage: 260 | 261 | `kafkaAddress_addressDPE("Consumer.myDPE", kafkaAddress_DATA_TYPE_STRING, kafkaAddress_MODE_IN, 2, "my_kafka_topic", "my_kafka_key");` 262 | `kafkaAddress_addressDPE("Producer.myDPE", kafkaAddress_DATA_TYPE_STRING, kafkaAddress_MODE_OUT, 2, "my_kafka_topic", "my_kafka_key", 100);` 263 | 264 | 265 | See [7.3 Driver configuration](#toc7.3) section for a brief descprition of relevant CONFIG_KAFKA DPEs. 266 | 267 | 268 | # 6. Ready-to-launch project (CERN only) # 269 | A ready-to-launch WinCCOA 3.16 kafka consumer demo project available here: 270 | * [winccoa316_demo_project/kafka_producer_consumer_demo.zip](./winccoa316_demo_project/kafka_producer_consumer_demo.zip) 271 | 272 | 273 | 274 | ## 6.1 Project demo set-up ## 275 | 276 | 277 | 278 | 279 | ### 6.1.1 Download & unzip 280 | 281 | Download [winccoa316_demo_project/kafka_producer_consumer_demo.zip](./winccoa316_demo_project/kafka_producer_consumer_demo.zip). 282 | 283 | Unzip it on a CentOS7 machine that have WinCCOA 3.16 installed. You will need sudo rights to install libraries. 284 | 285 | 286 | 287 | ### 6.1.2 Kafka Topic 288 | 289 | The topic `remustest_demo` from where the values are retrieved in the demo is available on the CERN `kafka-gptn` cluster. 290 | 291 | To be able to retrieve the values for the demo project, you need to get access for this topic. Please subscribe to the [kafka-remustest-demo](https://e-groups.cern.ch/e-groups/Egroup.do?egroupId=10328116) egroup. 292 | 293 | 294 | 295 | ## 6.1.3. Keytab 296 | 297 | To be able to access kafka, you will need to create a keytab, to replace the empty one in config/ . 298 | 299 | go to [PROJECT PATH]/config 300 | and run 301 | 302 | cern-get-keytab --user --keytab user.keytab --login [NICE login] 303 | 304 | See [4.1. Keytab (CERN Only)](#toc4.1) section for more details 305 | 306 | 307 | 308 | ## 6.1.4. Config 309 | 310 | Edit the `config/config.kafka` file and update: 311 | 312 | * `CONSUMER.CONFIG.sasl.kerberos.keytab` path to your keytab file (consumer). 313 | * `CONSUMER.CONFIG.sasl.kerberos.principal` your CERN's NICE username (consumer). 314 | * `PRODUCER.CONFIG.sasl.kerberos.keytab` path to your keytab file (producer). 315 | * `PRODUCER.CONFIG.sasl.kerberos.principal` your CERN's NICE username (producer). 316 | 317 | Edit the `config/config` file to point to the appropriate project path. 318 | 319 | 320 | 321 | ## 6.1.5. Install libraries 322 | 323 | First run : 324 | 325 | sudo yum install cyrus-sasl-gssapi 326 | 327 | Then go to `bin/libs/` and run: 328 | 329 | ./installLocal.sh 330 | 331 | This will copy the libraries and update dependencies. Note that you need sudo rights. 332 | 333 | 334 | 335 | ## 6.1.6. Run the project 336 | 337 | Simply register then start the project in WinCCOA. 338 | 339 | A UI manager will launch para module, where producer and consumer DPEs are already set-up and running. 340 | 341 | ![](./doc/kafkaDemoProjectDPEs.gif) 342 | 343 | 344 | The manager -f kafka_producer_example.lst is continuously doing dpSets into the Producer DPEs, that are sending messages to the kafka topic "remustest_demo". The messages are transmitted to Kafka, then read by the Consumer DPEs. 345 | 346 | Since the messages are going through Kafka, they can of course be read/write from external software (other WCCOA project, java, python, MQTT, ...) communicating with the same topic, as demonstrated on the screeshot, where messages are sent through the kafka command line tool (https://kafka.apache.org/quickstart) 347 | 348 | (Optional) Running the manager -f kafka_addressing_example.lst will demonstrate a dynamic addressing of kafka DPE, both producers and consumers, from Control scripts. 349 | 350 | 351 | 352 | # 7. Kafka Driver Technical Documentation # 353 | 354 | 355 | 356 | ## 7.1 Main entry points ## 357 | After the driver startup, the main entry points in the driver are: 358 | 359 | * kafkaHwService::writeData() -> WinCC to Driver communication 360 | 361 | This is how the kafka streaming is performed. Thanks to the addressing `$[$]`, the driver will be able to stream to the right topic. 362 | 363 | * kafkaHwService::workProc() -> Driver to WinCC communication 364 | 365 | This is how we push data to WinCC from Kafka. Thanks to the addressing `$`,the driver will be able to map the data ingested from the respective Kafka topic to the WinCC DPE. 366 | 367 | Please refer to the WinCC documentation for more information on the WinCC OA API. 368 | For more info on the debouncing, see [Remus RealTime Evolution - KAFKA presentation](./doc/REMUS_RealTime_Evolution_-_KAFKA.pptx). 369 | 370 | 371 | 372 | ## 7.2 Addressing DPEs with the Kafka driver ## 373 | 374 | 375 | 376 | ### 7.2.1 Data Types ### 377 | When the kafka driver pushes a DPE value to WinCC, a transformation takes place. See [Transformations folder](./Transformations). We are currently supporting the following data types for the periphery address: 378 | 379 | -------------------------------------------------------------------------------------------------------------------------------- 380 | | WinCC DataType | Transformation class | Periphery data type value | 381 | | ------------------| --------------------------------------------------------------| ----------------------------------------- | 382 | | bool | [kafkaBoolTrans.cxx](./Transformations/kafkaBoolTrans.cxx) | 1000 (TransUserType def in WinCC OA API) | 383 | | uint8 | [kafkaUint8Trans.cxx](./Transformations/kafkaUint8Trans.cxx) | 1001 (TransUserType + 1) | 384 | | int32 | [kafkaInt32Trans.cxx](./Transformations/kafkaInt32Trans.cxx) | 1002 (TransUserType + 2) | 385 | | int64 | [kafkaInt64Trans.cxx](./Transformations/kafkaInt64Trans.cxx) | 1003 (TransUserType + 3) | 386 | | float | [kafkaFloatTrans.cxx](./Transformations/kafkaFloatTrans.cxx) | 1004 (TransUserType + 4) | 387 | | string | [kafkaStringTrans.cxx](./Transformations/kafkaStringTrans.cxx)| 1005 (TransUserType + 5) | 388 | | time | [kafkaTimeTrans.cxx](./Transformations/kafkaTimeTrans.cxx) | 1006 (TransUserType + 6) | 389 | -------------------------------------------------------------------------------------------------------------------------------- 390 | 391 | 392 | 393 | ### 7.2.2 For streaming ### 394 | 395 | The periphery address data type has to be set to `string` (1005). 396 | 397 | The direction in the periphery has to be set to `OUT`. The addressing is generic: `$[$]`. The `$DEBOUNCING_TIMEFRAME` is optional and if not set then it is 0 by default. 398 | 399 | 400 | * Example 1 401 | 402 | remustest_demo$MeasurementJSON123$200 403 | 404 | Gives us : 405 | * The kafka topic: remustest_demo 406 | * The key key: MeasurementJSON123 407 | * Debouncing timeframe: 200 milliseconds 408 | 409 | * Example 2 410 | 411 | remustest_demo$MeasurementJSON123 412 | 413 | Gives us : 414 | * The kafka topic: remustest_demo 415 | * The key key: MeasurementJSON123 416 | * No debouncing timeframe 417 | 418 | 419 | For more info on the addressing, see [Remus RealTime Evolution - KAFKA presentation](./doc/REMUS_RealTime_Evolution_-_KAFKA.pptx). 420 | 421 | 422 | 423 | ### 7.2.3 For data ingestion ### 424 | 425 | 426 | The direction in the periphery address has to be set to `IN`. The addressing is generic: `$`. 427 | 428 | For example 429 | 430 | remustest_demo$MeasurementJSON123 431 | 432 | Gives us : 433 | * The kafka topic: remustest_demo 434 | * The key key: MeasurementJSON123 435 | 436 | 437 | 438 | ### 7.2.4 Adding a new transformation ### 439 | 440 | To add a new transformation you need to do the following: 441 | 442 | * create a define in `kafkaHWMappeer.hxx` 443 | 444 | #define kafkaDrvDoubleTransType (TransUserType + 7) 445 | 446 | * handle the new transformation type in `kafkaHWMapper::addDpPa()` 447 | * implement the transformation type class. The important functions here are 448 | 449 | * `::toPeriph(...)` for WinCC OA to Kafka driver transformation 450 | * `::toVar(...)` for Kafka driver to WinCC OA transformation 451 | 452 | 453 | 454 | 455 | ## 7.3 Driver Configuration ## 456 | 457 | Available via the WinCC OA `CONFIG_KAFKA` DataPoint, we have the following 458 | 459 | | Config DPE | Direction | Addressing | Type | Description | 460 | | ------------- | --------- | ------------- | --------- | ------------- | 461 | | DebugLvl | OUT | DEBUGLVL | INT32 | Debug Level for logging. You can use this to debug issues. (default 0) | 462 | | DebouncingThreadInterval | OUT | DEBOUNCINGTHREADINTERVAL | INT32 | In milliseconds. The sleep interval for the debouncing thread (default 50 ms) | 463 | | MaxPollRecords | OUT | MAXPOLLRECORDS | INT32 | MAximum number of records should the consumer poll retrieve (default 1000) | 464 | | IN.ConsumerStatsDP | IN | CONSUMER_STATISTICS | STRING | This is where the driver periodically pushes statistics from kafka consumer (json) | 465 | | IN.ProducerStatsDP | IN | PRODUCER_STATISTICS | STRING | This is where the driver periodically pushes statistics from kafka producer (json) | 466 | | IN.ProducerAllBrokersDown | IN | PRODUCER_ALL_BROKERS_DOWN | BOOL | Set by the when all brokers are down for the producer. | 467 | | IN.NoConsumerConfig | IN | CONSUMER_NO_CONFIG | BOOL | Indicates there is no consumer configuration available. | 468 | | IN.NoProducerConfig | IN | PRODUCER_NO_CONFIG | BOOL | Indicates there is no producer configuration available. | 469 | 470 | 471 | 472 | 473 | ## 7.4 Activity Diagram ## 474 | 475 | ![](./doc/kafkaActivity.svg) 476 | -------------------------------------------------------------------------------- /Transformations/kafkaBoolTrans.cxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #include "kafkaBoolTrans.hxx" 16 | 17 | #include "kafkaHWMapper.hxx" 18 | 19 | #include "Common/Logger.hxx" 20 | 21 | #include 22 | 23 | #include "BitVar.hxx" 24 | 25 | namespace Transformations{ 26 | 27 | TransformationType kafkaBoolTrans::isA() const { 28 | return (TransformationType) kafkaDrvBoolTransType; 29 | } 30 | 31 | TransformationType kafkaBoolTrans::isA(TransformationType type) const { 32 | if (type == isA()) 33 | return type; 34 | else 35 | return Transformation::isA(type); 36 | } 37 | 38 | Transformation *kafkaBoolTrans::clone() const { 39 | return new kafkaBoolTrans; 40 | } 41 | 42 | int kafkaBoolTrans::itemSize() const { 43 | return size; 44 | } 45 | 46 | VariableType kafkaBoolTrans::getVariableType() const { 47 | return BIT_VAR; 48 | } 49 | 50 | PVSSboolean kafkaBoolTrans::toPeriph(PVSSchar *buffer, PVSSuint len, const Variable &var, const PVSSuint subix) const { 51 | 52 | if(var.isA() != BIT_VAR){ 53 | ErrHdl::error(ErrClass::PRIO_SEVERE, // Data will be lost 54 | ErrClass::ERR_PARAM, // Wrong parametrization 55 | ErrClass::UNEXPECTEDSTATE, // Nothing else appropriate 56 | "kafkaBoolTrans", "toPeriph", // File and function name 57 | "Wrong variable type " + CharString(subix)// Unfortunately we don't know which DP 58 | ); 59 | return PVSS_FALSE; 60 | } 61 | 62 | *reinterpret_cast(buffer) = (reinterpret_cast(var)).getValue(); 63 | 64 | return PVSS_TRUE; 65 | } 66 | 67 | VariablePtr kafkaBoolTrans::toVar(const PVSSchar *buffer, const PVSSuint dlen, const PVSSuint subix) const { 68 | 69 | if(!dlen || buffer == NULL) 70 | { 71 | 72 | ErrHdl::error(ErrClass::PRIO_SEVERE, // Data will be lost 73 | ErrClass::ERR_PARAM, // Wrong parametrization 74 | ErrClass::UNEXPECTEDSTATE, // Nothing else appropriate 75 | "kafkaBoolTrans", "toVar", // File and function name 76 | "Wrong buffer" // Unfortunately we don't know which DP 77 | ); 78 | } 79 | else 80 | { 81 | if(!strcmp((const char*)buffer,"FALSE")) 82 | return new BitVar(false); 83 | else if(!strcmp((const char*)buffer,"TRUE")) 84 | return new BitVar(true); 85 | else 86 | { 87 | Common::Logger::globalWarning(__PRETTY_FUNCTION__, "Unable to parse boolean for buffer:", (const char*)buffer); 88 | } 89 | } 90 | return NULL; 91 | } 92 | 93 | }//namespace 94 | -------------------------------------------------------------------------------- /Transformations/kafkaBoolTrans.hxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #ifndef kafkaBOOLTRANS_HXX_ 16 | #define kafkaBOOLTRANS_HXX_ 17 | 18 | #include 19 | 20 | namespace Transformations{ 21 | 22 | class kafkaBoolTrans: public Transformation { 23 | /*! 24 | * Transformations typ 25 | * \return transformation type 26 | */ 27 | TransformationType isA() const; 28 | /*! 29 | * Transformations typ comparison 30 | * \param type object to return type 31 | * \return transformation type 32 | */ 33 | TransformationType isA(TransformationType type) const; 34 | 35 | /*! 36 | * Size of transformation buffer 37 | * \return size of buffer 38 | */ 39 | int itemSize() const; 40 | 41 | /*! 42 | * The type of Variable we are expecting here 43 | * \return actual variable type 44 | */ 45 | VariableType getVariableType() const; 46 | 47 | /*! 48 | * Clone of our class 49 | * \return pointer to new object 50 | */ 51 | Transformation *clone() const; 52 | 53 | /*! 54 | * Conversion from PVSS to Hardware 55 | * \param dataPtr pointer to buffer where data will be written 56 | * \param len size of data buffer 57 | * \param var reference to current translated value 58 | * \param subix subindex of value in data point 59 | * \return flag if translation was successful 60 | */ 61 | PVSSboolean toPeriph(PVSSchar *dataPtr, PVSSuint len, const Variable &var, 62 | const PVSSuint subix) const; 63 | 64 | /*! 65 | * Conversion from Hardware to PVSS 66 | * \param data pointer to buffer from where data will be read 67 | * \param dlen length of data buffer 68 | * \param subix subindex of value associated with peripheral address 69 | * \return flag if translation was successful 70 | */ 71 | VariablePtr toVar(const PVSSchar *data, const PVSSuint dlen, 72 | const PVSSuint subix) const; 73 | 74 | private: 75 | const static uint8_t size = sizeof(bool); 76 | }; 77 | 78 | }//namepsace 79 | #endif /* kafkaBOOLTRANS_HXX_ */ 80 | 81 | -------------------------------------------------------------------------------- /Transformations/kafkaFloatTrans.cxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #include "kafkaFloatTrans.hxx" 16 | 17 | #include "kafkaHWMapper.hxx" 18 | 19 | #include "Common/Logger.hxx" 20 | 21 | #include 22 | 23 | #include 24 | 25 | namespace Transformations{ 26 | 27 | TransformationType kafkaFloatTrans::isA() const { 28 | return (TransformationType) kafkaDrvFloatTransType; 29 | } 30 | 31 | TransformationType kafkaFloatTrans::isA(TransformationType type) const { 32 | if (type == isA()) 33 | return type; 34 | else 35 | return Transformation::isA(type); 36 | } 37 | 38 | Transformation *kafkaFloatTrans::clone() const { 39 | return new kafkaFloatTrans; 40 | } 41 | 42 | int kafkaFloatTrans::itemSize() const { 43 | return size; 44 | } 45 | 46 | VariableType kafkaFloatTrans::getVariableType() const { 47 | return FLOAT_VAR; 48 | } 49 | 50 | PVSSboolean kafkaFloatTrans::toPeriph(PVSSchar *buffer, PVSSuint len, const Variable &var, const PVSSuint subix) const { 51 | 52 | if(var.isA() != FLOAT_VAR){ 53 | ErrHdl::error(ErrClass::PRIO_SEVERE, // Data will be lost 54 | ErrClass::ERR_PARAM, // Wrong parametrization 55 | ErrClass::UNEXPECTEDSTATE, // Nothing else appropriate 56 | "kafkaFloatTrans", "toPeriph", // File and function name 57 | "Wrong variable type" // Unfortunately we don't know which DP 58 | ); 59 | 60 | return PVSS_FALSE; 61 | } 62 | reinterpret_cast(buffer)[subix] = (float)(reinterpret_cast(var)).getValue(); 63 | return PVSS_TRUE; 64 | } 65 | 66 | VariablePtr kafkaFloatTrans::toVar(const PVSSchar *buffer, const PVSSuint dlen, const PVSSuint subix) const { 67 | 68 | if(buffer == NULL || !dlen/*|| dlen%size > 0 || dlen < size*(subix+1)*/){ 69 | ErrHdl::error(ErrClass::PRIO_SEVERE, // Data will be lost 70 | ErrClass::ERR_PARAM, // Wrong parametrization 71 | ErrClass::UNEXPECTEDSTATE, // Nothing else appropriate 72 | "kafkaFloatTrans", "toVar", // File and function name 73 | "Null buffer pointer or wrong length: " + CharString(dlen) // Unfortunately we don't know which DP 74 | ); 75 | } 76 | else 77 | { 78 | try 79 | { 80 | float f = std::stof((char*)buffer); 81 | return new FloatVar(f); 82 | } 83 | catch(std::exception& e) 84 | { 85 | Common::Logger::globalWarning(__PRETTY_FUNCTION__, e.what()); 86 | } 87 | } 88 | 89 | return NULL; 90 | } 91 | 92 | }//namespace 93 | -------------------------------------------------------------------------------- /Transformations/kafkaFloatTrans.hxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #ifndef kafkaFLOATTRANS_HXX_ 16 | #define kafkaFLOATTRANS_HXX_ 17 | 18 | #include 19 | 20 | namespace Transformations{ 21 | 22 | class kafkaFloatTrans: public Transformation { 23 | /*! 24 | * Transformations typ 25 | * \return transformation type 26 | */ 27 | TransformationType isA() const; 28 | /*! 29 | * Transformations typ comparison 30 | * \param type object to return type 31 | * \return transformation type 32 | */ 33 | TransformationType isA(TransformationType type) const; 34 | 35 | /*! 36 | * Size of transformation buffer 37 | * \return size of buffer 38 | */ 39 | int itemSize() const; 40 | 41 | /*! 42 | * The type of Variable we are expecting here 43 | * \return actual variable type 44 | */ 45 | VariableType getVariableType() const; 46 | 47 | /*! 48 | * Clone of our class 49 | * \return pointer to new object 50 | */ 51 | Transformation *clone() const; 52 | 53 | /*! 54 | * Conversion from PVSS to Hardware 55 | * \param dataPtr pointer to buffer where data will be written 56 | * \param len size of data buffer 57 | * \param var reference to current translated value 58 | * \param subix subindex of value in data point 59 | * \return flag if translation was successful 60 | */ 61 | PVSSboolean toPeriph(PVSSchar *dataPtr, PVSSuint len, const Variable &var, 62 | const PVSSuint subix) const; 63 | 64 | /*! 65 | * Conversion from Hardware to PVSS 66 | * \param data pointer to buffer from where data will be read 67 | * \param dlen length of data buffer 68 | * \param subix subindex of value associated with peripheral address 69 | * \return flag if translation was successful 70 | */ 71 | VariablePtr toVar(const PVSSchar *data, const PVSSuint dlen, 72 | const PVSSuint subix) const; 73 | 74 | 75 | private: 76 | const static uint8_t size = sizeof(float); 77 | }; 78 | 79 | 80 | }//namespace 81 | #endif /* kafkaFLOATTRANS_HXX_ */ 82 | -------------------------------------------------------------------------------- /Transformations/kafkaInt32Trans.cxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #include "kafkaInt32Trans.hxx" 16 | 17 | #include "kafkaHWMapper.hxx" 18 | 19 | #include "Common/Logger.hxx" 20 | 21 | #include 22 | 23 | #include 24 | 25 | namespace Transformations { 26 | 27 | TransformationType kafkaInt32Trans::isA() const { 28 | return (TransformationType) kafkaDrvInt32TransType; 29 | } 30 | 31 | TransformationType kafkaInt32Trans::isA(TransformationType type) const { 32 | if (type == isA()) 33 | return type; 34 | else 35 | return Transformation::isA(type); 36 | } 37 | 38 | Transformation *kafkaInt32Trans::clone() const { 39 | return new kafkaInt32Trans; 40 | } 41 | 42 | int kafkaInt32Trans::itemSize() const { 43 | return size; 44 | } 45 | 46 | VariableType kafkaInt32Trans::getVariableType() const { 47 | return INTEGER_VAR; 48 | } 49 | 50 | PVSSboolean kafkaInt32Trans::toPeriph(PVSSchar *buffer, PVSSuint len, const Variable &var, const PVSSuint subix) const { 51 | 52 | if(var.isA() != INTEGER_VAR /* || subix >= Transformation::getNumberOfElements() */){ 53 | ErrHdl::error(ErrClass::PRIO_SEVERE, // Data will be lost 54 | ErrClass::ERR_PARAM, // Wrong parametrization 55 | ErrClass::UNEXPECTEDSTATE, // Nothing else appropriate 56 | "kafkaInt32Trans", "toPeriph", // File and function name 57 | "Wrong variable type or wrong length: " + CharString(len) + ", subix: " + CharString(subix) // Unfortunately we don't know which DP 58 | ); 59 | 60 | return PVSS_FALSE; 61 | } 62 | reinterpret_cast(buffer)[subix] = reinterpret_cast(var).getValue(); 63 | return PVSS_TRUE; 64 | } 65 | 66 | VariablePtr kafkaInt32Trans::toVar(const PVSSchar *buffer, const PVSSuint dlen, const PVSSuint subix) const { 67 | 68 | if(buffer == NULL || !dlen){ 69 | ErrHdl::error(ErrClass::PRIO_SEVERE, // Data will be lost 70 | ErrClass::ERR_PARAM, // Wrong parametrization 71 | ErrClass::UNEXPECTEDSTATE, // Nothing else appropriate 72 | "kafkaInt32Trans", "toVar", // File and function name 73 | "Null buffer pointer or wrong length: " + CharString(dlen) // Unfortunately we don't know which DP 74 | ); 75 | } 76 | else 77 | { 78 | try 79 | { 80 | int32_t i = std::stoi((char*)buffer); 81 | return new IntegerVar(i); 82 | } 83 | catch(std::exception& e) 84 | { 85 | Common::Logger::globalWarning(__PRETTY_FUNCTION__, e.what()); 86 | } 87 | } 88 | 89 | return NULL; 90 | } 91 | 92 | } 93 | -------------------------------------------------------------------------------- /Transformations/kafkaInt32Trans.hxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #ifndef kafkaINT32TRANS_HXX_ 16 | #define kafkaINT32TRANS_HXX_ 17 | 18 | #include 19 | namespace Transformations { 20 | 21 | 22 | class kafkaInt32Trans: public Transformation { 23 | /*! 24 | * Transformations typ 25 | * \return transformation type 26 | */ 27 | TransformationType isA() const; 28 | /*! 29 | * Transformations typ comparison 30 | * \param type object to return type 31 | * \return transformation type 32 | */ 33 | TransformationType isA(TransformationType type) const; 34 | 35 | /*! 36 | * Size of transformation buffer 37 | * \return size of buffer 38 | */ 39 | int itemSize() const; 40 | 41 | /*! 42 | * The type of Variable we are expecting here 43 | * \return actual variable type 44 | */ 45 | VariableType getVariableType() const; 46 | 47 | /*! 48 | * Clone of our class 49 | * \return pointer to new object 50 | */ 51 | Transformation *clone() const; 52 | 53 | /*! 54 | * Conversion from PVSS to Hardware 55 | * \param dataPtr pointer to buffer where data will be written 56 | * \param len size of data buffer 57 | * \param var reference to current translated value 58 | * \param subix subindex of value in data point 59 | * \return flag if translation was successful 60 | */ 61 | PVSSboolean toPeriph(PVSSchar *dataPtr, PVSSuint len, const Variable &var, 62 | const PVSSuint subix) const; 63 | 64 | /*! 65 | * Conversion from Hardware to PVSS 66 | * \param data pointer to buffer from where data will be read 67 | * \param dlen length of data buffer 68 | * \param subix subindex of value associated with peripheral address 69 | * \return flag if translation was successful 70 | */ 71 | VariablePtr toVar(const PVSSchar *data, const PVSSuint dlen, 72 | const PVSSuint subix) const; 73 | 74 | private: 75 | const static uint8_t size = sizeof(int32_t); 76 | }; 77 | 78 | } 79 | 80 | #endif /* kafkaINT32TRANS_HXX_ */ 81 | 82 | -------------------------------------------------------------------------------- /Transformations/kafkaInt64Trans.cxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #include "kafkaInt64Trans.hxx" 16 | 17 | #include "kafkaHWMapper.hxx" 18 | 19 | #include "Common/Logger.hxx" 20 | 21 | #include 22 | 23 | #include 24 | 25 | namespace Transformations { 26 | 27 | TransformationType kafkaInt64Trans::isA() const { 28 | return (TransformationType) kafkaDrvInt64TransType; 29 | } 30 | 31 | TransformationType kafkaInt64Trans::isA(TransformationType type) const { 32 | if (type == isA()) 33 | return type; 34 | else 35 | return Transformation::isA(type); 36 | } 37 | 38 | Transformation *kafkaInt64Trans::clone() const { 39 | return new kafkaInt64Trans; 40 | } 41 | 42 | int kafkaInt64Trans::itemSize() const { 43 | return size; 44 | } 45 | 46 | VariableType kafkaInt64Trans::getVariableType() const { 47 | return INTEGER_VAR; 48 | } 49 | 50 | PVSSboolean kafkaInt64Trans::toPeriph(PVSSchar *buffer, PVSSuint len, const Variable &var, const PVSSuint subix) const { 51 | 52 | if(var.isA() != INTEGER_VAR /* || subix >= Transformation::getNumberOfElements() */){ 53 | ErrHdl::error(ErrClass::PRIO_SEVERE, // Data will be lost 54 | ErrClass::ERR_PARAM, // Wrong parametrization 55 | ErrClass::UNEXPECTEDSTATE, // Nothing else appropriate 56 | "kafkaInt64Trans", "toPeriph", // File and function name 57 | "Wrong variable type or wrong length: " + CharString(len) + ", subix: " + CharString(subix) // Unfortunately we don't know which DP 58 | ); 59 | 60 | return PVSS_FALSE; 61 | } 62 | reinterpret_cast(buffer)[subix] = reinterpret_cast(var).getValue(); 63 | return PVSS_TRUE; 64 | } 65 | 66 | VariablePtr kafkaInt64Trans::toVar(const PVSSchar *buffer, const PVSSuint dlen, const PVSSuint subix) const { 67 | 68 | if(buffer == NULL || !dlen){ 69 | ErrHdl::error(ErrClass::PRIO_SEVERE, // Data will be lost 70 | ErrClass::ERR_PARAM, // Wrong parametrization 71 | ErrClass::UNEXPECTEDSTATE, // Nothing else appropriate 72 | "kafkaInt64Trans", "toVar", // File and function name 73 | "Null buffer pointer or wrong length: " + CharString(dlen) // Unfortunately we don't know which DP 74 | ); 75 | } 76 | else 77 | { 78 | try 79 | { 80 | int64_t ll = std::stoll((char*)buffer); 81 | return new IntegerVar(ll); 82 | } 83 | catch(std::exception& e) 84 | { 85 | Common::Logger::globalWarning(__PRETTY_FUNCTION__, e.what()); 86 | } 87 | } 88 | return NULL; 89 | 90 | } 91 | 92 | } 93 | -------------------------------------------------------------------------------- /Transformations/kafkaInt64Trans.hxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | #ifndef kafkaINT64TRANS_HXX_ 15 | #define kafkaINT64TRANS_HXX_ 16 | 17 | #include 18 | namespace Transformations { 19 | 20 | 21 | class kafkaInt64Trans: public Transformation { 22 | /*! 23 | * Transformations typ 24 | * \return transformation type 25 | */ 26 | TransformationType isA() const; 27 | /*! 28 | * Transformations typ comparison 29 | * \param type object to return type 30 | * \return transformation type 31 | */ 32 | TransformationType isA(TransformationType type) const; 33 | 34 | /*! 35 | * Size of transformation buffer 36 | * \return size of buffer 37 | */ 38 | int itemSize() const; 39 | 40 | /*! 41 | * The type of Variable we are expecting here 42 | * \return actual variable type 43 | */ 44 | VariableType getVariableType() const; 45 | 46 | /*! 47 | * Clone of our class 48 | * \return pointer to new object 49 | */ 50 | Transformation *clone() const; 51 | 52 | /*! 53 | * Conversion from PVSS to Hardware 54 | * \param dataPtr pointer to buffer where data will be written 55 | * \param len size of data buffer 56 | * \param var reference to current translated value 57 | * \param subix subindex of value in data point 58 | * \return flag if translation was successful 59 | */ 60 | PVSSboolean toPeriph(PVSSchar *dataPtr, PVSSuint len, const Variable &var, 61 | const PVSSuint subix) const; 62 | 63 | /*! 64 | * Conversion from Hardware to PVSS 65 | * \param data pointer to buffer from where data will be read 66 | * \param dlen length of data buffer 67 | * \param subix subindex of value associated with peripheral address 68 | * \return flag if translation was successful 69 | */ 70 | VariablePtr toVar(const PVSSchar *data, const PVSSuint dlen, 71 | const PVSSuint subix) const; 72 | 73 | private: 74 | const static uint8_t size = sizeof(int64_t); 75 | }; 76 | 77 | } 78 | #endif /* kafkaINT64TRANS_HXX_ */ 79 | 80 | -------------------------------------------------------------------------------- /Transformations/kafkaStringTrans.cxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | // Our transformation class PVSS <--> Hardware 16 | #include "kafkaStringTrans.hxx" 17 | #include // The Error handler Basics/Utilities 18 | #include 19 | 20 | 21 | #include "kafkaHWMapper.hxx" 22 | 23 | #include "Common/Logger.hxx" 24 | 25 | //---------------------------------------------------------------------------- 26 | namespace Transformations { 27 | 28 | 29 | //kafkaStringTrans::kafkaStringTrans() : Transformation() { } 30 | 31 | TransformationType kafkaStringTrans::isA() const 32 | { 33 | return (TransformationType) kafkaDrvStringTransType; 34 | } 35 | 36 | TransformationType kafkaStringTrans::isA(TransformationType type) const { 37 | if (type == isA()) 38 | return type; 39 | else 40 | return Transformation::isA(type); 41 | } 42 | 43 | 44 | 45 | //---------------------------------------------------------------------------- 46 | 47 | Transformation *kafkaStringTrans::clone() const 48 | { 49 | return new kafkaStringTrans; 50 | } 51 | 52 | //---------------------------------------------------------------------------- 53 | // Our item size. The max we will use is 256 Bytes. 54 | // This is an arbitrary value! A Transformation for a long e.g. would return 4 55 | 56 | int kafkaStringTrans::itemSize() const 57 | { 58 | // TODO - check maximum possible size 59 | return _size; 60 | } 61 | 62 | //---------------------------------------------------------------------------- 63 | // Our preferred Variable type. Data will be converted to this type 64 | // before toPeriph is called. 65 | 66 | VariableType kafkaStringTrans::getVariableType() const 67 | { 68 | return TEXT_VAR; 69 | } 70 | 71 | //---------------------------------------------------------------------------- 72 | // Convert data from PVSS to Hardware. 73 | 74 | PVSSboolean kafkaStringTrans::toPeriph(PVSSchar *buffer, PVSSuint len, 75 | const Variable &var, const PVSSuint subix) const 76 | { 77 | 78 | // Be paranoic, check variable type 79 | if ( var.isA() != TEXT_VAR ) 80 | { 81 | // Throw error message 82 | ErrHdl::error( 83 | ErrClass::PRIO_SEVERE, // Data will be lost 84 | ErrClass::ERR_PARAM, // Wrong parametrization 85 | ErrClass::UNEXPECTEDSTATE, // Nothing else appropriate 86 | "kafkaStringTrans", "toPeriph", // File and function name 87 | "Wrong variable type for data" // Unfortunately we don't know which DP 88 | ); 89 | 90 | return PVSS_FALSE; 91 | } 92 | 93 | // Check data len. TextVar::getString returns a CharString 94 | const TextVar& tv = static_cast(var); 95 | if (len < tv.getString().len() + 1) 96 | { 97 | // Throw error message 98 | ErrHdl::error( 99 | ErrClass::PRIO_SEVERE, // Data will be lost 100 | ErrClass::ERR_IMPL, // Mus be implementation 101 | ErrClass::UNEXPECTEDSTATE, // Nothing else appropriate 102 | "kafkaStringTrans::toPeriph", // File and function name 103 | "Data buffer too small; need:" + 104 | CharString(tv.getString().len() + 1) + 105 | " have:" + CharString(len) 106 | ); 107 | 108 | return PVSS_FALSE; 109 | } 110 | 111 | if(tv.getString().len() > _size){ 112 | ErrHdl::error(ErrClass::PRIO_SEVERE, // Data will be lost 113 | ErrClass::ERR_PARAM, // Wrong parametrization 114 | ErrClass::UNEXPECTEDSTATE, // Nothing else appropriate 115 | "kafkaStringTrans::toPeriph", // File and function name 116 | "String too long" // Unfortunately we don't know which DP 117 | ); 118 | 119 | return PVSS_FALSE; 120 | } 121 | 122 | sprintf((char*)buffer, "%s", tv.getValue()); 123 | 124 | return PVSS_TRUE; 125 | } 126 | 127 | //---------------------------------------------------------------------------- 128 | // Conversion from Hardware to PVSS 129 | 130 | VariablePtr kafkaStringTrans::toVar(const PVSSchar *buffer, const PVSSuint dlen, 131 | const PVSSuint /* subix */) const 132 | { 133 | 134 | return new TextVar((const char *)buffer, dlen); 135 | } 136 | 137 | //---------------------------------------------------------------------------- 138 | } 139 | -------------------------------------------------------------------------------- /Transformations/kafkaStringTrans.hxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #ifndef kafkaStringTrans_H 16 | #define kafkaStringTrans_H 17 | 18 | #include 19 | 20 | // Our Transformation class for Text 21 | // As the Transformation really depends on the format of data you send and 22 | // receive in your protocol (see HWService), this template is just an 23 | // example. 24 | // Things you have to change are marked with TODO 25 | 26 | 27 | namespace Transformations { 28 | 29 | class kafkaStringTrans : public Transformation 30 | { 31 | public: 32 | // TODO probably your ctor looks completely different ... 33 | //kafkaStringTrans(); 34 | virtual ~kafkaStringTrans() {} 35 | 36 | virtual TransformationType isA() const; 37 | 38 | virtual TransformationType isA(TransformationType type) const; 39 | 40 | // (max) size of one item. This is needed by DrvManager to 41 | // create the buffer used in toPeriph and by the Low-Level-Compare 42 | // For our Text-Transformation we set it arbitrarly to 256 Bytes 43 | virtual int itemSize() const; 44 | 45 | // The type of Variable we are expecting here 46 | virtual VariableType getVariableType() const; 47 | 48 | // Clone of our class 49 | virtual Transformation *clone() const; 50 | 51 | // Conversion from PVSS to Hardware 52 | virtual PVSSboolean toPeriph(PVSSchar *dataPtr, PVSSuint len, const Variable &var, const PVSSuint subix) const; 53 | 54 | // Conversion from Hardware to PVSS 55 | virtual VariablePtr toVar(const PVSSchar *data, const PVSSuint dlen, const PVSSuint subix) const; 56 | private: 57 | const static size_t _size = 1024 * 10; 58 | 59 | }; 60 | 61 | } 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /Transformations/kafkaTimeTrans.cxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #include "kafkaTimeTrans.hxx" 16 | 17 | #include "kafkaHWMapper.hxx" 18 | 19 | #include "Common/Logger.hxx" 20 | 21 | #include 22 | 23 | #include 24 | 25 | #include 26 | 27 | namespace Transformations{ 28 | 29 | TransformationType kafkaTimeTrans::isA() const { 30 | return (TransformationType) kafkaDrvTimeTransType; 31 | } 32 | 33 | TransformationType kafkaTimeTrans::isA(TransformationType type) const { 34 | if (type == isA()) 35 | return type; 36 | else 37 | return Transformation::isA(type); 38 | } 39 | 40 | Transformation *kafkaTimeTrans::clone() const { 41 | return new kafkaTimeTrans; 42 | } 43 | 44 | int kafkaTimeTrans::itemSize() const { 45 | return size; 46 | } 47 | 48 | VariableType kafkaTimeTrans::getVariableType() const { 49 | return TIME_VAR; 50 | } 51 | 52 | PVSSboolean kafkaTimeTrans::toPeriph(PVSSchar *buffer, PVSSuint len, const Variable &var, const PVSSuint subix) const { 53 | 54 | if(var.isA() != TIME_VAR){ 55 | ErrHdl::error(ErrClass::PRIO_SEVERE, // Data will be lost 56 | ErrClass::ERR_PARAM, // Wrong parametrization 57 | ErrClass::UNEXPECTEDSTATE, // Nothing else appropriate 58 | "kafkaTimeTrans", "toPeriph", // File and function name 59 | "THis is not a time variable" // Unfortunately we don't know which DP 60 | ); 61 | return PVSS_FALSE; 62 | } 63 | reinterpret_cast(buffer)[subix] = reinterpret_cast(var).getSeconds() * 1000 + reinterpret_cast(var).getMilli(); 64 | return PVSS_TRUE; 65 | } 66 | 67 | VariablePtr kafkaTimeTrans::toVar(const PVSSchar *buffer, const PVSSuint dlen, const PVSSuint subix) const { 68 | 69 | if(buffer == NULL || !dlen){ 70 | ErrHdl::error(ErrClass::PRIO_SEVERE, // Data will be lost 71 | ErrClass::ERR_PARAM, // Wrong parametrization 72 | ErrClass::UNEXPECTEDSTATE, // Nothing else appropriate 73 | "kafkaTimeTrans", "toVar", // File and function name 74 | "Null buffer pointer or wrong length: " + CharString(dlen) // Unfortunately we don't know which DP 75 | ); 76 | } 77 | else 78 | { 79 | unsigned int miliseconds; 80 | std::tm t; 81 | if (std::sscanf((char*)buffer, "%4d.%2d.%2d %2d:%2d:%2d.%3d", &t.tm_year, &t.tm_mon, 82 | &t.tm_mday, &t.tm_hour, &t.tm_min, &t.tm_sec, &miliseconds) == 7) 83 | { 84 | t.tm_year -= 1900; 85 | t.tm_mon -= 1; 86 | return new TimeVar(PVSSTime(mktime(&t), miliseconds)); 87 | } 88 | } 89 | 90 | return NULL; 91 | } 92 | }//namesapce 93 | -------------------------------------------------------------------------------- /Transformations/kafkaTimeTrans.hxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #ifndef kafkaTIMETRANS_HXX_ 16 | #define kafkaTIMETRANS_HXX_ 17 | 18 | #include 19 | 20 | namespace Transformations{ 21 | 22 | class kafkaTimeTrans: public Transformation { 23 | /*! 24 | * Transformations typ 25 | * \return transformation type 26 | */ 27 | TransformationType isA() const; 28 | /*! 29 | * Transformations typ comparison 30 | * \param type object to return type 31 | * \return transformation type 32 | */ 33 | TransformationType isA(TransformationType type) const; 34 | 35 | /*! 36 | * Size of transformation buffer 37 | * \return size of buffer 38 | */ 39 | int itemSize() const; 40 | 41 | /*! 42 | * The type of Variable we are expecting here 43 | * \return actual variable type 44 | */ 45 | VariableType getVariableType() const; 46 | 47 | /*! 48 | * Clone of our class 49 | * \return pointer to new object 50 | */ 51 | Transformation *clone() const; 52 | 53 | /*! 54 | * Conversion from PVSS to Hardware 55 | * \param dataPtr pointer to buffer where data will be written 56 | * \param len size of data buffer 57 | * \param var reference to current translated value 58 | * \param subix subindex of value in data point 59 | * \return flag if translation was successful 60 | */ 61 | PVSSboolean toPeriph(PVSSchar *dataPtr, PVSSuint len, const Variable &var, 62 | const PVSSuint subix) const; 63 | 64 | /*! 65 | * Conversion from Hardware to PVSS 66 | * \param data pointer to buffer from where data will be read 67 | * \param dlen length of data buffer 68 | * \param subix subindex of value associated with peripheral address 69 | * \return flag if translation was successful 70 | */ 71 | VariablePtr toVar(const PVSSchar *data, const PVSSuint dlen, 72 | const PVSSuint subix) const; 73 | 74 | private: 75 | const static uint8_t size = sizeof(int64_t); 76 | }; 77 | 78 | }//namespace 79 | #endif /* kafkaTIMETRANS_HXX_ */ 80 | -------------------------------------------------------------------------------- /Transformations/kafkaUint8Trans.cxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #include "kafkaUint8Trans.hxx" 16 | 17 | #include "kafkaHWMapper.hxx" 18 | 19 | #include "Common/Logger.hxx" 20 | 21 | #include 22 | 23 | #include 24 | 25 | namespace Transformations{ 26 | 27 | TransformationType kafkaUint8Trans::isA() const { 28 | return (TransformationType) kafkaDrvUint8TransType; 29 | } 30 | 31 | TransformationType kafkaUint8Trans::isA(TransformationType type) const { 32 | if (type == isA()) 33 | return type; 34 | else 35 | return Transformation::isA(type); 36 | } 37 | 38 | Transformation *kafkaUint8Trans::clone() const { 39 | return new kafkaUint8Trans; 40 | } 41 | 42 | int kafkaUint8Trans::itemSize() const { 43 | return size; 44 | } 45 | 46 | VariableType kafkaUint8Trans::getVariableType() const { 47 | return INTEGER_VAR; 48 | } 49 | 50 | PVSSboolean kafkaUint8Trans::toPeriph(PVSSchar *buffer, PVSSuint len, const Variable &var, const PVSSuint subix) const { 51 | 52 | if(var.isA() != INTEGER_VAR){ 53 | ErrHdl::error(ErrClass::PRIO_SEVERE, // Data will be lost 54 | ErrClass::ERR_PARAM, // Wrong parametrization 55 | ErrClass::UNEXPECTEDSTATE, // Nothing else appropriate 56 | "kafkaUint8Trans", "toPeriph", // File and function name 57 | "Wrong variable type" // Unfortunately we don't know which DP 58 | ); 59 | 60 | return PVSS_FALSE; 61 | } 62 | // this one is a bit special as the number is handled by wincc oa as int32, but we handle it as 8 bit unsigned integer 63 | // thus any info above the 8 first bits is lost 64 | *(reinterpret_cast(buffer + (subix * size))) = (uint8_t)(reinterpret_cast(var)).getValue(); 65 | return PVSS_TRUE; 66 | } 67 | 68 | VariablePtr kafkaUint8Trans::toVar(const PVSSchar *buffer, const PVSSuint dlen, const PVSSuint subix) const { 69 | 70 | if(buffer == NULL || !dlen){ 71 | ErrHdl::error(ErrClass::PRIO_SEVERE, // Data will be lost 72 | ErrClass::ERR_PARAM, // Wrong parametrization 73 | ErrClass::UNEXPECTEDSTATE, // Nothing else appropriate 74 | "kafkaUint8Trans", "toVar", // File and function name 75 | "Null buffer pointer or wrong length: " + CharString(dlen) // Unfortunately we don't know which DP 76 | ); 77 | 78 | } 79 | else 80 | { 81 | try 82 | { 83 | uint8_t c = std::stoul((char*)buffer); 84 | return new IntegerVar(c); 85 | } 86 | catch(std::exception& e) 87 | { 88 | Common::Logger::globalWarning(__PRETTY_FUNCTION__, e.what()); 89 | } 90 | 91 | } 92 | 93 | return NULL; 94 | } 95 | 96 | }//namespace 97 | -------------------------------------------------------------------------------- /Transformations/kafkaUint8Trans.hxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #ifndef kafkaUINT8TRANS_HXX_ 16 | #define kafkaUINT8TRANS_HXX_ 17 | 18 | #include 19 | 20 | namespace Transformations{ 21 | 22 | class kafkaUint8Trans: public Transformation { 23 | /*! 24 | * Transformations type 25 | * \return transformation type 26 | */ 27 | TransformationType isA() const; 28 | /*! 29 | * Transformations typ comparison 30 | * \param type object to return type 31 | * \return transformation type 32 | */ 33 | TransformationType isA(TransformationType type) const; 34 | 35 | /*! 36 | * Size of transformation buffer 37 | * \return size of buffer 38 | */ 39 | int itemSize() const; 40 | 41 | /*! 42 | * The type of Variable we are expecting here 43 | * \return actual variable type 44 | */ 45 | VariableType getVariableType() const; 46 | 47 | /*! 48 | * Clone of our class 49 | * \return pointer to new object 50 | */ 51 | Transformation *clone() const; 52 | 53 | /*! 54 | * Conversion from PVSS to Hardware 55 | * \param dataPtr pointer to buffer where data will be written 56 | * \param len size of data buffer 57 | * \param var reference to current translated value 58 | * \param subix subindex of value in data point 59 | * \return flag if translation was successful 60 | */ 61 | PVSSboolean toPeriph(PVSSchar *dataPtr, PVSSuint len, const Variable &var, 62 | const PVSSuint subix) const; 63 | 64 | /*! 65 | * Conversion from Hardware to PVSS 66 | * \param data pointer to buffer from where data will be read 67 | * \param dlen length of data buffer 68 | * \param subix subindex of value associated with peripheral address 69 | * \return flag if translation was successful 70 | */ 71 | VariablePtr toVar(const PVSSchar *data, const PVSSuint dlen, 72 | const PVSSuint subix) const; 73 | 74 | 75 | private: 76 | const static uint8_t size = sizeof(uint8_t); 77 | }; 78 | 79 | }//namesace 80 | 81 | #endif /* kafkaUINT8TRANS_HXX_ */ 82 | -------------------------------------------------------------------------------- /config.h.in: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #define PROJECT_NAME "@PROJECT_NAME@" 4 | #define PROJECT_VER "@PROJECT_VERSION@" 5 | #define PROJECT_VER_MAJOR "@PROJECT_VERSION_MAJOR@" 6 | #define PROJECT_VER_MINOR "@PROJECT_VERSION_MINOR@" 7 | #define PTOJECT_VER_PATCH "@PROJECT_VERSION_PATCH@" 8 | -------------------------------------------------------------------------------- /doc/REMUS_RealTime_Evolution_-_KAFKA.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cern-hse-computing/WCCOAkafkaDrv/a37d333d8148a4df5ac4c33d01de0605b6ca5e8a/doc/REMUS_RealTime_Evolution_-_KAFKA.pptx -------------------------------------------------------------------------------- /doc/kafkaActivity.uml: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | kafkaActivity.uml 16 | Created on: 10.04.2019 17 | Author: Alexandru Savulescu (HSE-CEN-CO) 18 | 19 | 20 | @startuml 21 | start 22 | 23 | :Read config.kafka file; 24 | note right 25 | Reads all producer and consumer 26 | configuration entries. 27 | kafkaResources::readSection() 28 | end note 29 | :kafkaHWService::start() ; 30 | split 31 | if(Producer config available?) then (yes) 32 | :Start Kafka Producer 33 | + set callbacks; 34 | (P) 35 | note left 36 | kafkaProducerFacade 37 | end note 38 | :start AsynchronousTask for polling 39 | kafkaHWService::doPoll() 40 | This will trigger producer statistics; 41 | :start AsynchronousTask for stream debouncing 42 | kafkaHWService::doStream(); 43 | else (no) 44 | endif 45 | 46 | split again 47 | :For each topic addressed in the driver 48 | launch a consumer thread (+callbacks); 49 | note left 50 | kafkaHWService:: 51 | handleNewConsumerTopic() 52 | end note 53 | fork 54 | :Consumer 1; 55 | (C) 56 | detach 57 | fork again 58 | :Consumer 2; 59 | (C) 60 | detach 61 | fork again 62 | -[#black,dotted]-> 63 | :Consumer N; 64 | -[#black,dotted]-> 65 | (C) 66 | note right 67 | kafkaConsumerFacade 68 | end note 69 | detach 70 | end fork 71 | 72 | split again 73 | :callback for new consumer threads 74 | whenever a new topic is addressed; 75 | (C) 76 | note left 77 | kafkaHWMapper:: 78 | setNewConsumerTopicCallback() 79 | end note 80 | detach 81 | 82 | end split 83 | 84 | 85 | fork 86 | partition kafkaHWService::writeData() { 87 | start 88 | :Decode HWObject from WinCC; 89 | if(object is Kafka stream) then 90 | if(debouncing timeframe > 0?) then (yes, it's bigger than 0) 91 | :add stream to _streamMap; 92 | fork 93 | partition kafkaHWService::doStream() { 94 | start 95 | while (_streamMap entries to process?) is (yes) 96 | :retrieve entry; 97 | if(debouncing timeframe is over for entry?) then (yes) 98 | :remove entry from _streamMap 99 | and stream to kafka; 100 | (P) 101 | else (no) 102 | :skip entry; 103 | endif 104 | endwhile(no) 105 | stop 106 | } 107 | end fork 108 | else (no) 109 | ->no, stream to kafka; 110 | (P) 111 | endif; 112 | else if(object is Config item) then 113 | :Apply config; 114 | endif 115 | stop 116 | } 117 | fork again 118 | partition ConsumerThread{ 119 | start 120 | while(_consumerRun?) is (yes) 121 | (C) 122 | note right 123 | kafkaConsumerFacade.poll() 124 | end note 125 | while (polled messages to process?) is (yes) 126 | :kafkaHWService:: 127 | handleConsumeNewMessage; 128 | :add DPE to _toDPqueue; 129 | note right 130 | kafkaHWService::insertInDataToDP 131 | end note 132 | endwhile (no) 133 | endwhile (no) 134 | stop 135 | } 136 | partition kafkaHWService::workProc() { 137 | start 138 | while (_toDPqueue items to process?) is (yes) 139 | if (item addressed in the driver?) then (yes) 140 | :push DPE entry to WinCC; 141 | note right 142 | DrvManager::getSelfPtr()->toDp 143 | end note 144 | endif 145 | endwhile (no) 146 | stop 147 | } 148 | 149 | end fork 150 | 151 | 152 | end 153 | @enduml 154 | -------------------------------------------------------------------------------- /doc/kafkaDemoProjectDPEs.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cern-hse-computing/WCCOAkafkaDrv/a37d333d8148a4df5ac4c33d01de0605b6ca5e8a/doc/kafkaDemoProjectDPEs.gif -------------------------------------------------------------------------------- /kafkaConsumerFacade.cxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #include 16 | 17 | #include "kafkaConsumerFacade.hxx" 18 | #include "Common/Constants.hxx" 19 | #include "Common/Logger.hxx" 20 | 21 | #include "cppkafka/message.h" 22 | #include "cppkafka/kafka_handle_base.h" 23 | #include "cppkafka/error.h" 24 | #include "cppkafka/configuration.h" 25 | #include "cppkafka/topic.h" 26 | #include "cppkafka/consumer.h" 27 | #include "cppkafka/error.h" 28 | 29 | #include 30 | 31 | 32 | using cppkafka::Consumer; 33 | using cppkafka::Message; 34 | using cppkafka::Error; 35 | using cppkafka::Configuration; 36 | using cppkafka::KafkaHandleBase; 37 | using cppkafka::ConfigOptionNotFound; 38 | 39 | kafkaConsumerFacade::kafkaConsumerFacade(const std::string& topic, consumeCallbackConsumer cb, errorCallbackConsumer erc = nullptr, statsCallbackConsumer sc = nullptr) 40 | : _topic(topic), _consumeCB(cb), _errorCB(erc), _statsCB(sc) 41 | { 42 | 43 | try{ 44 | //Create kafka configuration 45 | Configuration config; 46 | for (const auto & kv : Common::Constants::GetConsumerConfig()) 47 | { 48 | std::cerr << kv.first << " - "<< kv.second << std::endl; 49 | config.set(kv.first, kv.second); 50 | 51 | } 52 | //Update consumer group id to be unique by appending topic and current timestamp 53 | try 54 | { 55 | config.set(Common::Constants::GROUP_ID_KEYWORD, config.get(Common::Constants::GROUP_ID_KEYWORD) + "_" + topic + "_" + std::to_string(std::chrono::seconds(std::time(NULL)).count())); 56 | } 57 | catch (ConfigOptionNotFound& e) 58 | { 59 | Common::Logger::globalWarning("No group id found! Adding default one: topicname_unixtimestamp", e.what()); 60 | config.set(Common::Constants::GROUP_ID_KEYWORD, topic + "_" + std::to_string(std::chrono::seconds(std::time(NULL)).count())); 61 | } 62 | 63 | // set the error and statistics callbacks on the kafka config 64 | if(_errorCB) 65 | config.set_error_callback([this](KafkaHandleBase& handle, int error , const std::string& reason) 66 | { 67 | this->_errorCB(_topic, error, reason); 68 | }); 69 | if(_statsCB) 70 | config.set_stats_callback([this](KafkaHandleBase& handle, const std::string& json) 71 | { 72 | this->_statsCB(_topic, json); 73 | }); 74 | 75 | // Create Consumer 76 | _consumer.reset(new Consumer(config)); 77 | _consumer->subscribe({_topic}); 78 | 79 | _initialized = true; 80 | } 81 | catch(std::exception& e) 82 | { 83 | Common::Logger::globalWarning("Unable to initialize consumer!", e.what()); 84 | } 85 | } 86 | 87 | void kafkaConsumerFacade::poll(const size_t& maxPollRecords) 88 | { 89 | Common::Logger::globalInfo(Common::Logger::L2, __PRETTY_FUNCTION__, "Poll batch with maxPollRecords:", CharString(maxPollRecords)); 90 | std::vector msgVec = std::move(_consumer->poll_batch(maxPollRecords)); 91 | for (auto & msg : msgVec ) 92 | { 93 | if (msg) { 94 | // If we managed to get a message 95 | if (msg.get_error()) { 96 | // Ignore EOF notifications from rdkafka 97 | if (!msg.is_eof()) { 98 | this->_errorCB(_topic, msg.get_error().get_error(), msg.get_error().to_string()); 99 | } 100 | } 101 | else { 102 | this->_consumeCB(msg.get_topic(), msg.get_key(), msg.get_payload()); 103 | //_consumer->commit(msg); 104 | } 105 | } 106 | } 107 | } 108 | 109 | -------------------------------------------------------------------------------- /kafkaConsumerFacade.hxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #ifndef KAFKACONSUMERFACADE_HXX 16 | #define KAFKACONSUMERFACADE_HXX 17 | 18 | #include 19 | 20 | #include 21 | #include 22 | 23 | 24 | #include 25 | #include 26 | 27 | using cppkafka::ConsumerDispatcher; 28 | using cppkafka::Consumer; 29 | 30 | using statsCallbackConsumer = std::function; 31 | using errorCallbackConsumer = std::function; 32 | using consumeCallbackConsumer = std::function; 33 | 34 | 35 | /** 36 | * @brief The kafkaConsumerFacade class is a facade and encompasses all the consumer interaction with librdkafka and cppkafka 37 | */ 38 | 39 | class kafkaConsumerFacade 40 | { 41 | public: 42 | /** 43 | * @brief kafkaConsumerFacade constructor 44 | * @param topic : the topic name 45 | * @param consumeCallbackConsumer : a callback that will be called for eached polled message 46 | * @param errorCallbackConsumer : an error callback for underlying kafka errors 47 | * @param statsCallbackConsumer : a callback for handling consumer statistics 48 | */ 49 | kafkaConsumerFacade(const std::string& topic, consumeCallbackConsumer, errorCallbackConsumer, statsCallbackConsumer); 50 | 51 | kafkaConsumerFacade(const kafkaConsumerFacade&) = delete; 52 | kafkaConsumerFacade& operator=(const kafkaConsumerFacade&) = delete; 53 | 54 | bool isInitialized(){return _initialized;} 55 | void poll(const size_t&); 56 | private: 57 | std::unique_ptr _consumer; 58 | std::string _topic; 59 | consumeCallbackConsumer _consumeCB; 60 | errorCallbackConsumer _errorCB; 61 | statsCallbackConsumer _statsCB; 62 | bool _initialized{false}; 63 | }; 64 | 65 | #endif // KAFKACONSUMERFACADE_HXX 66 | 67 | -------------------------------------------------------------------------------- /kafkaDrv.cxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | #include 20 | 21 | //------------------------------------------------------------------------------------ 22 | 23 | void kafkaDrv::install_HWMapper() 24 | { 25 | hwMapper = new kafkaHWMapper; 26 | } 27 | 28 | //-------------------------------------------------------------------------------- 29 | 30 | void kafkaDrv::install_HWService() 31 | { 32 | hwService = new kafkaHWService; 33 | } 34 | 35 | //-------------------------------------------------------------------------------- 36 | 37 | HWObject * kafkaDrv::getHWObject() const 38 | { 39 | return new HWObject(); 40 | } 41 | 42 | //-------------------------------------------------------------------------------- 43 | 44 | void kafkaDrv::install_AlertService() 45 | { 46 | DrvManager::install_AlertService(); 47 | } 48 | -------------------------------------------------------------------------------- /kafkaDrv.hxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #ifndef kafkaDRV_H_ 16 | #define kafkaDRV_H_ 17 | 18 | #include 19 | 20 | class kafkaDrv : public DrvManager 21 | { 22 | public: 23 | virtual void install_HWMapper(); 24 | virtual void install_HWService(); 25 | virtual void install_AlertService(); 26 | 27 | virtual HWObject *getHWObject() const; 28 | }; 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /kafkaHWMapper.cxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #include "kafkaHWMapper.hxx" 16 | #include "Transformations/kafkaStringTrans.hxx" 17 | #include "Transformations/kafkaInt32Trans.hxx" 18 | #include "Transformations/kafkaInt64Trans.hxx" 19 | #include "Transformations/kafkaFloatTrans.hxx" 20 | #include "Transformations/kafkaBoolTrans.hxx" 21 | #include "Transformations/kafkaUint8Trans.hxx" 22 | #include "Transformations/kafkaTimeTrans.hxx" 23 | 24 | #include "Common/Logger.hxx" 25 | #include "Common/Utils.hxx" 26 | #include // DEBUG macros 27 | 28 | //-------------------------------------------------------------------------------- 29 | // We get new configs here. Create a new HW-Object on arrival and insert it. 30 | 31 | PVSSboolean kafkaHWMapper::addDpPa(DpIdentifier &dpId, PeriphAddr *confPtr) 32 | { 33 | // We don't use Subindices here, so its simple. 34 | // Otherwise we had to look if we already have a HWObject and adapt its length. 35 | 36 | Common::Logger::globalInfo(Common::Logger::L1,"addDpPa called for ", confPtr->getName().c_str()); 37 | Common::Logger::globalInfo(Common::Logger::L1,"addDpPa direction ", CharString(confPtr->getDirection())); 38 | 39 | 40 | // tell the config how we will transform data to/from the device 41 | // by installing a Transformation object into the PeriphAddr 42 | // In this template, the Transformation type was set via the 43 | // configuration panel (it is already set in the PeriphAddr) 44 | 45 | // TODO this really depends on your protocol and is therefore just an example 46 | // in this example we use the ones from Pbus, as those can be selected 47 | // with the SIM driver parametrization panel 48 | switch ((uint32_t)confPtr->getTransformationType()) { 49 | case TransUndefinedType: 50 | Common::Logger::globalInfo(Common::Logger::L3,"Undefined transformation" + CharString(confPtr->getTransformationType())); 51 | return HWMapper::addDpPa(dpId, confPtr); 52 | case kafkaDrvBoolTransType: 53 | Common::Logger::globalInfo(Common::Logger::L3,"Bool transformation"); 54 | confPtr->setTransform(new Transformations::kafkaBoolTrans); 55 | break; 56 | case kafkaDrvUint8TransType: 57 | Common::Logger::globalInfo(Common::Logger::L3,"Uint8 transformation"); 58 | confPtr->setTransform(new Transformations::kafkaUint8Trans); 59 | break; 60 | case kafkaDrvInt32TransType: 61 | Common::Logger::globalInfo(Common::Logger::L3,"Int32 transformation"); 62 | confPtr->setTransform(new Transformations::kafkaInt32Trans); 63 | break; 64 | case kafkaDrvInt64TransType: 65 | Common::Logger::globalInfo(Common::Logger::L3,"Int64 transformation"); 66 | confPtr->setTransform(new Transformations::kafkaInt64Trans); 67 | break; 68 | case kafkaDrvFloatTransType: 69 | Common::Logger::globalInfo(Common::Logger::L3,"Float transformation"); 70 | confPtr->setTransform(new Transformations::kafkaFloatTrans); 71 | break; 72 | case kafkaDrvStringTransType: 73 | Common::Logger::globalInfo(Common::Logger::L3,"String transformation"); 74 | confPtr->setTransform(new Transformations::kafkaStringTrans); 75 | break; 76 | case kafkaDrvTimeTransType: 77 | Common::Logger::globalInfo(Common::Logger::L3,"Time transformation"); 78 | confPtr->setTransform(new Transformations::kafkaTimeTrans); 79 | break; 80 | default: 81 | Common::Logger::globalError("kafkaHWMapper::addDpPa", CharString("Illegal transformation type ") + CharString((int) confPtr->getTransformationType())); 82 | return HWMapper::addDpPa(dpId, confPtr); 83 | } 84 | 85 | // First add the config, then the HW-Object 86 | if ( !HWMapper::addDpPa(dpId, confPtr) ) // FAILED !! 87 | return PVSS_FALSE; 88 | 89 | HWObject *hwObj = new HWObject; 90 | // Set Address and Subindex 91 | Common::Logger::globalInfo(Common::Logger::L3, "New Object", "name:" + confPtr->getName()); 92 | hwObj->setConnectionId(confPtr->getConnectionId()); 93 | hwObj->setAddress(confPtr->getName()); // Resolve the HW-Address, too 94 | 95 | // Set the data type. 96 | hwObj->setType(confPtr->getTransform()->isA()); 97 | 98 | // Set the len needed for data from _all_ subindices of this PVSS-Address. 99 | // Because we will deal with subix 0 only this is the Transformation::itemSize 100 | hwObj->setDlen(confPtr->getTransform()->itemSize()); 101 | //TODO - number of elements? 102 | // Add it to the list 103 | addHWObject(hwObj); 104 | 105 | if(confPtr->getDirection() == DIRECTION_IN) 106 | { 107 | std::vector streamOptions = Common::Utils::split(hwObj->getAddress().c_str()); 108 | if (streamOptions.size() == 2) // TOPIC + KEY 109 | { 110 | addTopic(streamOptions[0]); 111 | } 112 | } 113 | 114 | return PVSS_TRUE; 115 | } 116 | 117 | //-------------------------------------------------------------------------------- 118 | 119 | PVSSboolean kafkaHWMapper::clrDpPa(DpIdentifier &dpId, PeriphAddr *confPtr) 120 | { 121 | DEBUG_DRV_USR1("clrDpPa called for " << confPtr->getName()); 122 | 123 | // Find our HWObject via a template 124 | HWObject adrObj; 125 | adrObj.setAddress(confPtr->getName()); 126 | 127 | // Lookup HW-Object via the Name, not via the HW-Address 128 | // The class type isn't important here 129 | HWObject *hwObj = findHWAddr(&adrObj); 130 | 131 | if ( hwObj ) 132 | { 133 | // Object exists, remove it from the list and delete it. 134 | clrHWObject(hwObj); 135 | delete hwObj; 136 | } 137 | 138 | // Call function of base class to remove config 139 | return HWMapper::clrDpPa(dpId, confPtr); 140 | } 141 | 142 | void kafkaHWMapper::addTopic(const std::string &topic) 143 | { 144 | if(consumer_topics.find(topic) == consumer_topics.end()) 145 | { 146 | consumer_topics.insert(topic); 147 | if(_newConsumerTopicCB) 148 | { 149 | _newConsumerTopicCB(topic); 150 | } 151 | } 152 | } 153 | 154 | //-------------------------------------------------------------------------------- 155 | -------------------------------------------------------------------------------- /kafkaHWMapper.hxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | 16 | #ifndef kafkaHWMAPPER_H_ 17 | #define kafkaHWMAPPER_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | // TODO: Write here all the Transformation types, one for every transformation 24 | #define kafkaDrvBoolTransType (TransUserType) 25 | #define kafkaDrvUint8TransType (TransUserType + 1) 26 | #define kafkaDrvInt32TransType (TransUserType + 2) 27 | #define kafkaDrvInt64TransType (TransUserType + 3) 28 | #define kafkaDrvFloatTransType (TransUserType + 4) 29 | #define kafkaDrvStringTransType (TransUserType + 5) 30 | #define kafkaDrvTimeTransType (TransUserType + 6) 31 | 32 | 33 | using newConsumerTopicCallback = std::function; 34 | 35 | class kafkaHWMapper : public HWMapper 36 | { 37 | public: 38 | virtual PVSSboolean addDpPa(DpIdentifier &dpId, PeriphAddr *confPtr); 39 | virtual PVSSboolean clrDpPa(DpIdentifier &dpId, PeriphAddr *confPtr); 40 | 41 | void setNewConsumerTopicCallback(newConsumerTopicCallback cb) {_newConsumerTopicCB = cb;} 42 | const std::unordered_set& getConsumerTopics(){return consumer_topics;} 43 | 44 | private: 45 | void addTopic(const std::string& topic); 46 | 47 | std::unordered_set consumer_topics; 48 | newConsumerTopicCallback _newConsumerTopicCB = {nullptr}; 49 | 50 | enum Direction 51 | { 52 | DIRECTION_OUT = 1, 53 | DIRECTION_IN = 2 54 | }; 55 | }; 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /kafkaHWService.cxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #include 16 | 17 | #include 18 | #include // DEBUG macros 19 | #include "Common/Logger.hxx" 20 | #include "Common/Constants.hxx" 21 | #include "Common/Utils.hxx" 22 | #include "Common/AsyncRecurringTask.hxx" 23 | 24 | #include "kafkaHWMapper.hxx" 25 | #include "kafkaConsumerFacade.hxx" 26 | 27 | 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | static std::atomic _consumerRun{true}; 35 | 36 | //-------------------------------------------------------------------------------- 37 | // called after connect to data 38 | 39 | kafkaHWService::kafkaHWService() 40 | { 41 | signal(SIGSEGV, handleSegfault); 42 | } 43 | 44 | PVSSboolean kafkaHWService::initialize(int argc, char *argv[]) 45 | { 46 | // use this function to initialize internals 47 | // if you don't need it, you can safely remove the whole method 48 | Common::Logger::globalInfo(Common::Logger::L1,__PRETTY_FUNCTION__,"start"); 49 | 50 | //Constants (at this point, the config file has been read) 51 | Common::Constants::setProducerKafkaStatsInterval(); 52 | Common::Constants::setConsumerKafkaStatsInterval(); 53 | 54 | // add callback for new consumer topics 55 | static_cast(DrvManager::getHWMapperPtr())->setNewConsumerTopicCallback(_newConsumerTopicCB); 56 | 57 | Common::Logger::globalInfo(Common::Logger::L1,__PRETTY_FUNCTION__,"end"); 58 | // To stop driver return PVSS_FALSE 59 | return PVSS_TRUE; 60 | } 61 | 62 | // in order to get statistics, poll() has to be called in the producer facade. 63 | void kafkaHWService::doPoll() 64 | { 65 | Common::Logger::globalInfo(Common::Logger::L3,__PRETTY_FUNCTION__,"Calling poll"); 66 | this->_producerFacade->poll(); 67 | } 68 | 69 | void kafkaHWService::doStream() 70 | { 71 | Common::Logger::globalInfo(Common::Logger::L4,__PRETTY_FUNCTION__,"Calling doStream"); 72 | auto now = std::chrono::high_resolution_clock::now(); 73 | for(auto& kv : _streamMap) 74 | { 75 | if(kv.second.JSONvalue.length()>0) 76 | { 77 | // Check debouncing time 78 | auto millis = std::chrono::duration_cast(now - kv.second.time).count(); 79 | if(millis >= kv.second.debouncing_timeframe) 80 | { 81 | streamStruct s ; 82 | { //lock_guard scope 83 | std::lock_guard lck(_streamMutex); 84 | s = std::move(kv.second); 85 | } 86 | Common::Logger::globalInfo(Common::Logger::L2,"",(s.topic+"$"+s.key).c_str(), s.JSONvalue.c_str()); 87 | _producerFacade->stream(s.topic,/* partition,*/ s.key,std::move(s.JSONvalue)); 88 | } 89 | } 90 | 91 | } 92 | } 93 | 94 | void kafkaHWService::handleProducerConfigError(int code, const std::string& str) 95 | { 96 | Common::Logger::globalWarning(__PRETTY_FUNCTION__,CharString(code),str.c_str()); 97 | if(code == -187) 98 | { 99 | _brokersDown = true; 100 | _producerFacade->setProducerSuccessCallback(_producerSuccessCB); 101 | insertInDataToDp(PRODUCER_ALL_BROKERS_DOWN_DP, "TRUE"); 102 | } 103 | } 104 | 105 | 106 | void kafkaHWService::handleProducerConfigStats(const std::string& jsonStats) 107 | { 108 | Common::Logger::globalInfo(Common::Logger::L1, __PRETTY_FUNCTION__,"Statistics: ",jsonStats.c_str()); 109 | insertInDataToDp(PRODUCER_STATS_DP,CharString(jsonStats.c_str(),jsonStats.length())); 110 | } 111 | 112 | 113 | void kafkaHWService::handleProducerSuccess(const std::string& message) 114 | { 115 | Common::Logger::globalInfo(Common::Logger::L3,__PRETTY_FUNCTION__, message.c_str()); 116 | if(_brokersDown) 117 | { 118 | _brokersDown = false; 119 | _producerFacade->setProducerSuccessCallback(nullptr); 120 | insertInDataToDp(PRODUCER_ALL_BROKERS_DOWN_DP, "FALSE"); 121 | } 122 | } 123 | 124 | 125 | 126 | void kafkaHWService::handleConsumerConfigError(const std::string& topic, int code, const std::string& str) 127 | { 128 | Common::Logger::globalWarning(__PRETTY_FUNCTION__, CharString(topic.c_str(), topic.length()), str.c_str()); 129 | } 130 | 131 | void kafkaHWService::handleConsumerConfigStats(const std::string&topic, const std::string& jsonStats) 132 | { 133 | Common::Logger::globalInfo(Common::Logger::L1, __PRETTY_FUNCTION__,topic.c_str(), jsonStats.c_str()); 134 | 135 | insertInDataToDp(CONSUMER_STATS_DP,CharString(jsonStats.c_str(),jsonStats.length())); 136 | //insertInDataToDp(CharString(CONSUMER_STATS_DP_BASE)+CharString(topic.c_str(), topic.length()),CharString(jsonStats.c_str(),jsonStats.length())); //TODO - consumer logs per topic 137 | } 138 | 139 | void kafkaHWService::handleConsumeNewMessage(const std::string& topic, const std::string& key, std::string&& payload ) 140 | { 141 | Common::Logger::globalInfo(Common::Logger::L3, __PRETTY_FUNCTION__, (topic + ":" + key + ":" + payload).c_str()); 142 | insertInDataToDp(std::move(CharString((topic+"$"+key).c_str())),std::move(CharString(payload.c_str(),payload.length()))); 143 | } 144 | 145 | void kafkaHWService::handleNewConsumerTopic(const std::string& topic) 146 | { 147 | Common::Logger::globalInfo(Common::Logger::L1,__PRETTY_FUNCTION__, "New topic:", topic.c_str()); 148 | _consumerThreads.emplace_back( 149 | [&] 150 | { 151 | 152 | kafkaConsumerFacade aConsumerFacade(topic,this->_configConsumeCB, this->_configErrorConsumerCB, this->_configStatsConsumerCB); 153 | if(!aConsumerFacade.isInitialized()) 154 | { 155 | Common::Logger::globalError("Unable to initialize topic:", topic.c_str()); 156 | } 157 | else 158 | { 159 | while(_consumerRun) 160 | { 161 | aConsumerFacade.poll(Common::Constants::getConsumerMaxPollRecords()); 162 | } 163 | } 164 | 165 | }); 166 | Common::Logger::globalInfo(Common::Logger::L1,__PRETTY_FUNCTION__, "New topic done:", topic.c_str()); 167 | } 168 | 169 | //-------------------------------------------------------------------------------- 170 | // called after connect to event 171 | 172 | PVSSboolean kafkaHWService::start() 173 | { 174 | // use this function to start your hardware activity. 175 | Common::Logger::globalInfo(Common::Logger::L1,__PRETTY_FUNCTION__); 176 | 177 | // See if we need to launch a producer 178 | if(Common::Constants::hasProducerConfig()) 179 | { 180 | // kafka Callbacks 181 | _producerFacade.reset(new kafkaProducerFacade(_configErrorProducerCB, _configStatsProducerCB)); 182 | if(!_producerFacade) 183 | { 184 | Common::Logger::globalError("kafkaHWService::initialize Unable to intialize the producer!"); 185 | return PVSS_FALSE; 186 | } 187 | 188 | // Async Recurring tasks 189 | _poolAsyncTask.reset(new Common::AsyncRecurringTask>(std::move([this]() { this->doPoll(); }), Common::Constants::getProducerKafkaStatsInterval())); 190 | _streamAsyncTask.reset(new Common::AsyncRecurringTask>(std::move([this]() { this->doStream(); }) , Common::Constants::getDebouncingThreadInterval())); 191 | } 192 | else 193 | { 194 | Common::Logger::globalInfo(Common::Logger::L1,__PRETTY_FUNCTION__,"No producer configuration provided"); 195 | } 196 | 197 | // Check if we need to launch consumer(s) 198 | // This list is automatically built by exisiting addresses sent at driver startup 199 | // new top 200 | for (const auto& topic : static_cast(DrvManager::getHWMapperPtr())->getConsumerTopics() ) 201 | { 202 | this->handleNewConsumerTopic(topic); 203 | } 204 | 205 | return PVSS_TRUE; 206 | } 207 | 208 | //-------------------------------------------------------------------------------- 209 | 210 | void kafkaHWService::stop() 211 | { 212 | // use this function to stop your hardware activity. 213 | Common::Logger::globalInfo(Common::Logger::L1,__PRETTY_FUNCTION__,"Stop"); 214 | _consumerRun = false; 215 | for(auto& ct : _consumerThreads) 216 | { 217 | if(ct.joinable()) 218 | ct.join(); 219 | } 220 | } 221 | 222 | //-------------------------------------------------------------------------------- 223 | 224 | void kafkaHWService::workProc() 225 | { 226 | HWObject obj; 227 | 228 | // TODO somehow receive a message from your device 229 | std::lock_guard lock{_toDPmutex}; 230 | Common::Logger::globalInfo(Common::Logger::L3,__PRETTY_FUNCTION__,"Size", CharString(_toDPqueue.size())); 231 | while (!_toDPqueue.empty()) 232 | { 233 | auto pair = std::move(_toDPqueue.front()); 234 | _toDPqueue.pop(); 235 | 236 | 237 | obj.setAddress(pair.first); 238 | 239 | 240 | // // a chance to see what's happening 241 | // if ( Resources::isDbgFlag(Resources::DBG_DRV_USR1) ) 242 | // obj.debugPrint(); 243 | 244 | // find the HWObject via the periphery address in the HWObject list, 245 | HWObject *addrObj = DrvManager::getHWMapperPtr()->findHWObject(&obj); 246 | 247 | // ok, we found it; now send to the DPEs 248 | if ( addrObj ) 249 | { 250 | Common::Logger::globalInfo(Common::Logger::L2,__PRETTY_FUNCTION__, pair.first, pair.second); 251 | //addrObj->debugPrint(); 252 | obj.setOrgTime(TimeVar()); // current time 253 | obj.setDlen(pair.second.len()); // data 254 | obj.setData((PVSSchar*)pair.second.cutCharPtr()); 255 | 256 | DrvManager::getSelfPtr()->toDp(&obj, addrObj); 257 | } 258 | } 259 | } 260 | 261 | 262 | void kafkaHWService::insertInDataToDp(CharString&& address, CharString&& value) 263 | { 264 | std::lock_guard lock{_toDPmutex}; 265 | _toDPqueue.push(std::move(std::make_pair(std::move(address),std::move(value)))); 266 | } 267 | 268 | //-------------------------------------------------------------------------------- 269 | // we get data from PVSS and shall send it to the periphery 270 | 271 | PVSSboolean kafkaHWService::writeData(HWObject *objPtr) 272 | { 273 | // Common::Logger::globalInfo(Common::Logger::L2,__PRETTY_FUNCTION__,"Incoming obj address",objPtr->getAddress()); 274 | 275 | std::vector streamOptions = Common::Utils::split(objPtr->getAddress().c_str()); 276 | 277 | // CONFIG DPs have just 1 278 | if(streamOptions.size() == 1) 279 | { 280 | try 281 | { 282 | Common::Logger::globalInfo(Common::Logger::L2,"Incoming CONFIG address",objPtr->getAddress(), objPtr->getInfo() ); 283 | Common::Constants::GetParseMap().at(std::string(objPtr->getAddress().c_str()))((const char*)objPtr->getData()); 284 | } 285 | catch (std::exception& e) 286 | { 287 | Common::Logger::globalWarning(__PRETTY_FUNCTION__," No configuration handling for address:", objPtr->getAddress().c_str()); 288 | } 289 | } 290 | else if (streamOptions.size() == STREAM_OPTIONS_SIZE || streamOptions.size() == STREAM_OPTIONS_SIZE - 1) // streaming 291 | { 292 | 293 | if(!streamOptions[STREAM_OPTIONS_TOPIC].length()) 294 | { 295 | Common::Logger::globalWarning(__PRETTY_FUNCTION__," Empty topic name"); 296 | return PVSS_FALSE; 297 | } 298 | int debouncing_timeframe = 0; // default value is 0 for debouncing timeframe 299 | if(streamOptions.size() == STREAM_OPTIONS_SIZE && !Common::Utils::convertToInt(streamOptions[STREAM_OPTIONS_DEBOUNCING_TIMEFRAME], debouncing_timeframe)) 300 | { 301 | Common::Logger::globalWarning(__PRETTY_FUNCTION__," Cannot parse debouncing timeframe", streamOptions[STREAM_OPTIONS_DEBOUNCING_TIMEFRAME].c_str()); 302 | return PVSS_FALSE; 303 | } 304 | // int partition = -1; 305 | // if(!Common::Utils::convertToInt(streamOptions[STREAM_OPTIONS_PARTITION], partition)) 306 | // { 307 | // Common::Logger::globalWarning(__PRETTY_FUNCTION__," Cannot parse partition", streamOptions[STREAM_OPTIONS_PARTITION].c_str()); 308 | // return PVSS_FALSE; 309 | // } 310 | 311 | if(!debouncing_timeframe) // no debouncing timeframe, stream directly 312 | { 313 | _producerFacade->stream(streamOptions[STREAM_OPTIONS_TOPIC],streamOptions[STREAM_OPTIONS_KEY], /* partition,*/std::move(std::string((char*)objPtr->getDataPtr()))); 314 | Common::Logger::globalInfo(Common::Logger::L2,"",objPtr->getAddress(), (char*)objPtr->getDataPtr() ); 315 | 316 | } 317 | else // debouncing time frame, add it to stream map. The async task doStream will take care of streaming 318 | { 319 | std::lock_guard lck(_streamMutex); 320 | _streamMap[std::string(objPtr->getAddress())] = {std::chrono::high_resolution_clock::now(), 321 | streamOptions[STREAM_OPTIONS_TOPIC], 322 | streamOptions[STREAM_OPTIONS_KEY], 323 | debouncing_timeframe, 324 | std::move(std::string((char*)objPtr->getDataPtr()/*, objPtr->getDlen()*/)) 325 | }; 326 | Common::Logger::globalInfo(Common::Logger::L2,"",objPtr->getAddress(), (char*)objPtr->getDataPtr() ); 327 | 328 | } 329 | 330 | // _producerFacade->stream(streamOptions[STREAM_OPTIONS_TOPIC],/* partition,*/ streamOptions[STREAM_OPTIONS_KEY] ,std::move(std::string((char *)objPtr->getDataPtr(), objPtr->getDlen()))); 331 | } 332 | else 333 | { 334 | Common::Logger::globalWarning(__PRETTY_FUNCTION__," Invalid stream options size for address: ", objPtr->getAddress().c_str()); 335 | } 336 | 337 | return PVSS_TRUE; 338 | } 339 | 340 | //-------------------------------------------------------------------------------- 341 | 342 | 343 | void handleSegfault(int signal_code){ 344 | void *array[50]; 345 | size_t size; 346 | 347 | // get void*'s for all entries on the stack 348 | size = backtrace(array, 50); 349 | 350 | // print out all the frames to stderr 351 | fprintf(stderr, "Error: signal %d:\n", signal_code); 352 | Common::Logger::globalWarning("kafkaHWService suffered a segmentation fault, code " + CharString(signal_code)); 353 | backtrace_symbols_fd(array, size, STDERR_FILENO); 354 | 355 | // restore and trigger default handle (to get the core dump) 356 | signal(signal_code, SIG_DFL); 357 | kill(getpid(), signal_code); 358 | 359 | exit(1); 360 | } 361 | -------------------------------------------------------------------------------- /kafkaHWService.hxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | 16 | #ifndef kafkaHWSERVICE_H_ 17 | #define kafkaHWSERVICE_H_ 18 | 19 | #include 20 | #include 21 | #include "kafkaProducerFacade.hxx" 22 | #include "kafkaConsumerFacade.hxx" 23 | 24 | #include "Common/Logger.hxx" 25 | #include "Common/AsyncRecurringTask.hxx" 26 | #include 27 | #include 28 | #include 29 | 30 | class kafkaProducerFacade; 31 | 32 | class kafkaHWService : public HWService 33 | { 34 | public: 35 | kafkaHWService(); 36 | virtual PVSSboolean initialize(int argc, char *argv[]); 37 | virtual PVSSboolean start(); 38 | virtual void stop(); 39 | virtual void workProc(); 40 | virtual PVSSboolean writeData(HWObject *objPtr); 41 | 42 | 43 | 44 | private: 45 | //Producer related 46 | std::unique_ptr _producerFacade{nullptr}; 47 | void handleProducerConfigError(int, const std::string&); 48 | void handleProducerConfigStats(const std::string&); 49 | void handleProducerSuccess(const std::string&); 50 | void doPoll(); 51 | //TODO handleQueueFullError 52 | successCallback _producerSuccessCB{[this](const std::string& str){this->handleProducerSuccess(str);}}; 53 | errorCallback _configErrorProducerCB{[this](int err, const string& reason) { this->handleProducerConfigError(err, reason);}}; 54 | statsCallback _configStatsProducerCB{[this](const std::string& json){this->handleProducerConfigStats(json);}}; 55 | 56 | 57 | // Consumer related 58 | void handleConsumerConfigError(const std::string&, int, const std::string&); 59 | void handleConsumerConfigStats(const std::string&, const std::string&); 60 | void handleConsumeNewMessage(const std::string&, const std::string&, std::string&&); 61 | void handleNewConsumerTopic(const std::string& topic); 62 | 63 | errorCallbackConsumer _configErrorConsumerCB{[this](const std::string& topic, int err, const string& reason) { this->handleConsumerConfigError(topic, err, reason);}}; 64 | statsCallbackConsumer _configStatsConsumerCB{[this](const std::string& topic, const std::string& json){this->handleConsumerConfigStats(topic, json);}}; 65 | consumeCallbackConsumer _configConsumeCB{[this](const std::string& topic, const std::string& key, std::string&& payload){this->handleConsumeNewMessage(topic, key, std::move(payload));}}; 66 | std::function _newConsumerTopicCB{[this](const std::string& topic){this->handleNewConsumerTopic(topic);}}; 67 | 68 | 69 | //Common 70 | void insertInDataToDp(CharString&& address, CharString&& value); 71 | std::mutex _toDPmutex; 72 | 73 | std::queue> _toDPqueue; 74 | std::atomic _brokersDown{false}; 75 | 76 | enum 77 | { 78 | STREAM_OPTIONS_TOPIC = 0, 79 | STREAM_OPTIONS_KEY, 80 | STREAM_OPTIONS_DEBOUNCING_TIMEFRAME, 81 | //STREAM_OPTIONS_PARTITION, 82 | STREAM_OPTIONS_SIZE 83 | } STREAM_OPTIONS; 84 | 85 | constexpr static const char* PRODUCER_ALL_BROKERS_DOWN_DP = "PRODUCER_ALL_BROKERS_DOWN"; // Used to signal if all brokers are down (due to different reasons, i.e. configuration is not OK) 86 | constexpr static const char* NO_PRODUCER_CONFIG= "PRODUCER_NO_CONFIG"; // Usded to signal if producer config is absent and the driver is addressed for producing messages 87 | constexpr static const char* PRODUCER_STATS_DP= "PRODUCER_STATISTICS"; 88 | 89 | 90 | constexpr static const char* NO_CONSUMER_CONFIG= "CONSUMER_NO_CONFIG"; // Used to signal if consumer config is absent and the driver is addressed for consuming messages 91 | constexpr static const char* CONSUMER_STATS_DP = "CONSUMER_STATISTICS"; 92 | 93 | 94 | 95 | std::mutex _streamMutex; 96 | struct streamStruct 97 | { 98 | std::chrono::high_resolution_clock::time_point time; 99 | std::string topic; 100 | std::string key; 101 | int debouncing_timeframe; 102 | std::string JSONvalue; 103 | }; 104 | 105 | std::unordered_map _streamMap; 106 | std::unique_ptr>> _streamAsyncTask; 107 | std::unique_ptr>> _poolAsyncTask; 108 | void doStream(); 109 | 110 | std::vector _consumerThreads; 111 | }; 112 | 113 | 114 | void handleSegfault(int signal_code); 115 | 116 | #endif 117 | -------------------------------------------------------------------------------- /kafkaMain.cxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #include 16 | #include 17 | #include "Common/Logger.hxx" 18 | #include "Common/Constants.hxx" 19 | 20 | #include 21 | #include 22 | 23 | int main(int argc, char **argv) 24 | { 25 | // an instance of our own Resources class is needed 26 | kafkaResources& resources = kafkaResources::GetInstance(); 27 | 28 | resources.init(argc, argv); 29 | 30 | kafkaResources::setDriverType(Common::Constants::getDrvName().c_str()); 31 | 32 | // the user wants to know std. commandline options 33 | if ( resources.getHelpFlag() ) 34 | { 35 | resources.printHelp(); 36 | return 0; 37 | } 38 | 39 | // the user wants to know std. debug flags 40 | if ( resources.getHelpDbgFlag() ) 41 | { 42 | resources.printHelpDbg(); 43 | return 0; 44 | } 45 | 46 | // the user wants to know std. report flags 47 | if ( resources.getHelpReportFlag() ) 48 | { 49 | resources.printHelpReport(); 50 | return 0; 51 | } 52 | 53 | kafkaDrv *driver = nullptr; 54 | 55 | try{ 56 | 57 | 58 | // handle std. signals 59 | signal(SIGINT, Manager::sigHdl); 60 | signal(SIGTERM, Manager::sigHdl); 61 | 62 | // a pointer is needed, since the Manager dtor does a delete 63 | kafkaDrv *driver = new kafkaDrv; 64 | 65 | driver->mainProcedure(argc, argv); 66 | 67 | return 0; 68 | } 69 | catch(std::exception& e){ 70 | Common::Logger::globalWarning("Exception"); 71 | if(driver) 72 | driver->exit(0); 73 | Common::Logger::globalError(e.what()); 74 | } 75 | return 0; 76 | 77 | } 78 | -------------------------------------------------------------------------------- /kafkaProducerFacade.cxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #include "kafkaProducerFacade.hxx" 16 | #include "Common/Constants.hxx" 17 | #include "Common/Logger.hxx" 18 | 19 | #include "cppkafka/message.h" 20 | #include "cppkafka/kafka_handle_base.h" 21 | #include "cppkafka/error.h" 22 | #include "cppkafka/topic.h" 23 | #include "cppkafka/configuration.h" 24 | #include "cppkafka/producer.h" 25 | #include "cppkafka/utils/buffered_producer.h" 26 | 27 | 28 | #include 29 | 30 | using cppkafka::Message; 31 | using cppkafka::MessageBuilder; 32 | using cppkafka::Error; 33 | using cppkafka::KafkaHandleBase; 34 | using cppkafka::Configuration; 35 | using cppkafka::Exception; 36 | using cppkafka::Producer; 37 | using cppkafka::BufferedProducer; 38 | using cppkafka::MessageBuilder; 39 | 40 | 41 | 42 | kafkaProducerFacade::kafkaProducerFacade(errorCallback ecb=nullptr, statsCallback scb = nullptr) 43 | : _errorCB(ecb), _statsCB(scb) 44 | { 45 | Configuration config; 46 | for (const auto & kv : Common::Constants::GetProducerConfig()) 47 | { 48 | config.set(kv.first, kv.second); 49 | } 50 | if(_errorCB) 51 | config.set_error_callback([this](KafkaHandleBase& handle, int error , const std::string& reason) 52 | { 53 | this->_errorCB(error, reason); 54 | }); 55 | if(_statsCB) 56 | config.set_stats_callback([this](KafkaHandleBase& handle, const std::string& json) 57 | { 58 | this->_statsCB(json); 59 | }); 60 | _buffProducer.reset(new BufferedProducer(config)); 61 | _buffProducer->set_flush_method(BufferedProducer::FlushMethod::Async); 62 | } 63 | 64 | kafkaProducerFacade::~kafkaProducerFacade() 65 | { 66 | _buffProducer->async_flush(); 67 | } 68 | 69 | void kafkaProducerFacade::poll() 70 | { 71 | _buffProducer->get_producer().poll(); 72 | } 73 | 74 | void kafkaProducerFacade::setProducerSuccessCallback(successCallback cb) 75 | { 76 | if(cb) 77 | _buffProducer->set_produce_success_callback([cb](const Message& msg){ 78 | std::string str = msg.get_payload().operator std::string(); 79 | cb(str); 80 | }); 81 | else 82 | _buffProducer->set_produce_success_callback(nullptr); 83 | } 84 | 85 | void kafkaProducerFacade::stream(const std::string& topic, const std::string& key, std::string&& message) 86 | { 87 | // Actually produce the message we've built 88 | _buffProducer->produce(MessageBuilder(topic).key(key).payload(message)); 89 | } 90 | 91 | void kafkaProducerFacade::stream(const std::string& topic, int partition, const std::string& key, std::string&& message) 92 | { 93 | // Actually produce the message we've built 94 | _buffProducer->produce(MessageBuilder(topic).partition(partition).key(key).payload(message)); 95 | } 96 | -------------------------------------------------------------------------------- /kafkaProducerFacade.hxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | 16 | #ifndef KAFKAPRODUCERFACADE_HXX 17 | #define KAFKAPRODUCERFACADE_HXX 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | using std::string; 26 | 27 | 28 | using cppkafka::BufferedProducer; 29 | 30 | using statsCallback = std::function; 31 | using errorCallback = std::function; 32 | using successCallback = statsCallback; 33 | 34 | 35 | /** 36 | * @brief The kafkaProducerFacade class is a facade and encompasses all the producer interaction with librdkafka and cppkafka 37 | */ 38 | 39 | class kafkaProducerFacade 40 | { 41 | public: 42 | /** 43 | * @brief kafkaProducerFacade constructor 44 | * @param errorCallbackConsumer : an error callback for underlying kafka errors 45 | * @param statsCallbackConsumer : a callback for handling producer statistics 46 | */ 47 | kafkaProducerFacade(errorCallback, statsCallback); 48 | ~kafkaProducerFacade(); 49 | 50 | kafkaProducerFacade(const kafkaProducerFacade& ) = delete; 51 | kafkaProducerFacade& operator=(const kafkaProducerFacade& ) = delete; 52 | 53 | void stream(const std::string& topic, int partition, const std::string&, std::string&& message); 54 | void stream(const std::string& topic, const std::string&, std::string&& message); 55 | 56 | void setProducerSuccessCallback(successCallback); 57 | 58 | void poll(); 59 | 60 | private: 61 | std::unique_ptr> _buffProducer; 62 | errorCallback _errorCB; 63 | statsCallback _statsCB; 64 | }; 65 | 66 | 67 | #endif // KAFKAPRODUCERFACADE_HXX 68 | -------------------------------------------------------------------------------- /kafkaResources.cxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | // Our Resource class. 16 | // This class will interpret the command line and read the config file 17 | 18 | #include "kafkaResources.hxx" 19 | #include "Common/Logger.hxx" 20 | #include "Common/Constants.hxx" 21 | #include 22 | 23 | //------------------------------------------------------------------------------- 24 | // init is a wrapper around begin, readSection and end 25 | 26 | void kafkaResources::init(int &argc, char *argv[]) 27 | { 28 | // Prepass of commandline arguments, e.g. get the arguments needed to 29 | // find the config file. 30 | begin(argc, argv); 31 | 32 | // Read the config file 33 | while ( readSection() || generalSection() ) ; 34 | 35 | // Postpass of the commandline arguments, e.g. get the arguments that 36 | // will override entries in the config file 37 | end(argc, argv); 38 | } 39 | 40 | //------------------------------------------------------------------------------- 41 | 42 | PVSSboolean kafkaResources::readSection() 43 | { 44 | // Are we in our section ? 45 | if ( !isSection("kafka") ) return PVSS_FALSE; 46 | 47 | // skip "[kafka]" 48 | getNextEntry(); 49 | 50 | // Now read the section until new section or end of file 51 | try{ 52 | while ( (cfgState != CFG_SECT_START) && (cfgState != CFG_EOF) ) 53 | { 54 | // TODO whatever you have to read from the config file 55 | if( keyWord.startsWith(Common::Constants::PRODUCER_CONFIG_KEYWORD)) 56 | { 57 | std::string key = keyWord.substring(strlen(Common::Constants::PRODUCER_CONFIG_KEYWORD)).operator const char *(); 58 | std::string value; 59 | cfgStream >> value; 60 | Common::Constants::setProducerConfig(key, value); 61 | Common::Logger::globalInfo(Common::Logger::L1, " kafkaResources::readSection Added PRODUCER key/value:", key.c_str(), value.c_str()); 62 | } 63 | else if (keyWord.startsWith(Common::Constants::CONSUMER_CONFIG_KEYWORD)) 64 | { 65 | std::string key = keyWord.substring(strlen(Common::Constants::CONSUMER_CONFIG_KEYWORD)).operator const char *(); 66 | std::string value; 67 | cfgStream >> value; 68 | Common::Constants::setConsumerConfig(key, value); 69 | Common::Logger::globalInfo(Common::Logger::L1, " kafkaResources::readSection CONSUMER Added key/value:", key.c_str(), value.c_str()); 70 | } 71 | else if ( !commonKeyWord() ) 72 | { 73 | ErrHdl::error(ErrClass::PRIO_WARNING, // Not that bad 74 | ErrClass::ERR_PARAM, // Error is with config file, not with driver 75 | ErrClass::ILLEGAL_KEYWORD, // Illegal keyword in res. 76 | keyWord); 77 | 78 | // signal Error, so we stop later 79 | cfgError = PVSS_TRUE; 80 | } 81 | 82 | // get next entry 83 | getNextEntry(); 84 | } 85 | } 86 | catch(std::runtime_error& e){ 87 | Common::Logger::globalError(e.what()); 88 | return PVSS_FALSE; 89 | } 90 | // So the loop will stop at the end of the file 91 | return cfgState != CFG_EOF; 92 | } 93 | 94 | 95 | kafkaResources& kafkaResources::GetInstance() 96 | { 97 | static kafkaResources krs; 98 | return krs; 99 | } 100 | 101 | //------------------------------------------------------------------------------- 102 | // Interface to internal Datapoints 103 | // Get the number of names we need the DpId for 104 | 105 | int kafkaResources::getNumberOfDpNames() 106 | { 107 | // TODO if you use internal DPs 108 | return 0; 109 | } 110 | 111 | //------------------------------------------------------------------------------- 112 | -------------------------------------------------------------------------------- /kafkaResources.hxx: -------------------------------------------------------------------------------- 1 | /** © Copyright 2019 CERN 2 | * 3 | * This software is distributed under the terms of the 4 | * GNU Lesser General Public Licence version 3 (LGPL Version 3), 5 | * copied verbatim in the file “LICENSE” 6 | * 7 | * In applying this licence, CERN does not waive the privileges 8 | * and immunities granted to it by virtue of its status as an 9 | * Intergovernmental Organization or submit itself to any jurisdiction. 10 | * 11 | * Author: Alexandru Savulescu (HSE-CEN-CO) 12 | * 13 | **/ 14 | 15 | #ifndef kafkaRESOURCES_H_ 16 | #define kafkaRESOURCES_H_ 17 | 18 | // Our Resources class 19 | // This class has two tasks: 20 | // - Interpret commandline and read config file 21 | // - Be an interface to internal datapoints 22 | 23 | #include 24 | 25 | class kafkaResources : public DrvRsrce 26 | { 27 | public: 28 | 29 | static void init(int &argc, char *argv[]); // Initialize statics 30 | static PVSSboolean readSection(); // read config file 31 | static kafkaResources& GetInstance(); 32 | 33 | // Get the number of names we need the DpId for 34 | virtual int getNumberOfDpNames(); 35 | 36 | // TODO in this template we do not use internal DPs in the driver 37 | // If you need DPs, then also some other methods must be implemented 38 | kafkaResources(kafkaResources const&) = delete; 39 | void operator= (kafkaResources const&) = delete; 40 | private: 41 | kafkaResources(){} 42 | 43 | }; 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /winccoa/config/config.kafka: -------------------------------------------------------------------------------- 1 | [kafka] 2 | PRODUCER.CONFIG.metadata.broker.list = dbnile-kafka-a-8.cern.ch:9093,dbnile-kafka-b-8.cern.ch:9093,dbnile-kafka-c-8.cern.ch:9093 3 | PRODUCER.CONFIG.security.protocol = SASL_SSL 4 | PRODUCER.CONFIG.sasl.mechanism = GSSAPI 5 | PRODUCER.CONFIG.sasl.kerberos.service.name = kafka 6 | PRODUCER.CONFIG.sasl.kerberos.principal = [USERNAME] 7 | PRODUCER.CONFIG.sasl.kerberos.keytab = [BASE_PATH]/kafka_producer_consumer_demo/config/user.keytab 8 | PRODUCER.CONFIG.group.id = test-producer-group-demo 9 | PRODUCER.CONFIG.statistics.interval.ms = 600000 10 | 11 | CONSUMER.CONFIG.metadata.broker.list = dbnile-kafka-a-8.cern.ch:9093,dbnile-kafka-b-8.cern.ch:9093,dbnile-kafka-c-8.cern.ch:9093 12 | CONSUMER.CONFIG.security.protocol = SASL_SSL 13 | CONSUMER.CONFIG.sasl.mechanism = GSSAPI 14 | CONSUMER.CONFIG.sasl.kerberos.service.name = kafka 15 | CONSUMER.CONFIG.sasl.kerberos.principal = [USERNAME] 16 | CONSUMER.CONFIG.sasl.kerberos.keytab = [BASE_PATH]/kafka_producer_consumer_demo/config/user.keytab 17 | CONSUMER.CONFIG.statistics.interval.ms = 600000 18 | CONSUMER.CONFIG.group.id = test-consumer-group-demo 19 | CONSUMER.CONFIG.enable.auto.commit = true 20 | CONSUMER.CONFIG.auto.offset.reset = latest 21 | -------------------------------------------------------------------------------- /winccoa/config/user.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cern-hse-computing/WCCOAkafkaDrv/a37d333d8148a4df5ac4c33d01de0605b6ca5e8a/winccoa/config/user.keytab -------------------------------------------------------------------------------- /winccoa/dplist/kafka_driver_config.dpl: -------------------------------------------------------------------------------- 1 | # ascii dump of database 2 | 3 | # DpType 4 | TypeName 5 | CONFIG_KAFKA.CONFIG_KAFKA 1#1 6 | DebouncingThreadInterval 21#8 7 | DebugLvl 21#9 8 | IN 1#10 9 | ConsumerStatsDP 25#18 10 | NoConsumerConfig 23#20 11 | NoProducerConfig 23#21 12 | ProducerAllBrokersDown 25#16 13 | ProducerStatsDP 25#17 14 | MaxPollRecords 21#22 15 | 16 | # Datapoint/DpId 17 | DpName TypeName ID 18 | CONFIG_KAFKA CONFIG_KAFKA 7670665 19 | 20 | # Aliases/Comments 21 | AliasId AliasName CommentName 22 | CONFIG_KAFKA.DebouncingThreadInterval "" lt:1 LANG:10001 "Debouncing thread interval for stream out@@" 23 | CONFIG_KAFKA.DebugLvl "" lt:1 LANG:10001 "Debug Level@@" 24 | CONFIG_KAFKA.IN.ProducerAllBrokersDown "" lt:1 LANG:10001 "Producer All Brokers Down@@" 25 | CONFIG_KAFKA.IN.ProducerStatsDP "" lt:1 LANG:10001 "Producer Statisticss DP@@" 26 | CONFIG_KAFKA.IN.ConsumerStatsDP "" lt:1 LANG:10001 "Consumer Statisticss DP@@" 27 | CONFIG_KAFKA.IN.NoConsumerConfig "" lt:1 LANG:10001 "No consumer config@@" 28 | CONFIG_KAFKA.IN.NoProducerConfig "" lt:1 LANG:10001 "No producer config@@" 29 | CONFIG_KAFKA.MaxPollRecords "" lt:1 LANG:10001 "Max poll records@@" 30 | 31 | # DpValue 32 | Manager/User ElementName TypeName _original.._value _original.._status64 _original.._stime 33 | UI (1)/0 CONFIG_KAFKA.DebouncingThreadInterval CONFIG_KAFKA 50 0x8300000000000101 30.09.2021 13:37:31.179 34 | UI (1)/0 CONFIG_KAFKA.DebugLvl CONFIG_KAFKA 1 0x8300000000100101 08.10.2021 12:36:07.581 35 | ASC (1)/0 CONFIG_KAFKA.IN.ProducerAllBrokersDown CONFIG_KAFKA "0" 0x8300000000000101 29.09.2021 09:53:44.666 36 | UI (2)/0 CONFIG_KAFKA.IN.ProducerStatsDP CONFIG_KAFKA "" 0x8300000000100101 08.10.2021 12:40:32.134 37 | UI (2)/0 CONFIG_KAFKA.IN.ConsumerStatsDP CONFIG_KAFKA "" 0x8300000000100101 08.10.2021 12:40:24.980 38 | ASC (1)/0 CONFIG_KAFKA.IN.NoConsumerConfig CONFIG_KAFKA 0 0x8300000000000101 29.09.2021 09:53:44.666 39 | ASC (1)/0 CONFIG_KAFKA.IN.NoProducerConfig CONFIG_KAFKA 0 0x8300000000000101 29.09.2021 09:53:44.666 40 | UI (1)/0 CONFIG_KAFKA.MaxPollRecords CONFIG_KAFKA 100 0x8300000000300101 30.09.2021 13:39:42.283 41 | 42 | # DistributionInfo 43 | Manager/User ElementName TypeName _distrib.._type _distrib.._driver 44 | UI (1)/0 CONFIG_KAFKA.DebouncingThreadInterval CONFIG_KAFKA 56 \2 45 | UI (1)/0 CONFIG_KAFKA.DebugLvl CONFIG_KAFKA 56 \2 46 | UI (1)/0 CONFIG_KAFKA.IN.ProducerAllBrokersDown CONFIG_KAFKA 56 \2 47 | UI (1)/0 CONFIG_KAFKA.IN.ProducerStatsDP CONFIG_KAFKA 56 \2 48 | UI (1)/0 CONFIG_KAFKA.IN.ConsumerStatsDP CONFIG_KAFKA 56 \2 49 | UI (1)/0 CONFIG_KAFKA.IN.NoConsumerConfig CONFIG_KAFKA 56 \2 50 | UI (1)/0 CONFIG_KAFKA.IN.NoProducerConfig CONFIG_KAFKA 56 \2 51 | UI (1)/0 CONFIG_KAFKA.MaxPollRecords CONFIG_KAFKA 56 \2 52 | 53 | # PeriphAddrMain 54 | Manager/User ElementName TypeName _address.._type _address.._reference _address.._poll_group _address.._connection _address.._offset _address.._subindex _address.._direction _address.._internal _address.._lowlevel _address.._active _address.._start _address.._interval _address.._reply _address.._datatype _address.._drv_ident 55 | UI (1)/0 CONFIG_KAFKA.DebouncingThreadInterval CONFIG_KAFKA 16 "DEBOUNCINGTHREADINTERVAL" 0 0 \1 0 0 1 01.01.1970 00:00:00.000 01.01.1970 00:00:00.000 01.01.1970 00:00:00.000 1005 "KAFKA" 56 | UI (1)/0 CONFIG_KAFKA.DebugLvl CONFIG_KAFKA 16 "DEBUGLVL" 0 0 \1 0 0 1 01.01.1970 00:00:00.000 01.01.1970 00:00:00.000 01.01.1970 00:00:00.000 1005 "KAFKA" 57 | UI (1)/0 CONFIG_KAFKA.IN.ProducerAllBrokersDown CONFIG_KAFKA 16 "PRODUCER_ALL_BROKERS_DOWN" 0 0 \2 0 0 1 01.01.1970 00:00:00.000 01.01.1970 00:00:00.000 01.01.1970 00:00:00.000 1000 "KAFKA" 58 | UI (1)/0 CONFIG_KAFKA.IN.ProducerStatsDP CONFIG_KAFKA 16 "PRODUCER_STATISTICS" 0 0 \2 0 0 1 01.01.1970 00:00:00.000 01.01.1970 00:00:00.000 01.01.1970 00:00:00.000 1005 "KAFKA" 59 | UI (1)/0 CONFIG_KAFKA.IN.ConsumerStatsDP CONFIG_KAFKA 16 "CONSUMER_STATISTICS" 0 0 \2 0 0 1 01.01.1970 00:00:00.000 01.01.1970 00:00:00.000 01.01.1970 00:00:00.000 1005 "KAFKA" 60 | UI (1)/0 CONFIG_KAFKA.IN.NoConsumerConfig CONFIG_KAFKA 16 "CONSUMER_NO_CONFIG" 0 0 \2 0 0 1 01.01.1970 00:00:00.000 01.01.1970 00:00:00.000 01.01.1970 00:00:00.000 1000 "KAFKA" 61 | UI (1)/0 CONFIG_KAFKA.IN.NoProducerConfig CONFIG_KAFKA 16 "PRODUCER_NO_CONFIG" 0 0 \2 0 0 1 01.01.1970 00:00:00.000 01.01.1970 00:00:00.000 01.01.1970 00:00:00.000 1000 "KAFKA" 62 | UI (1)/0 CONFIG_KAFKA.MaxPollRecords CONFIG_KAFKA 16 "MAXPOLLRECORDS" 0 0 \1 0 0 1 01.01.1970 00:00:00.000 01.01.1970 00:00:00.000 01.01.1970 00:00:00.000 1005 "KAFKA" 63 | -------------------------------------------------------------------------------- /winccoa/panels/para/address_kafka.pnl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 620 480 8 | _3DFace 9 | 6 4.5 10 | True 11 | False 12 | 13 | 96.03287671232877 14 | None 15 | 16 | 17 | 49 | 50 | 51 | 52 | 53 | 0 54 | 55 | 29.5 6.5 56 | True 57 | True 58 | schwarz 59 | _Transparent 60 | 3 61 | 62 | 63 | 64 | AlignCenter 65 | Point 66 | _Transparent 67 | False 68 | [solid,oneColor,JoinMiter,CapButt,1] 69 | False 70 | [outline] 71 | 31.5 6.5 72 | 73 | arial,-1,18,5,50,0,0,0,0,0 74 | 75 | 76 | Periphery - Kafka 77 | 78 | 0 79 | 2 80 | False 81 | True 82 | True 83 | [0s,,,AlignLeft] 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | para/dpe.ref 92 | 308 10 93 | 1 0 0 1 -0.5 -5.5 94 | 0 95 | 96 | 97 | $DPE 98 | $1 99 | 100 | 101 | AlignCenter 102 | 103 | 104 | 105 | 106 | para/lock_unlock.ref 107 | 6 12 108 | 1 0 0 1 -0.5 -5.5 109 | 1 110 | 111 | 112 | $1 113 | $1 114 | 115 | 116 | AlignCenter 117 | 118 | 119 | 120 | 121 | 4 122 | 123 | 10 60 124 | True 125 | True 126 | _3DText 127 | _Transparent 128 | 4 129 | 130 | 131 | 132 | AlignCenter 133 | Point 134 | _Transparent 135 | False 136 | [solid,oneColor,JoinMiter,CapButt,1] 137 | False 138 | [outline] 139 | 1 0 0 1 -0.5 -5.5 140 | Normal 141 | 10 60 142 | 601 51 143 | 0 144 | True 145 | 146 | 147 | 148 | 149 | 5 150 | 151 | 29.5 44.5 152 | True 153 | True 154 | _3DText 155 | _3DFace 156 | 5 157 | 158 | 159 | 160 | AlignCenter 161 | Point 162 | _Transparent 163 | False 164 | [solid,oneColor,JoinMiter,CapButt,1] 165 | False 166 | [solid] 167 | 29.5 44.5 168 | 169 | arial,-1,13,5,50,0,0,0,0,0 170 | 171 | 172 | reference: Topic$Key[$Output Debouncing (ms)] 173 | 174 | 0 175 | 0 176 | False 177 | True 178 | True 179 | [0s,,,AlignLeft] 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 6 188 | 189 | 509.5 44.5 190 | True 191 | True 192 | _3DText 193 | _3DFace 194 | 6 195 | 196 | 197 | 198 | AlignCenter 199 | Point 200 | _Transparent 201 | False 202 | [solid,oneColor,JoinMiter,CapButt,1] 203 | False 204 | [solid] 205 | 509.5 44.5 206 | 207 | arial,-1,13,5,50,0,0,0,0,0 208 | 209 | 210 | driver number 211 | 212 | 0 213 | 0 214 | False 215 | True 216 | True 217 | [0s,,,AlignLeft] 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 7 226 | 227 | 10 200 228 | True 229 | True 230 | _3DText 231 | _Transparent 232 | 7 233 | 234 | 235 | 236 | AlignCenter 237 | Point 238 | _Transparent 239 | False 240 | [solid,oneColor,JoinMiter,CapButt,1] 241 | False 242 | [outline] 243 | 0.2675 0 0 0.655 6.824999999999999 -7.25 244 | Normal 245 | 10 200 246 | 601 101 247 | 0 248 | True 249 | 250 | 251 | 252 | 253 | 8 254 | 255 | 294.5 113.75 256 | True 257 | True 258 | _3DText 259 | _3DFace 260 | 8 261 | 262 | 263 | 264 | AlignCenter 265 | Point 266 | _Transparent 267 | False 268 | [solid,oneColor,JoinMiter,CapButt,1] 269 | False 270 | [solid] 271 | 29.5 113.75 272 | 273 | arial,-1,13,5,50,0,0,0,0,0 274 | 275 | 276 | direction 277 | 278 | 0 279 | 0 280 | False 281 | True 282 | True 283 | [0s,,,AlignLeft] 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 9 292 | 293 | 10 130 294 | True 295 | True 296 | _3DText 297 | _Transparent 298 | 9 299 | 300 | 301 | 302 | AlignCenter 303 | Point 304 | _Transparent 305 | False 306 | [solid,oneColor,JoinMiter,CapButt,1] 307 | False 308 | [outline] 309 | 0.4172222222222222 0 0 1 205.4944444444444 -5.5 310 | Normal 311 | 10 130 312 | 601 51 313 | 0 314 | True 315 | 316 | 317 | 318 | 319 | 10 320 | 321 | 229.5 114.5 322 | True 323 | True 324 | _3DText 325 | _3DFace 326 | 10 327 | 328 | 329 | 330 | AlignCenter 331 | Point 332 | _Transparent 333 | False 334 | [solid,oneColor,JoinMiter,CapButt,1] 335 | False 336 | [solid] 337 | 229.5 114.5 338 | 339 | arial,-1,13,5,50,0,0,0,0,0 340 | 341 | 342 | Transformation 343 | 344 | 0 345 | 0 346 | False 347 | True 348 | True 349 | [0s,,,AlignLeft] 350 | 351 | 352 | 353 | 354 | 355 | 356 | 357 | 11 358 | 359 | 161.4375 328.75 360 | True 361 | False 362 | _WindowText 363 | _3DFace 364 | 11 365 | 366 | 367 | 368 | AlignCenter 369 | Point 370 | _Transparent 371 | False 372 | [solid,oneColor,JoinMiter,CapButt,1] 373 | False 374 | [solid] 375 | 163.4375 330.75 376 | 377 | Arial,-1,13,5,40,0,0,0,0,0 378 | 379 | 380 | Subindex 381 | 382 | 0 383 | 2 384 | False 385 | True 386 | True 387 | [0s,,,AlignLeft] 388 | 389 | 390 | 391 | 392 | 12 393 | 394 | 164.5 381.3125 395 | True 396 | False 397 | _3DText 398 | _3DFace 399 | 12 400 | 401 | 402 | 403 | AlignCenter 404 | Point 405 | _Transparent 406 | False 407 | [solid,oneColor,JoinMiter,CapButt,1] 408 | False 409 | [solid] 410 | 164.5 381.3125 411 | 412 | arial,-1,13,5,50,0,0,0,0,0 413 | 414 | 415 | poll group 416 | 417 | 0 418 | 0 419 | False 420 | True 421 | True 422 | [0s,,,AlignLeft] 423 | 424 | 425 | 426 | 427 | 428 | 429 | 430 | 13 431 | 432 | 269.9999999999999 330.75 433 | True 434 | False 435 | _3DText 436 | _3DFace 437 | 13 438 | 439 | 440 | 441 | AlignCenter 442 | Point 443 | _Transparent 444 | False 445 | [solid,oneColor,JoinMiter,CapButt,1] 446 | False 447 | [solid] 448 | 26.99999999999994 330.75 449 | 450 | arial,-1,13,5,50,0,0,0,0,0 451 | 452 | 453 | input mode 454 | 455 | 0 456 | 0 457 | False 458 | True 459 | True 460 | [0s,,,AlignLeft] 461 | 462 | 463 | 464 | 465 | 466 | 467 | 468 | 14 469 | 470 | 336 391.125 471 | True 472 | False 473 | _3DText 474 | _3DFace 475 | 14 476 | 477 | 478 | 479 | AlignCenter 480 | Point 481 | 482 | arial,-1,13,5,50,0,0,0,0,0 483 | 484 | 337.5 392.625 485 | 151 30 486 | 487 | 488 | 489 | low level comparison 490 | 491 | False 492 | 493 | 494 | 495 | 496 | 497 | 498 | para/buttons.ref 499 | 220 440 500 | 1 0 0 1 -210 5 501 | 2 502 | AlignCenter 503 | 504 | 505 | 506 | 507 | 20 508 | 509 | 165 395.125 510 | True 511 | False 512 | _WindowText 513 | _Window 514 | 15 515 | 516 | 517 | 518 | AlignCenter 519 | Point 520 | 521 | Arial,-1,13,5,40,0,0,0,0,0 522 | 523 | 165.5 395.625 524 | 141 24 525 | Normal 526 | True 527 | [0s,,,AlignLeft] 528 | 529 | 530 | 531 | 532 | 23 533 | 534 | 29.5 347.25 535 | True 536 | False 537 | _3DText 538 | _3DFace 539 | 16 540 | 541 | 542 | 543 | AlignCenter 544 | Point 545 | 546 | arial,-1,13,5,50,0,0,0,0,0 547 | 548 | 30 348.75 549 | 121 71 550 | 551 | 552 | 553 | Spontaneous 554 | 555 | False 556 | 557 | 558 | 559 | Polling 560 | 561 | True 562 | 563 | 564 | 565 | Single query 566 | 567 | False 568 | 569 | 570 | 571 | 572 | 573 | 574 | 24 575 | 576 | 526.5 63.5 577 | True 578 | True 579 | _WindowText 580 | _Window 581 | 17 582 | 583 | 584 | 585 | AlignCenter 586 | Point 587 | 588 | arial,-1,13,5,50,0,0,0,0,0 589 | 590 | 529 65 591 | 57 24 592 | 1 593 | 256 594 | 1 595 | 1 596 | 597 | 598 | 599 | 600 | 25 601 | 602 | 29.5 133.2857142857143 603 | True 604 | True 605 | _3DText 606 | _3DFace 607 | 18 608 | 609 | 610 | 611 | AlignCenter 612 | Point 613 | 614 | arial,-1,13,5,50,0,0,0,0,0 615 | 616 | 30 134.25 617 | 121 46 618 | 619 | 620 | 621 | Output 622 | 623 | False 624 | 625 | 626 | 627 | Input 628 | 629 | True 630 | 631 | 632 | 633 | 634 | 650 | 651 | 652 | 653 | 654 | 26 655 | 656 | 227.5 132.5 657 | True 658 | True 659 | {0,0,0} 660 | _Window 661 | 19 662 | 663 | 664 | 665 | AlignCenter 666 | Point 667 | 668 | arial,-1,13,5,50,0,0,0,0,0 669 | 670 | 230 135 671 | 217 26 672 | 673 | 674 | 675 | Bool 676 | 677 | True 678 | 679 | 680 | 681 | Char 682 | 683 | False 684 | 685 | 686 | 687 | Int32 688 | 689 | False 690 | 691 | 692 | 693 | Int64 694 | 695 | False 696 | 697 | 698 | 699 | Float 700 | 701 | False 702 | 703 | 704 | 705 | String 706 | 707 | False 708 | 709 | 710 | 711 | Time 712 | 713 | False 714 | 715 | 716 | True 717 | 718 | 719 | 720 | 721 | 27 722 | 723 | 494.5 126.25 724 | True 725 | True 726 | _3DText 727 | _3DFace 728 | 20 729 | 730 | 731 | 732 | AlignCenter 733 | Point 734 | 735 | Arial,-1,13,5,40,0,0,0,0,0 736 | 737 | 495 126.75 738 | 91 31 739 | 740 | 741 | 742 | Active 743 | 744 | False 745 | 746 | 747 | 748 | 749 | 750 | 751 | 28 752 | 753 | 164.5 348.875 754 | True 755 | False 756 | _WindowText 757 | _Window 758 | 21 759 | 760 | 761 | 762 | AlignCenter 763 | Point 764 | 765 | Arial,-1,13,5,40,0,0,0,0,0 766 | 767 | 165 349.375 768 | 151 24 769 | 0 770 | 100 771 | 1 772 | 0 773 | 774 | 775 | 776 | 777 | 29 778 | 779 | 28.28571428571429 64.5 780 | True 781 | True 782 | _WindowText 783 | _Window 784 | 22 785 | 786 | 787 | 788 | AlignCenter 789 | Point 790 | 791 | Arial,-1,13,5,40,0,0,0,0,0 792 | 793 | 30 65 794 | 481 24 795 | Normal 796 | True 797 | [0s,,,AlignLeft] 798 | 799 | 800 | 801 | 802 | -------------------------------------------------------------------------------- /winccoa/scripts/libs/kafka_dpe_addressing.ctl: -------------------------------------------------------------------------------- 1 | /** 2 | * Library of functions used to address Kafaka Driver DPE 3 | * @file kafka_dpe_addressing.ctl 4 | * @author Adrien Ledeul 5 | * @date 08/10/2021 6 | * @modifications: 7 | * -[author] [date] [object] 8 | */ 9 | 10 | 11 | 12 | const unsigned kafkaAddress_MODE_OUT = 1; 13 | const unsigned kafkaAddress_MODE_IN = 2; 14 | 15 | const unsigned kafkaAddress_DATA_TYPE_BOOL = 1000; 16 | const unsigned kafkaAddress_DATA_TYPE_CHAR = 1001; 17 | const unsigned kafkaAddress_DATA_TYPE_INT32 = 1002; 18 | const unsigned kafkaAddress_DATA_TYPE_INT64 = 1003; 19 | const unsigned kafkaAddress_DATA_TYPE_FLOAT = 1004; 20 | const unsigned kafkaAddress_DATA_TYPE_STRING = 1005; 21 | const unsigned kafkaAddress_DATA_TYPE_TIME = 1006; 22 | 23 | private const unsigned kafkaAddress_TYPE = 1; 24 | private const unsigned kafkaAddress_DRIVER_NUMBER = 2; 25 | private const unsigned kafkaAddress_REFERENCE = 3; 26 | private const unsigned kafkaAddress_DIRECTION = 4; 27 | private const unsigned kafkaAddress_DATATYPE = 5; 28 | private const unsigned kafkaAddress_ACTIVE = 6; 29 | private const unsigned kafkaAddress_SUBINDEX = 7; 30 | 31 | 32 | /** 33 | * Called when a DPE connected to this driver is adressed (used to set the proper periph. address config) 34 | * @param dpe path to dpe to address (e.g. Consumer.valueDpe) 35 | * @param dataType Data Type (kafkaAddress_DATA_TYPE_*) 36 | * @param mode adressing mode ( kafkaAddress_MODE_IN or kafkaAddress_MODE_OUT) 37 | * @param driverNum driver manager number 38 | * @param topic Kafka topic 39 | * @param key Kafka key 40 | * @param debouncingMs Debouncing timeframe, in ms (used for kafkaAddress_MODE_OUT only) 41 | * @return 1 if OK, 0 if not 42 | */ 43 | 44 | public int kafkaAddress_addressDPE(string dpe, unsigned dataType, unsigned mode, unsigned driverNum, string topic, string key, unsigned debouncingMs = 0) 45 | { 46 | dyn_anytype params; 47 | try 48 | { 49 | params[kafkaAddress_DRIVER_NUMBER] = driverNum; 50 | params[kafkaAddress_DIRECTION]= mode; 51 | params[kafkaAddress_ACTIVE] = true; 52 | params[kafkaAddress_SUBINDEX] = 0; 53 | params[kafkaAddress_DATATYPE] = dataType; 54 | params[kafkaAddress_REFERENCE] = topic + "$" + key + (debouncingMs > 0 && mode==kafkaAddress_MODE_OUT ? ("$" + debouncingMs) : ""); 55 | kafkaAddress_setPeriphAddress(dpe, params); 56 | } 57 | catch 58 | { 59 | DebugN("Error: Uncaught exception in kafkaAddress_addressDPE: " + getLastException()); 60 | return 0; 61 | } 62 | return 1; 63 | } 64 | 65 | /** 66 | * Called when a DPE connected to this driver is adressed (used to set the proper periph. address config) 67 | * @param dpe path to dpe to address (e.g. Consumer.configDpe) 68 | * @param dataType Data Type (kafkaAddress_DATA_TYPE_*) 69 | * @param mode adressing mode ( kafkaAddress_MODE_IN or kafkaAddress_MODE_OUT) 70 | * @param driverNum driver manager number 71 | * @param mode adressing mode ( kafkaAddress_MODE_IN or kafkaAddress_MODE_OUT) 72 | * @param configName name of the config (e.g. CONSUMER_STATISTICS) 73 | * @return 1 if OK, 0 if not 74 | */ 75 | 76 | public int kafkaAddress_addressConfigDPE(string dpe, unsigned dataType, unsigned mode, unsigned driverNum, string configName) 77 | { 78 | dyn_anytype params; 79 | try 80 | { 81 | params[kafkaAddress_DRIVER_NUMBER] = driverNum; 82 | params[kafkaAddress_DIRECTION]= mode; 83 | params[kafkaAddress_ACTIVE] = true; 84 | params[kafkaAddress_SUBINDEX] = 0; 85 | params[kafkaAddress_DATATYPE] = dataType; 86 | params[kafkaAddress_REFERENCE] = configName; 87 | kafkaAddress_setPeriphAddress(dpe, params); 88 | } 89 | catch 90 | { 91 | DebugN("Error: Uncaught exception in kafkaAddress_addressConfigDPE: " + getLastException()); 92 | return 0; 93 | } 94 | return 1; 95 | } 96 | 97 | /** 98 | * Method setting addressing for KAFKA datapoints elements 99 | * @param datapoint element for which address will be set 100 | * @param configuration parameteres 101 | */ 102 | private void kafkaAddress_setPeriphAddress(string dpe, dyn_anytype configParameters){ 103 | 104 | int i = 1; 105 | dyn_string names; 106 | dyn_anytype values; 107 | 108 | dpSetWait(dpe + ":_distrib.._type", DPCONFIG_DISTRIBUTION_INFO, 109 | dpe + ":_distrib.._driver", configParameters[kafkaAddress_DRIVER_NUMBER] ); 110 | dyn_string errors = getLastError(); 111 | if(dynlen(errors) > 0){ 112 | throwError(errors); 113 | DebugN("Error: Could not create the distrib config"); 114 | return; 115 | } 116 | 117 | names[i] = dpe + ":_address.._type"; 118 | values[i++] = DPCONFIG_PERIPH_ADDR_MAIN; 119 | names[i] = dpe + ":_address.._drv_ident"; 120 | values[i++] = "KAFKA"; 121 | names[i] = dpe + ":_address.._reference"; 122 | values[i++] = configParameters[kafkaAddress_REFERENCE]; 123 | names[i] = dpe + ":_address.._mode"; 124 | values[i++] = configParameters[kafkaAddress_DIRECTION]; 125 | names[i] = dpe + ":_address.._datatype"; 126 | values[i++] = configParameters[kafkaAddress_DATATYPE]; 127 | names[i] = dpe + ":_address.._subindex"; 128 | values[i++] = configParameters[kafkaAddress_SUBINDEX]; 129 | 130 | dpSetWait(names, values); 131 | } 132 | -------------------------------------------------------------------------------- /winccoa/scripts/userDrivers.ctl: -------------------------------------------------------------------------------- 1 | ////////////////////////////////////////////////////////////////////////////// 2 | // If you want to include a new driver 3 | // 4 | // 1. In this script below 5 | // add new item to dds[i] for names, types and drivers respectively below 6 | // 7 | // 2. In panels/para 8 | // copy and change address_skeleton.pnl to create a new para-panel for 9 | // a new driver 10 | // panel name must be: address_newdrivertype.pnl 11 | // (in our example below: address_tstdrv1.pnl) 12 | // 13 | // IMPORTANT: don't change the script in the panel-attributes and the buttons! 14 | // 15 | // 3. In scripts/userPara.ctl 16 | // add new case selection ( in our example case "tstdrv1":) 17 | // into the next four functions 18 | // into scripts/userPara.ctl: 19 | // upDpGetAddress 20 | // upDpSetAddress 21 | // upWritePanelAllAddressAttributes 22 | // upReadPanelAllAddressAttributes 23 | // and write the appropriate commands 24 | // 25 | // global variable used for the _address.. -attributes is 26 | // anytype dpc; 27 | // dpc[1]=reference; 28 | // dpc[2]=subindex; 29 | // dpc[3]=mode; 30 | // dpc[4]=start; 31 | // dpc[5]=interval; 32 | // dpc[6]=reply; 33 | // dpc[7]=datatype; 34 | // dpc[8]=drv_ident; 35 | // dpc[9]=driver; 36 | // you dont't have to set all of them, use only the necessary elements! 37 | ////////////////////////////////////////////////////////////////////////////// 38 | // 39 | // be careful: always use the same number of driver elements 40 | // (e.g. dyn_strings, cases, etc.) 41 | // 42 | ////////////////////////////////////////////////////////////////////////////// 43 | // The examples in this script use a copy of panels/para/address_sim.pnl 44 | ////////////////////////////////////////////////////////////////////////////// 45 | 46 | ////////////////////////////////////////////////////////////////////////////// 47 | // fill the makeDynString with the new driver datas 48 | ////////////////////////////////////////////////////////////////////////////// 49 | dyn_dyn_string main() 50 | { 51 | dyn_dyn_string dds; 52 | 53 | // names 54 | // this text will be displayed in the driver type selection combobox in address.pnl 55 | // Example: 56 | dds[1]=makeDynString("KAFKA Driver"); 57 | 58 | 59 | // types 60 | // this text identifies the driver type. The panel name must have the name 61 | // "address_"+typename+".pnl" and must be in panels/para 62 | // Example: 63 | dds[2]=makeDynString("kafka"); 64 | 65 | // drivers 66 | // dds[3]=makeDynString(); 67 | // this text will be set in _address.._drv_ident 68 | // Example: 69 | dds[3]=makeDynString("KAFKA"); 70 | 71 | return (dds); 72 | } 73 | -------------------------------------------------------------------------------- /winccoa/scripts/userPara.ctl: -------------------------------------------------------------------------------- 1 | ////////////////////////////////////////////////////////////////////////////// 2 | // If you want to include a new driver: 3 | // 4 | // 1. In scripts/userDrivers.ctl 5 | // add new item to dds[i] for names, types and drivers respectively below 6 | // 7 | // 2. In panels/para 8 | // copy and change address_skeleton.pnl to create a new para-panel for 9 | // a new driver 10 | // panel name must be: address_newdrivertype.pnl 11 | // (in our example below: address_tstdrv1.pnl) 12 | // 13 | // IMPORTANT: don't change the script in the panel-attributes and the buttons! 14 | // 15 | // 3. In this script below 16 | // add new case selection ( in our example case "tstdrv1":) 17 | // into the next four functions 18 | // upDpGetAddress 19 | // upDpSetAddress 20 | // upWritePanelAllAddressAttributes 21 | // upReadPanelAllAddressAttributes 22 | // and write the appropriate commands 23 | // 24 | // global variable used for the _address.. -attributes is 25 | // anytype dpc; 26 | // dpc[1]=reference; 27 | // dpc[2]=subindex; 28 | // dpc[3]=mode; 29 | // dpc[4]=start; 30 | // dpc[5]=interval; 31 | // dpc[6]=reply; 32 | // dpc[7]=datatype; 33 | // dpc[8]=drv_ident; 34 | // dpc[9]=driver; 35 | // you don't have to set all of them, use only the necessary elements! 36 | ////////////////////////////////////////////////////////////////////////////// 37 | // 38 | // be careful: always use the same number of driver elements 39 | // (e.g. dyn_strings, cases, etc.) 40 | // 41 | ////////////////////////////////////////////////////////////////////////////// 42 | // The examples in this script use a copy of panels/para/address_sim.pnl 43 | ////////////////////////////////////////////////////////////////////////////// 44 | 45 | ////////////////////////////////////////////////////////////////////////////// 46 | // main calls the needed function 47 | // fct = 1: upDpGetAddress 48 | // fct = 2: upDpSetAddress 49 | // fct = 3: upWritePanelAllAddressAttributes 50 | // fct = 4: upReadPanelAllAddressAttributes 51 | ////////////////////////////////////////////////////////////////////////////// 52 | anytype main(string dpe, int Id, anytype dpc, int fct) 53 | { 54 | bool ok, all_right; 55 | 56 | switch (fct) 57 | { 58 | case 1: upDpGetAddress(dpe, Id, dpc); 59 | break; 60 | case 2: upDpSetAddress(dpe, Id, dpc, all_right); dpc[99]=all_right; 61 | break; 62 | case 3: upWritePanelAllAddressAttributes(dpe, Id, dpc); 63 | break; 64 | case 4: upReadPanelAllAddressAttributes(dpe, Id, dpc, all_right); dpc[99]=all_right; 65 | break; 66 | } 67 | return (dpc); 68 | } 69 | 70 | ////////////////////////////////////////////////////////////////////////////// 71 | // this function reads the datapoint values 72 | ////////////////////////////////////////////////////////////////////////////// 73 | upDpGetAddress(string dpe, int Id, anytype &dpc) 74 | { 75 | int datatype,driver,i,distrib_type; 76 | bool all_right,active; 77 | char mode; 78 | time start,interval,reply; 79 | string drv_ident,reference,config=paGetDpConfig(globalOpenConfig[Id]), 80 | dpn=dpSubStr(dpe,DPSUB_DP)+".",pg,ser_nr; 81 | unsigned subindex,offset; 82 | 83 | switch (globalAddressOld[Id]) 84 | { 85 | case "kafka": 86 | dpGet(dpe+":"+config+".._active",active, 87 | dpe+":"+config+".._reference",reference, 88 | dpe+":"+config+".._subindex",subindex, 89 | dpe+":"+config+".._mode",mode, 90 | dpe+":"+config+".._reply",reply, 91 | dpe+":"+config+".._datatype",datatype, 92 | //dpe+":"+config+".._poll_group",pg, 93 | dpe+":"+config+".._drv_ident",drv_ident, 94 | dpe+":_distrib.._driver",driver); 95 | dpc[1]=reference; 96 | dpc[2]=subindex; 97 | dpc[3]=mode; 98 | dpc[6]=reply; 99 | dpc[7]=datatype; 100 | dpc[8]=drv_ident; 101 | if (driver<1) driver=1; 102 | dpc[9]=driver; 103 | dpc[11]=pg; 104 | dpc[12]=active; 105 | break; 106 | default: break; 107 | } 108 | } 109 | 110 | ////////////////////////////////////////////////////////////////////////////// 111 | upDpSetAddress(string dpe, int Id, dyn_anytype dpc, bool &all_right) 112 | { 113 | bool ok; 114 | string config=paGetDpConfig(globalOpenConfig[Id]),dpn=dpSubStr(dpe,DPSUB_DP)+"."; 115 | dyn_int drivers; 116 | dyn_string sPara; 117 | 118 | switch (globalAddressOld[Id]) 119 | { 120 | case "kafka": 121 | paErrorHandlingDpSet( 122 | dpSetWait(dpe+":_distrib.._driver",dpc[9]),all_right); 123 | paErrorHandlingDpSet( 124 | dpSetWait(dpe+":_address.._type", DPCONFIG_PERIPH_ADDR_MAIN, 125 | dpe+":"+config+".._reference",dpc[1], 126 | dpe+":"+config+".._subindex",dpc[2], 127 | dpe+":"+config+".._mode",dpc[3], 128 | dpe+":"+config+".._reply",dpc[6], 129 | dpe+":"+config+".._datatype",dpc[7], 130 | //dpe+":"+config+".._poll_group", dpc[11], 131 | dpe+":"+config+".._drv_ident",dpc[8]),all_right); 132 | paErrorHandlingDpSet( 133 | dpSetWait(dpe+":_address.._active",dpc[12]),all_right); 134 | break; 135 | default: break; 136 | } 137 | } 138 | 139 | ////////////////////////////////////////////////////////////////////////////// 140 | upWritePanelAllAddressAttributes(string dpe, int Id, anytype dpc) 141 | { 142 | int i,j=0,pos=1,trafoPos, 143 | lowlevel,q,intern, 144 | plc=0,comp=0,rack=0,slot=0,mode=0, 145 | cbase=0,cfactor=1,ctime, 146 | ibase=0,ifactor=0,icommand, 147 | smooth=0,flutter=1, 148 | art, df=0, m, m13; 149 | bool err=false; 150 | float itime; 151 | string s,text,reference, pg; 152 | dyn_int typ; 153 | dyn_string ds,dss; 154 | 155 | switch (globalAddressOld[Id]) 156 | { 157 | case "kafka": 158 | if (dpc[3]>=64) { dpc[3]-=64; lowlevel=1;} 159 | if (dpc[3]>=32) { dpc[3]-=32;} 160 | if (dpc[3]<=0) dpc[3]=1; 161 | if (dpc[3]==1 ||dpc[3]==5) 162 | { 163 | setMultiValue(//"lowlevel","visible",false, 164 | "lowlevel", "state", 0,lowlevel); 165 | setValue("einaus","number",0); 166 | } 167 | else if (dpc[3] >= 6) 168 | { 169 | setMultiValue("einaus","number",2, 170 | //"lowlevel","visible",true, 171 | "lowlevel", "state", 0,lowlevel); 172 | } 173 | else 174 | { 175 | setMultiValue("einaus","number",1, 176 | //"lowlevel","visible",true, 177 | "lowlevel", "state", 0,lowlevel); 178 | } 179 | 180 | if (dpc[3] == 2 || dpc[3] == 6) 181 | setMultiValue("inputmode","number",0); 182 | else if (dpc[3] == 4 || dpc[3] == 7) 183 | setMultiValue("inputmode","number",1); 184 | else if (dpc[3] == 3 || dpc[3] == 8) 185 | setMultiValue("inputmode","number",2); 186 | //pg = dpc[11]; 187 | //pg = strltrim(pg, "_"); 188 | setMultiValue("cboAddressActive","state",0,dpc[12], 189 | "reference","text",dpc[1], 190 | "subindex","text",dpc[2], 191 | //"pollgroup","text",pg, 192 | "trans_art","selectedPos",dpc[7]-999, 193 | "Treiber","text",dpc[9]); 194 | break; 195 | default: 196 | break; 197 | } 198 | } 199 | 200 | ////////////////////////////////////////////////////////////////////////////// 201 | upReadPanelAllAddressAttributes(string dpe, int Id, dyn_anytype &dpc, bool &readOK) 202 | { 203 | int pos,i,j,k,l,mode,n,o,p,driver,datatype, 204 | lowlevel,q; 205 | bool active; 206 | string s,text,pg; 207 | 208 | readOK=true; 209 | switch (globalAddressOld[Id]) 210 | { 211 | case "kafka": 212 | getMultiValue("cboAddressActive","state",0,active, 213 | "trans_art","selectedPos",j, 214 | "Treiber","text",driver, 215 | "subindex","text",l, 216 | "reference","text",s, 217 | "einaus","number",p, 218 | "inputmode","number",q, 219 | //"pollgroup","text",pg, 220 | "lowlevel","state",0,lowlevel); 221 | // transformation 222 | j = j+999; 223 | 224 | // mode 225 | if (p == 0) 226 | //Output 227 | mode = 1; 228 | else if (p == 1) 229 | { 230 | // Input 231 | if (q == 0) 232 | mode = 2; 233 | else if (q == 1) 234 | mode = 4; 235 | else 236 | mode = 3; 237 | } 238 | else 239 | { 240 | // In/Output 241 | if (q == 0) 242 | mode = 6; 243 | else if (q == 1) 244 | mode = 7; 245 | else 246 | mode = 8; 247 | } 248 | if (lowlevel) mode+=64; 249 | 250 | // fill the dyn_anytype 251 | dpc[1]=s; 252 | dpc[2]=l; 253 | dpc[3]=mode; 254 | dpc[7]=j; 255 | dpc[8]=globalAddressDrivers[dynContains(globalAddressTypes,globalAddressNew[paMyModuleId()])]; 256 | dpc[9]=driver; 257 | dpc[11]= ""; //polling group 258 | dpc[12]=active; 259 | break; 260 | default: break; 261 | } 262 | } 263 | -------------------------------------------------------------------------------- /winccoa316_demo_project/kafka_producer_consumer_demo.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cern-hse-computing/WCCOAkafkaDrv/a37d333d8148a4df5ac4c33d01de0605b6ca5e8a/winccoa316_demo_project/kafka_producer_consumer_demo.zip --------------------------------------------------------------------------------