├── .gitmodules ├── gradle.properties ├── src ├── main │ ├── resources │ │ ├── build.properties │ │ └── log4j.properties │ ├── avro │ │ └── temporary_package_key.avsc │ ├── java │ │ └── org │ │ │ └── radarcns │ │ │ ├── stream │ │ │ ├── StreamWorker.java │ │ │ ├── phone │ │ │ │ ├── PhoneBatteryStream.java │ │ │ │ ├── PhoneAccelerationStream.java │ │ │ │ ├── PhoneUsageAggregationStream.java │ │ │ │ ├── PhoneUsageStream.java │ │ │ │ └── PhoneUsageCollector.java │ │ │ ├── empatica │ │ │ │ ├── E4TemperatureStream.java │ │ │ │ ├── E4BatteryLevelStream.java │ │ │ │ ├── E4BloodVolumePulseStream.java │ │ │ │ ├── E4AccelerationStream.java │ │ │ │ ├── E4InterBeatIntervalStream.java │ │ │ │ ├── E4ElectroDermalActivityStream.java │ │ │ │ └── E4HeartRateStream.java │ │ │ ├── DeviceTimestampExtractor.java │ │ │ └── KafkaStreamFactory.java │ │ │ ├── util │ │ │ ├── Comparison.java │ │ │ ├── StreamUtil.java │ │ │ ├── EmailSenders.java │ │ │ ├── PersistentStateStore.java │ │ │ ├── Monitor.java │ │ │ ├── serde │ │ │ │ ├── JsonSerializer.java │ │ │ │ ├── JsonDeserializer.java │ │ │ │ ├── RadarSerde.java │ │ │ │ └── RadarSerdes.java │ │ │ ├── RadarSingletonFactory.java │ │ │ ├── RadarUtilities.java │ │ │ ├── RadarThreadFactoryBuilder.java │ │ │ ├── EmailSender.java │ │ │ └── RadarUtilitiesImpl.java │ │ │ ├── monitor │ │ │ ├── KafkaMonitor.java │ │ │ ├── KafkaMonitorFactory.java │ │ │ └── CombinedKafkaMonitor.java │ │ │ ├── config │ │ │ ├── MockConfig.java │ │ │ ├── BatteryMonitorConfig.java │ │ │ ├── NotifyConfig.java │ │ │ ├── SubCommand.java │ │ │ ├── SingleStreamConfig.java │ │ │ ├── SourceStatisticsStreamConfig.java │ │ │ ├── DisconnectMonitorConfig.java │ │ │ ├── RadarPropertyHandler.java │ │ │ ├── MonitorConfig.java │ │ │ ├── RadarBackendOptions.java │ │ │ ├── KafkaProperty.java │ │ │ ├── StreamConfig.java │ │ │ ├── RadarPropertyHandlerImpl.java │ │ │ └── ConfigRadar.java │ │ │ ├── producer │ │ │ └── MockProducerCommand.java │ │ │ └── RadarBackend.java │ └── docker │ │ └── radar-backend-init ├── test │ ├── resources │ │ ├── org │ │ │ └── radarcns │ │ │ │ ├── collect │ │ │ │ ├── HR.csv │ │ │ │ ├── TEMP.csv │ │ │ │ ├── EDA.csv │ │ │ │ ├── tags.csv │ │ │ │ ├── IBI.csv │ │ │ │ ├── ACC.csv │ │ │ │ ├── BVP.csv │ │ │ │ └── info.txt │ │ │ │ └── stream │ │ │ │ └── phone │ │ │ │ ├── transmart_app.html.gz │ │ │ │ ├── transmart_app_broken.html.gz │ │ │ │ └── transmart_app_no_category.html.gz │ │ └── config │ │ │ ├── invalidradar.yml │ │ │ ├── invalid_stream_priority.yml │ │ │ └── radar.yml │ ├── radar.yml │ └── java │ │ └── org │ │ └── radarcns │ │ ├── util │ │ ├── serde │ │ │ ├── JsonSerializerTest.java │ │ │ ├── JsonDeserializerTest.java │ │ │ └── RadarSerdesTest.java │ │ ├── RadarUtilsTest.java │ │ ├── EmailServerRule.java │ │ ├── PersistentStateStoreTest.java │ │ └── EmailSenderTest.java │ │ ├── stream │ │ ├── StreamDefinitionTest.java │ │ ├── phone │ │ │ ├── PlayStoreLookupTest.java │ │ │ └── PlayStoreCategoryParserTest.java │ │ ├── SensorStreamWorkerTest.java │ │ └── DeviceTimestampExtractorTest.java │ │ ├── config │ │ ├── RadarBackendOptionsTest.java │ │ └── RadarPropertyHandlerTest.java │ │ └── monitor │ │ ├── CombinedKafkaMonitorTest.java │ │ └── BatteryLevelMonitorTest.java └── integrationTest │ ├── resources │ ├── integration_test.csv │ ├── mock_file.yml │ ├── mock_devices.yml │ └── org │ │ └── radarcns │ │ └── kafka │ │ └── radar.yml │ ├── docker │ ├── Dockerfile │ └── docker-compose.yml │ └── java │ └── org │ └── radarcns │ └── integration │ └── E4AggregatedAccelerationMonitor.java ├── .dockerignore ├── settings.gradle ├── gradle ├── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── profile.dev.gradle ├── profile.prod.gradle ├── profile.test.gradle ├── utilities.gradle ├── codacy.gradle ├── style.gradle ├── test.gradle └── publishing.gradle ├── smtp.env.template ├── CODEOWNERS ├── .gitignore ├── sink-radar.properties ├── .travis.yml ├── Dockerfile ├── gradlew.bat ├── config └── pmd │ └── ruleset.xml └── radar.yml /.gitmodules: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | bintrayUser= 2 | bintrayApiKey= -------------------------------------------------------------------------------- /src/main/resources/build.properties: -------------------------------------------------------------------------------- 1 | version=${version} -------------------------------------------------------------------------------- /src/test/resources/org/radarcns/collect/HR.csv: -------------------------------------------------------------------------------- 1 | 1469111691.000000 2 | 1.000000 3 | 82.00 4 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | /.idea/ 2 | /.gradle* 3 | /.git* 4 | /out/ 5 | /config/ 6 | /build/ 7 | /libs/ 8 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'radar-backend' 2 | enableFeaturePreview('STABLE_PUBLISHING') 3 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RADAR-base/RADAR-Backend/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /src/test/resources/org/radarcns/collect/TEMP.csv: -------------------------------------------------------------------------------- 1 | 1469111681.000000 2 | 4.000000 3 | 382.18 4 | 382.18 5 | 382.18 6 | 382.18 7 | -------------------------------------------------------------------------------- /src/test/resources/org/radarcns/collect/EDA.csv: -------------------------------------------------------------------------------- 1 | 1469111681.000000 2 | 4.000000 3 | 0.000000 4 | 0.014089 5 | 0.025616 6 | 0.026897 7 | -------------------------------------------------------------------------------- /smtp.env.template: -------------------------------------------------------------------------------- 1 | SMARTHOST_ADDRESS=mail.example.com 2 | SMARTHOST_PORT=587 3 | SMARTHOST_USER=user@example.com 4 | SMARTHOST_PASSWORD=XXXXXXXX 5 | -------------------------------------------------------------------------------- /src/test/resources/config/invalidradar.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | released: 2016-11-27 3 | 4 | #============================= Application =============================# 5 | somethingelse: 6 | -------------------------------------------------------------------------------- /src/test/resources/org/radarcns/stream/phone/transmart_app.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RADAR-base/RADAR-Backend/HEAD/src/test/resources/org/radarcns/stream/phone/transmart_app.html.gz -------------------------------------------------------------------------------- /src/test/resources/org/radarcns/stream/phone/transmart_app_broken.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RADAR-base/RADAR-Backend/HEAD/src/test/resources/org/radarcns/stream/phone/transmart_app_broken.html.gz -------------------------------------------------------------------------------- /src/test/resources/org/radarcns/stream/phone/transmart_app_no_category.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RADAR-base/RADAR-Backend/HEAD/src/test/resources/org/radarcns/stream/phone/transmart_app_no_category.html.gz -------------------------------------------------------------------------------- /src/integrationTest/resources/integration_test.csv: -------------------------------------------------------------------------------- 1 | userId,sourceId,time,timeReceived,acceleration 2 | a,b,14191933191.223,14191933193.223,0.001,0.3222,0.6342 3 | a,c,14191933194.223,14191933195.223,0.13131,0.6241,0.2423 4 | -------------------------------------------------------------------------------- /gradle/profile.dev.gradle: -------------------------------------------------------------------------------- 1 | apply from: 'gradle/test.gradle' 2 | apply from: 'gradle/codacy.gradle' 3 | apply from: 'gradle/style.gradle' 4 | apply from: 'gradle/utilities.gradle' 5 | apply from: 'gradle/publishing.gradle' 6 | -------------------------------------------------------------------------------- /src/integrationTest/resources/mock_file.yml: -------------------------------------------------------------------------------- 1 | data: 2 | - file: integration_test.csv 3 | topic: integration_test 4 | key_schema: org.radarcns.kafka.ObservationKey 5 | value_schema: org.radarcns.aggregator.DoubleArrayAggregator -------------------------------------------------------------------------------- /src/test/resources/org/radarcns/collect/tags.csv: -------------------------------------------------------------------------------- 1 | 1469112869.71 2 | 1469116922.63 3 | 1469118251.51 4 | 1469126138.00 5 | 1469127001.99 6 | 1469128272.99 7 | 1469129670.16 8 | 1469138804.35 9 | 1469139888.93 10 | 1469141086.97 11 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # These owners will be the default owners for everything in the repo. 2 | # Unless a later match takes precedence, they will be requested for review when someone 3 | # opens a pull request. 4 | * @blootsvoets @nivemaham @yatharthranjan 5 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.9-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradle/profile.prod.gradle: -------------------------------------------------------------------------------- 1 | task downloadRuntimeDependencies { 2 | description "Pre-downloads dependencies" 3 | configurations.compileClasspath.files 4 | configurations.runtimeClasspath.files 5 | } 6 | 7 | processResources { 8 | expand(version: version) 9 | } -------------------------------------------------------------------------------- /src/test/resources/org/radarcns/collect/IBI.csv: -------------------------------------------------------------------------------- 1 | 1469111681.000000, IBI 2 | 19.782156,0.812537 3 | 20.704073,0.921917 4 | 107.239283,0.843789 5 | 107.989318,0.750034 6 | 111.692612,0.906291 7 | 112.395769,0.703157 8 | 113.270809,0.875040 9 | 114.052095,0.781286 10 | 129.349670,0.781286 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Intellij 2 | .idea/ 3 | *.iml 4 | *.iws 5 | classes/ 6 | 7 | # Mac 8 | .DS_Store 9 | 10 | # Maven 11 | log/ 12 | 13 | # Gradle 14 | .gradle/ 15 | .gradletasknamecache 16 | 17 | # Build 18 | /build/ 19 | 20 | # logs 21 | backend.log 22 | 23 | # Configuration 24 | smtp.env 25 | 26 | /out/ 27 | /libs/ 28 | 29 | # Distribution 30 | /radar-backend-* -------------------------------------------------------------------------------- /gradle/profile.test.gradle: -------------------------------------------------------------------------------- 1 | apply from: 'gradle/test.gradle' 2 | 3 | task downloadTestDependencies { 4 | description "Pre-downloads *most* dependencies" 5 | configurations.compileClasspath.files 6 | configurations.testCompileClasspath.files 7 | configurations.integrationTestCompileClasspath.files 8 | configurations.integrationTestRuntimeClasspath.files 9 | } 10 | 11 | processResources { 12 | expand(version: version) 13 | } 14 | -------------------------------------------------------------------------------- /src/test/resources/config/invalid_stream_priority.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | released: 2016-11-27 3 | 4 | #============================= Application =============================# 5 | #Possible value are standalone or high_performance 6 | 7 | 8 | #============================ Kafka Streams ============================# 9 | #The number of threads that a stream must be run according is priority 10 | stream: 11 | threads_per_priority: 12 | low: 0 13 | normal: 1 14 | high: 1 15 | -------------------------------------------------------------------------------- /src/main/avro/temporary_package_key.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "org.radarcns.stream.phone", 3 | "name": "TemporaryPackageKey", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "projectId", "type": ["null", "string"], "doc": "Project ID."}, 7 | {"name": "userId", "type": "string", "doc": "User ID."}, 8 | {"name": "sourceId", "type": "string", "doc": "Source ID."}, 9 | {"name": "packageName", "type": "string", "doc": "Package name."} 10 | ] 11 | } -------------------------------------------------------------------------------- /src/main/java/org/radarcns/stream/StreamWorker.java: -------------------------------------------------------------------------------- 1 | package org.radarcns.stream; 2 | 3 | import java.util.stream.Stream; 4 | import org.radarcns.config.RadarPropertyHandler; 5 | import org.radarcns.config.SingleStreamConfig; 6 | 7 | public interface StreamWorker { 8 | void start(); 9 | void configure(StreamMaster streamMaster, RadarPropertyHandler properties, 10 | SingleStreamConfig singleConfig); 11 | Stream getStreamDefinitions(); 12 | void shutdown(); 13 | } 14 | -------------------------------------------------------------------------------- /src/test/resources/org/radarcns/collect/ACC.csv: -------------------------------------------------------------------------------- 1 | 1469111681.000000, 1469111681.000000, 1469111681.000000 2 | 32.000000, 32.000000, 32.000000 3 | -40,57,45 4 | -33,57,50 5 | -39,55,54 6 | -39,54,54 7 | -44,47,47 8 | -39,42,52 9 | -41,35,49 10 | -42,26,49 11 | -42,25,45 12 | -37,20,46 13 | -38,16,41 14 | -38,16,42 15 | -30,18,42 16 | -30,17,38 17 | -40,18,36 18 | -34,22,49 19 | -31,19,44 20 | -44,19,38 21 | -45,19,44 22 | -40,13,47 23 | -47,5,39 24 | -59,6,37 25 | -61,8,38 26 | -51,9,41 27 | -50,7,38 28 | -55,5,37 29 | -56,4,37 30 | -47,2,35 31 | -59,1,32 32 | -62,1,31 33 | -57,4,32 34 | -46,4,32 35 | -------------------------------------------------------------------------------- /src/integrationTest/resources/mock_devices.yml: -------------------------------------------------------------------------------- 1 | #=========================== Schema Registry ============================# 2 | #List of Schema Registry instances 3 | schema_registry: 4 | protocol: http 5 | host: schema-registry-1 6 | port: 8081 7 | 8 | #============================ Kafka broker ==============================# 9 | broker: 10 | - host: kafka-1 11 | port: 9092 12 | 13 | #=============================== Test ===================================# 14 | # time duration that mock data has to cover expressed in milliseconds. 15 | duration_millis: 15000 16 | 17 | #Possible value are direct, rest 18 | producer_mode: direct 19 | -------------------------------------------------------------------------------- /gradle/utilities.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'idea' 2 | 3 | task downloadDependencies { 4 | description "Pre-downloads *most* dependencies" 5 | doLast { 6 | configurations.getAsMap().each { name, config -> 7 | println "Retrieving dependencies for $name" 8 | try { 9 | config.files 10 | } catch (e) { 11 | project.logger.info e.message // some cannot be resolved, silentlyish skip them 12 | } 13 | } 14 | } 15 | } 16 | 17 | idea { 18 | module { 19 | downloadSources = true 20 | } 21 | } 22 | 23 | wrapper { 24 | gradleVersion '4.9' 25 | } 26 | 27 | -------------------------------------------------------------------------------- /src/test/resources/org/radarcns/collect/BVP.csv: -------------------------------------------------------------------------------- 1 | 1469111681.00 2 | 64.000000 3 | -0.00 4 | -0.00 5 | -0.00 6 | -0.00 7 | -0.00 8 | -0.00 9 | -0.00 10 | -0.00 11 | -0.00 12 | -0.00 13 | 0.00 14 | 0.01 15 | 0.01 16 | -0.00 17 | -0.03 18 | -0.05 19 | -0.05 20 | 0.00 21 | 0.13 22 | 0.36 23 | 0.66 24 | 1.01 25 | 1.37 26 | 1.72 27 | 2.06 28 | 2.40 29 | 2.79 30 | 3.24 31 | 3.80 32 | 4.42 33 | 5.06 34 | 5.66 35 | 6.20 36 | 6.68 37 | 7.15 38 | 7.68 39 | 8.31 40 | 9.14 41 | 10.09 42 | 11.04 43 | 11.88 44 | 12.56 45 | 13.14 46 | 13.76 47 | 14.65 48 | 15.96 49 | 17.91 50 | 20.32 51 | 22.90 52 | 25.38 53 | 27.58 54 | 29.57 55 | 31.68 56 | 34.32 57 | 37.81 58 | 42.54 59 | 48.15 60 | 54.10 61 | 59.83 62 | 64.97 63 | 69.58 64 | 74.14 65 | 79.34 66 | 85.71 67 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/util/Comparison.java: -------------------------------------------------------------------------------- 1 | package org.radarcns.util; 2 | 3 | import java.util.function.BiFunction; 4 | import java.util.function.Function; 5 | 6 | public interface Comparison extends BiFunction { 7 | static > Comparison compare(Function property) { 8 | return (a, b) -> property.apply(a).compareTo(property.apply(b)); 9 | } 10 | 11 | default > Comparison then(Function property) { 12 | return (a, b) -> { 13 | int ret = apply(a, b); 14 | if (ret != 0) { 15 | return ret; 16 | } 17 | return compare(property).apply(a, b); 18 | }; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /sink-radar.properties: -------------------------------------------------------------------------------- 1 | # Kafka consumer configuration 2 | name=radar-connector-mongodb-sink 3 | 4 | # Kafka connector configuration 5 | connector.class=org.radarcns.mongodb.MongoDbSinkConnector 6 | tasks.max=1 7 | 8 | # Topics that will be consumed 9 | topics=android_empatica_e4_battery_level,android_empatica_e4_battery_level_output 10 | 11 | # MongoDB server 12 | mongo.host=127.0.0.1 13 | mongo.port=27017 14 | 15 | # MongoDB configuration 16 | mongo.username= 17 | mongo.password= 18 | mongo.database=mydbase 19 | 20 | # Collection name for putting data into the MongoDB database. The {$topic} token will be replaced 21 | # by the Kafka topic name. 22 | #mongo.collection.format={$topic} 23 | 24 | # Factory class to do the actual record conversion 25 | record.converter.class=org.radarcns.sink.mongodb.RecordConverterFactoryRadar 26 | -------------------------------------------------------------------------------- /src/test/radar.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | released: 2016-11-27 3 | 4 | #============================== Zookeeper ==============================# 5 | #List of Zookeeper instances 6 | zookeeper: 7 | - host: localhost 8 | port: 2181 9 | 10 | #================================ Kafka ================================# 11 | #List of Kafka brokers 12 | broker: 13 | - host: localhost 14 | port: 9092 15 | 16 | #============================ Kafka Streams ============================# 17 | #The number of threads that a stream must be run according is priority 18 | stream: 19 | properties: 20 | auto.commit.interval.ms: 1000 21 | session.timeout.ms: 10000 22 | 23 | threads_per_priority: 24 | low: 1 25 | normal: 2 26 | high: 4 27 | 28 | #=========================== Schema Registry ===========================# 29 | #List of Schema Registry instances 30 | schema_registry: 31 | - host: localhost 32 | port: 8081 33 | protocol: http -------------------------------------------------------------------------------- /gradle/codacy.gradle: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------------------------// 2 | // Code coverage and codacy // 3 | //---------------------------------------------------------------------------// 4 | 5 | apply plugin: 'jacoco' 6 | 7 | configurations { 8 | codacy 9 | } 10 | 11 | jacoco { 12 | toolVersion = "0.8.1" 13 | } 14 | 15 | dependencies { 16 | codacy group: 'com.github.codacy', name: 'codacy-coverage-reporter', version: '4.0.1' 17 | } 18 | 19 | jacocoTestReport { 20 | reports { 21 | xml.enabled true 22 | csv.enabled false 23 | html.enabled true 24 | } 25 | executionData test, integrationTest 26 | } 27 | 28 | task sendCoverageToCodacy(type: JavaExec, dependsOn: jacocoTestReport) { 29 | main = 'com.codacy.CodacyCoverageReporter' 30 | classpath = configurations.codacy 31 | args = ['report', '-l', 'Java', '-r', "${buildDir}/reports/jacoco/test/jacocoTestReport.xml"] 32 | } 33 | -------------------------------------------------------------------------------- /src/test/resources/config/radar.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | released: 2016-11-27 3 | 4 | #============================== Zookeeper ==============================# 5 | #List of Zookeeper instances 6 | zookeeper: 7 | - host: localhost 8 | port: 2181 9 | 10 | #================================ Kafka ================================# 11 | #List of Kafka brokers 12 | broker: 13 | - host: localhost 14 | port: 9092 15 | 16 | 17 | #============================ Kafka Streams ============================# 18 | #The number of threads that a stream must be run according is priority 19 | stream: 20 | properties: 21 | auto.commit.interval.ms: 1000 22 | session.timeout.ms: 10000 23 | 24 | threads_per_priority: 25 | low: 1 26 | normal: 2 27 | high: 4 28 | 29 | #=========================== Schema Registry ===========================# 30 | #List of Schema Registry instances 31 | schema_registry: 32 | - host: localhost 33 | port: 8081 34 | protocol: http 35 | 36 | extras: 37 | somethingother: bla -------------------------------------------------------------------------------- /src/main/java/org/radarcns/monitor/KafkaMonitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.monitor; 18 | 19 | import java.time.Duration; 20 | import org.radarcns.config.SubCommand; 21 | 22 | public interface KafkaMonitor extends SubCommand { 23 | boolean isShutdown(); 24 | Duration getPollTimeout(); 25 | void setPollTimeout(Duration duration); 26 | } 27 | -------------------------------------------------------------------------------- /gradle/style.gradle: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------------------------// 2 | // Style checking // 3 | //---------------------------------------------------------------------------// 4 | apply plugin: 'pmd' 5 | apply plugin: 'checkstyle' 6 | 7 | checkstyle { 8 | // codacy version 9 | toolVersion '8.4' 10 | ignoreFailures false 11 | 12 | // ignore tests 13 | sourceSets = [sourceSets.main, sourceSets.test, sourceSets.integrationTest] 14 | } 15 | 16 | tasks.withType(Checkstyle) { 17 | exclude '**/org/radarcns/stream/phone/TemporaryPackageKey.java' 18 | } 19 | 20 | pmd { 21 | // pmd version 22 | toolVersion = '5.8.1' 23 | ignoreFailures = false 24 | 25 | sourceSets = [sourceSets.main] 26 | 27 | consoleOutput = true 28 | 29 | ruleSets = [] 30 | ruleSetFiles = files("config/pmd/ruleset.xml") 31 | } 32 | 33 | tasks.withType(Pmd) { 34 | exclude '**/org/radarcns/config/ConfigRadar.java' 35 | exclude '**/org/radarcns/stream/phone/TemporaryPackageKey.java' 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/config/MockConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.config; 18 | 19 | import java.util.List; 20 | import org.radarcns.mock.config.MockDataConfig; 21 | 22 | public class MockConfig { 23 | private List data; 24 | 25 | public List getData() { 26 | return data; 27 | } 28 | 29 | public void setData(List data) { 30 | this.data = data; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/config/BatteryMonitorConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.config; 18 | 19 | /** 20 | * POJO representing a battery status monitor configuration 21 | */ 22 | public class BatteryMonitorConfig extends MonitorConfig { 23 | private String level; 24 | 25 | public String getLevel() { 26 | return level; 27 | } 28 | 29 | public void setLevel(String level) { 30 | this.level = level; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/integrationTest/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | FROM openjdk:8 14 | 15 | RUN mkdir /code 16 | WORKDIR /code 17 | 18 | ENV GRADLE_OPTS -Dorg.gradle.daemon=false -Dorg.gradle.project.profile=test 19 | ENV TERM dumb 20 | 21 | COPY ./gradle/wrapper /code/gradle/wrapper 22 | COPY ./gradlew /code/ 23 | RUN ./gradlew --version 24 | 25 | COPY ./gradle/*test.gradle /code/gradle/ 26 | COPY ./build.gradle ./gradle.properties ./settings.gradle /code/ 27 | 28 | RUN ./gradlew downloadTestDependencies 29 | 30 | COPY ./src/ /code/src 31 | 32 | RUN ./gradlew integrationTestClasses 33 | 34 | ENTRYPOINT ["./gradlew"] 35 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/stream/phone/PhoneBatteryStream.java: -------------------------------------------------------------------------------- 1 | package org.radarcns.stream.phone; 2 | 3 | import javax.annotation.Nonnull; 4 | import org.apache.kafka.streams.kstream.KStream; 5 | import org.radarcns.config.RadarPropertyHandler.Priority; 6 | import org.radarcns.kafka.AggregateKey; 7 | import org.radarcns.kafka.ObservationKey; 8 | import org.radarcns.passive.phone.PhoneBatteryLevel; 9 | import org.radarcns.stream.SensorStreamWorker; 10 | import org.radarcns.stream.StreamDefinition; 11 | import org.radarcns.stream.aggregator.NumericAggregate; 12 | 13 | public class PhoneBatteryStream extends SensorStreamWorker { 14 | @Override 15 | protected void initialize() { 16 | defineWindowedSensorStream("android_phone_battery_level"); 17 | config.setDefaultPriority(Priority.LOW); 18 | } 19 | 20 | @Override 21 | protected KStream implementStream(StreamDefinition definition, 22 | @Nonnull KStream kstream) { 23 | return aggregateNumeric(definition, kstream, "batteryLevel", 24 | PhoneBatteryLevel.getClassSchema()); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/config/NotifyConfig.java: -------------------------------------------------------------------------------- 1 | package org.radarcns.config; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import java.util.List; 6 | 7 | 8 | /** 9 | * POJO to store each email Notification configuration. 10 | */ 11 | public class NotifyConfig { 12 | @JsonProperty("project_id") 13 | private String projectId; 14 | 15 | @JsonProperty("email_address") 16 | private List emailAddress; 17 | 18 | @JsonCreator 19 | public NotifyConfig(@JsonProperty("project_id") String projectId, 20 | @JsonProperty("email_address") List emailAddress) { 21 | this.projectId = projectId; 22 | this.emailAddress = emailAddress; 23 | } 24 | 25 | public String getProjectId() { 26 | return projectId; 27 | } 28 | 29 | public void setProjectId(String projectId) { 30 | this.projectId = projectId; 31 | } 32 | 33 | public List getEmailAddress() { 34 | return emailAddress; 35 | } 36 | 37 | public void setEmailAddress(List emailAddress) { 38 | this.emailAddress = emailAddress; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/stream/phone/PhoneAccelerationStream.java: -------------------------------------------------------------------------------- 1 | package org.radarcns.stream.phone; 2 | 3 | import javax.annotation.Nonnull; 4 | import org.apache.kafka.streams.kstream.KStream; 5 | import org.radarcns.config.RadarPropertyHandler.Priority; 6 | import org.radarcns.kafka.AggregateKey; 7 | import org.radarcns.kafka.ObservationKey; 8 | import org.radarcns.passive.phone.PhoneAcceleration; 9 | import org.radarcns.stream.SensorStreamWorker; 10 | import org.radarcns.stream.StreamDefinition; 11 | import org.radarcns.stream.aggregator.AggregateList; 12 | 13 | public class PhoneAccelerationStream extends SensorStreamWorker { 14 | @Override 15 | protected void initialize() { 16 | defineWindowedSensorStream("android_phone_acceleration"); 17 | config.setDefaultPriority(Priority.HIGH); 18 | } 19 | 20 | @Override 21 | protected KStream implementStream( 22 | StreamDefinition definition, 23 | @Nonnull KStream kstream) { 24 | return aggregateFields(definition, kstream, new String[] {"x", "y", "z"}, 25 | PhoneAcceleration.getClassSchema()); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/test/java/org/radarcns/util/serde/JsonSerializerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.util.serde; 18 | 19 | import junit.framework.TestCase; 20 | import org.radarcns.kafka.ObservationKey; 21 | 22 | public class JsonSerializerTest extends TestCase { 23 | public void testSerialize() throws Exception { 24 | JsonSerializer serializer = new JsonSerializer<>(); 25 | ObservationKey key = new ObservationKey("test", "user", "source"); 26 | String result = new String(serializer.serialize("mytest", key)); 27 | assertEquals("{\"projectId\":\"test\",\"userId\":\"user\",\"sourceId\":\"source\"}", result); 28 | } 29 | } -------------------------------------------------------------------------------- /src/main/java/org/radarcns/config/SubCommand.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.config; 18 | 19 | import java.io.IOException; 20 | 21 | /** 22 | * Subcommand of RadarBackend to run. 23 | */ 24 | public interface SubCommand { 25 | /** 26 | * Start the subcommand. The command is not guaranteed to return 27 | * immediately. 28 | * @throws IOException if the command cannot be started 29 | * @throws InterruptedException if the command is interrupted 30 | */ 31 | void start() throws IOException, InterruptedException; 32 | 33 | /** Stop the subcommand, possibly waiting for it to complete. */ 34 | void shutdown() throws IOException, InterruptedException; 35 | } 36 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - oraclejdk8 4 | sudo: required 5 | 6 | services: 7 | - docker 8 | 9 | env: 10 | DOCKER_COMPOSE_VERSION: 1.21.2 11 | 12 | cache: 13 | directories: 14 | - $HOME/.gradle/caches/jars-1 15 | - $HOME/.gradle/caches/jars-2 16 | - $HOME/.gradle/caches/jars-3 17 | - $HOME/.gradle/caches/modules-2/files-2.1/ 18 | - $HOME/.gradle/native 19 | - $HOME/.gradle/wrapper 20 | 21 | before_install: 22 | - mkdir -p "$HOME/bin"; 23 | - export PATH="$PATH:$HOME/bin"; 24 | - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > "$HOME/bin/docker-compose"; 25 | - chmod +x "$HOME/bin/docker-compose"; 26 | script: 27 | - ./gradlew check 28 | - cd src/integrationTest/docker 29 | - sudo docker-compose up -d zookeeper-1 kafka-1 schema-registry-1 && sleep 30 && sudo docker-compose run --rm integration-test 30 | - sudo docker-compose down 31 | - cd ../../.. 32 | after_script: 33 | - ./gradlew sendCoverageToCodacy 34 | 35 | deploy: 36 | provider: releases 37 | api_key: ${GH_TOKEN} 38 | file_glob: true 39 | file: 40 | - build/libs/*.jar 41 | - build/distributions/*.zip 42 | - build/distributions/*.tar.gz 43 | skip_cleanup: true 44 | on: 45 | tags: true 46 | 47 | after_deploy: 48 | - ./gradlew bintrayUpload 49 | -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2017 King's College London and The Hyve 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # Root logger option 18 | log4j.rootLogger=INFO, stdout 19 | 20 | log4j.logger.org.apache.kafka.streams.processor.internals=WARN 21 | log4j.logger.org.apache.kafka.clients.consumer.internals=WARN 22 | log4j.logger.org.apache.kafka.clients.producer.internals=WARN 23 | 24 | # Redirect log messages to console 25 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 26 | log4j.appender.stdout.Target=System.out 27 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 28 | log4j.appender.stdout.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ss zzz}] %5p [%t] (%F:%L) - %m (%c)%n 29 | #log4j.appender.stdout.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ss zzz}] %-5p %t %C.%M:%L - %m%n 30 | -------------------------------------------------------------------------------- /src/test/java/org/radarcns/util/serde/JsonDeserializerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.util.serde; 18 | 19 | import java.nio.charset.Charset; 20 | import junit.framework.TestCase; 21 | import org.radarcns.kafka.ObservationKey; 22 | 23 | public class JsonDeserializerTest extends TestCase { 24 | public void testSerialize() throws Exception { 25 | byte[] json = "{\"userId\":\"user\",\"sourceId\":\"source\"}" 26 | .getBytes(Charset.forName("UTF-8")); 27 | JsonDeserializer serializer = new JsonDeserializer<>(ObservationKey.class); 28 | ObservationKey key = serializer.deserialize("mytest", json); 29 | assertEquals("user", key.getUserId()); 30 | assertEquals("source", key.getSourceId()); 31 | } 32 | } -------------------------------------------------------------------------------- /src/main/java/org/radarcns/util/StreamUtil.java: -------------------------------------------------------------------------------- 1 | package org.radarcns.util; 2 | 3 | import java.util.function.BiFunction; 4 | import java.util.function.BiPredicate; 5 | import java.util.function.Function; 6 | import java.util.function.Predicate; 7 | import java.util.stream.Stream; 8 | import org.apache.kafka.streams.KeyValue; 9 | 10 | public final class StreamUtil { 11 | private StreamUtil() { 12 | // utility class 13 | } 14 | 15 | public static Predicate> test(BiPredicate bip) { 16 | return entry -> bip.test(entry.key, entry.value); 17 | } 18 | 19 | public static Function, R> apply( 20 | BiFunction bif) { 21 | return entry -> bif.apply(entry.key, entry.value); 22 | } 23 | 24 | public static Function, K> first() { 25 | return e -> e.key; 26 | } 27 | 28 | public static Function, V> second() { 29 | return e -> e.value; 30 | } 31 | 32 | @FunctionalInterface 33 | public interface StreamSupplier { 34 | Stream get(); 35 | 36 | default StreamSupplier concat(StreamSupplier other) { 37 | return () -> Stream.concat(get(), other.get()); 38 | } 39 | 40 | static StreamSupplier supply(StreamSupplier supplier) { 41 | return supplier; 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/config/SingleStreamConfig.java: -------------------------------------------------------------------------------- 1 | package org.radarcns.config; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import com.fasterxml.jackson.annotation.JsonSetter; 5 | import java.util.Collections; 6 | import java.util.Locale; 7 | import java.util.Map; 8 | import javax.annotation.Nonnull; 9 | import org.radarcns.config.RadarPropertyHandler.Priority; 10 | 11 | public class SingleStreamConfig { 12 | @JsonProperty("class") 13 | private Class streamClass; 14 | @JsonProperty 15 | private Map properties; 16 | @JsonProperty 17 | private Priority priority = null; 18 | 19 | public void setStreamClass(Class streamClass) { 20 | this.streamClass = streamClass; 21 | } 22 | 23 | public Class getStreamClass() { 24 | return streamClass; 25 | } 26 | 27 | @Nonnull 28 | public Map getProperties() { 29 | return properties != null ? properties : Collections.emptyMap(); 30 | } 31 | 32 | @JsonSetter("priority") 33 | protected void setPriority(String priority) { 34 | this.priority = Priority.valueOf(priority.toUpperCase(Locale.US)); 35 | } 36 | 37 | public void setDefaultPriority(Priority priority) { 38 | if (this.priority == null) { 39 | this.priority = priority; 40 | } 41 | } 42 | 43 | public Priority getPriority() { 44 | return priority != null ? priority : Priority.NORMAL; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/util/EmailSenders.java: -------------------------------------------------------------------------------- 1 | package org.radarcns.util; 2 | 3 | import java.io.IOException; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | import org.radarcns.config.MonitorConfig; 7 | import org.radarcns.config.NotifyConfig; 8 | 9 | 10 | /** 11 | * Class to store {@link EmailSender} associated with each project. 12 | */ 13 | public class EmailSenders { 14 | 15 | private final Map emailSenderMap; 16 | 17 | public EmailSenders(Map map) { 18 | this.emailSenderMap = map; 19 | } 20 | 21 | /** 22 | * 23 | * Parses the {@link MonitorConfig} to map the corresponding 24 | * {@link EmailSender} to each project. A project can have a list of 25 | * associated email addresses. 26 | * 27 | * @param config Configuration of the Monitor containing project 28 | * and email address mapping 29 | * @throws IOException 30 | */ 31 | 32 | public static EmailSenders parseConfig(MonitorConfig config) throws IOException{ 33 | Map map = new HashMap<>(); 34 | for(NotifyConfig notifyConfig : config.getNotifyConfig()) { 35 | map.put(notifyConfig.getProjectId(), 36 | new EmailSender(config.getEmailHost(), config.getEmailPort(), 37 | config.getEmailUser(), notifyConfig.getEmailAddress())); 38 | } 39 | 40 | return new EmailSenders(map); 41 | } 42 | 43 | public EmailSender getEmailSenderForProject(String projectId) { 44 | return emailSenderMap.get(projectId); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/test/resources/org/radarcns/collect/info.txt: -------------------------------------------------------------------------------- 1 | .csv files in this archive are in the following format: 2 | The first row is the initial time of the session expressed as unix timestamp in UTC. 3 | The second row is the sample rate expressed in Hz. 4 | 5 | TEMP.csv 6 | Data from temperature sensor expressed degrees on the Celsius (°C) scale. 7 | 8 | EDA.csv 9 | Data from the electrodermal activity sensor expressed as microsiemens (μS). 10 | 11 | BVP.csv 12 | Data from photoplethysmograph. 13 | 14 | ACC.csv 15 | Data from 3-axis accelerometer sensor. The accelerometer is configured to measure acceleration in the range [-2g, 2g]. Therefore the unit in this file is 1/64g. 16 | Data from x, y, and z axis are respectively in first, second, and third column. 17 | 18 | IBI.csv 19 | Time between individuals heart beats extracted from the BVP signal. 20 | No sample rate is needed for this file. 21 | The first column is the time (respect to the initial time) of the detected inter-beat interval expressed in seconds (s). 22 | The second column is the duration in seconds (s) of the detected inter-beat interval (i.e., the distance in seconds from the previous beat). 23 | 24 | HR.csv 25 | Average heart rate extracted from the BVP signal.The first row is the initial time of the session expressed as unix timestamp in UTC. 26 | The second row is the sample rate expressed in Hz. 27 | 28 | 29 | tags.csv 30 | Event mark times. 31 | Each row corresponds to a physical button press on the device; the same time as the status LED is first illuminated. 32 | The time is expressed as a unix timestamp in UTC and it is synchronized with initial time of the session indicated in the related data files from the corresponding session. 33 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/config/SourceStatisticsStreamConfig.java: -------------------------------------------------------------------------------- 1 | package org.radarcns.config; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import java.util.List; 5 | import org.radarcns.stream.statistics.SourceStatisticsStream; 6 | 7 | public class SourceStatisticsStreamConfig extends SingleStreamConfig { 8 | private String name; 9 | 10 | private List topics; 11 | 12 | @JsonProperty("output_topic") 13 | private String outputTopic = "source_statistics"; 14 | 15 | @JsonProperty("max_batch_size") 16 | private int maxBatchSize = 1000; 17 | 18 | @JsonProperty("flush_timeout") 19 | private long flushTimeout = 60_000L; 20 | 21 | public SourceStatisticsStreamConfig() { 22 | setStreamClass(SourceStatisticsStream.class); 23 | } 24 | 25 | public List getTopics() { 26 | return topics; 27 | } 28 | 29 | public void setTopics(List topics) { 30 | this.topics = topics; 31 | } 32 | 33 | public String getOutputTopic() { 34 | return outputTopic; 35 | } 36 | 37 | public void setOutputTopic(String outputTopic) { 38 | this.outputTopic = outputTopic; 39 | } 40 | 41 | public int getMaxBatchSize() { 42 | return maxBatchSize; 43 | } 44 | 45 | public void setMaxBatchSize(int maxBatchSize) { 46 | this.maxBatchSize = maxBatchSize; 47 | } 48 | 49 | public long getFlushTimeout() { 50 | return flushTimeout; 51 | } 52 | 53 | public void setFlushTimeout(long flushTimeout) { 54 | this.flushTimeout = flushTimeout; 55 | } 56 | 57 | public String getName() { 58 | return name; 59 | } 60 | 61 | public void setName(String name) { 62 | this.name = name; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | FROM openjdk:8-alpine AS builder 14 | 15 | RUN mkdir /code 16 | WORKDIR /code 17 | 18 | ENV GRADLE_OPTS -Dorg.gradle.daemon=false -Dorg.gradle.project.profile=prod 19 | 20 | COPY ./gradle/wrapper /code/gradle/wrapper 21 | COPY ./gradlew /code/ 22 | RUN ./gradlew --version 23 | 24 | COPY ./gradle/profile.prod.gradle /code/gradle/ 25 | COPY ./build.gradle ./gradle.properties ./settings.gradle /code/ 26 | 27 | RUN ./gradlew downloadRuntimeDependencies 28 | 29 | COPY ./src/ /code/src 30 | 31 | RUN ./gradlew distTar && \ 32 | tar xf build/distributions/*.tar && \ 33 | rm build/distributions/*.tar 34 | 35 | FROM confluentinc/cp-base:5.0.0 36 | 37 | MAINTAINER Nivethika M , Joris Borgdorff , Yatharth Ranjan 38 | 39 | LABEL description="RADAR-CNS Backend streams and monitor" 40 | 41 | ENV KAFKA_REST_PROXY http://rest-proxy:8082 42 | ENV KAFKA_SCHEMA_REGISTRY http://schema-registry:8081 43 | 44 | COPY --from=builder /code/radar-backend-*/bin/* /usr/bin/ 45 | COPY --from=builder /code/radar-backend-*/lib/* /usr/lib/ 46 | 47 | # Load topics validator 48 | COPY ./src/main/docker/radar-backend-init /usr/bin 49 | 50 | ENTRYPOINT ["radar-backend-init"] 51 | -------------------------------------------------------------------------------- /src/main/docker/radar-backend-init: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | # Busy waiting loop that waits untill all topic are available 6 | echo "===> Waiting for RADAR-CNS topics ... " 7 | 8 | # Check if variables exist 9 | if [ -z "$KAFKA_REST_PROXY" ]; then 10 | echo "KAFKA_REST_PROXY is not defined" 11 | exit 2 12 | fi 13 | 14 | if [ -z "$KAFKA_SCHEMA_REGISTRY" ]; then 15 | echo "KAFKA_SCHEMA_REGISTRY is not defined" 16 | exit 4 17 | fi 18 | 19 | KAFKA_BROKERS=${KAFKA_BROKERS:-3} 20 | 21 | max_timeout=32 22 | 23 | tries=10 24 | timeout=1 25 | while true; do 26 | IFS=, read -r -a array <<< $(curl -s "${KAFKA_REST_PROXY}/brokers" | sed 's/^.*\[\(.*\)\]\}/\1/') 27 | LENGTH=${#array[@]} 28 | if [ "$LENGTH" -eq "$KAFKA_BROKERS" ]; then 29 | echo "Kafka brokers available." 30 | break 31 | fi 32 | tries=$((tries - 1)) 33 | if [ $tries -eq 0 ]; then 34 | echo "FAILED: KAFKA BROKERs NOT READY." 35 | exit 5 36 | fi 37 | echo "Kafka brokers or Kafka REST proxy not ready. Retrying in ${timeout} seconds." 38 | sleep $timeout 39 | if [ $timeout -lt $max_timeout ]; then 40 | timeout=$((timeout * 2)) 41 | fi 42 | done 43 | 44 | tries=10 45 | timeout=1 46 | while true; do 47 | if wget --spider -q "${KAFKA_SCHEMA_REGISTRY}/subjects" 2>/dev/null; then 48 | break 49 | fi 50 | tries=$((tries - 1)) 51 | if [ $tries -eq 0 ]; then 52 | echo "FAILED TO REACH SCHEMA REGISTRY. SCHEMAS NOT REGISTERED." 53 | exit 6 54 | fi 55 | echo "Failed to reach schema registry. Retrying in ${timeout} seconds." 56 | sleep $timeout 57 | if [ $timeout -lt $max_timeout ]; then 58 | timeout=$((timeout * 2)) 59 | fi 60 | done 61 | 62 | 63 | echo "Kafka is available. Ready to go!" 64 | 65 | # Start streams 66 | echo "===> Starting " "$@" "...." 67 | exec radar-backend -c /etc/radar.yml "$@" -------------------------------------------------------------------------------- /src/main/java/org/radarcns/config/DisconnectMonitorConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.config; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | 21 | /** 22 | * POJO representing a disconnection status monitor configuration 23 | */ 24 | public class DisconnectMonitorConfig extends MonitorConfig { 25 | private long timeout = 1800L; // 30 minutes 26 | 27 | @JsonProperty("alert_repeat_interval") 28 | private long alertRepeatInterval = 86400; // 1 day 29 | 30 | @JsonProperty("alert_repetitions") 31 | private int alertRepetitions = 0; 32 | 33 | public Long getTimeout() { 34 | return timeout; 35 | } 36 | 37 | public void setTimeout(Long timeout) { 38 | this.timeout = timeout; 39 | } 40 | 41 | public long getAlertRepeatInterval() { 42 | return alertRepeatInterval; 43 | } 44 | 45 | public void setAlertRepeatInterval(long alertRepeatInterval) { 46 | this.alertRepeatInterval = alertRepeatInterval; 47 | } 48 | 49 | public int getAlertRepetitions() { 50 | return alertRepetitions; 51 | } 52 | 53 | public void setAlertRepetitions(int alertRepetitions) { 54 | this.alertRepetitions = alertRepetitions; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/config/RadarPropertyHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.config; 18 | 19 | import java.io.IOException; 20 | import org.radarcns.util.PersistentStateStore; 21 | 22 | /** 23 | * Interface that handles YAML configuration file loading 24 | */ 25 | public interface RadarPropertyHandler { 26 | 27 | enum Priority { 28 | LOW("low"), NORMAL("normal"), HIGH("high"); 29 | 30 | private final String param; 31 | 32 | Priority(String param) { 33 | this.param = param; 34 | } 35 | 36 | public String getParam() { 37 | return param; 38 | } 39 | } 40 | 41 | ConfigRadar getRadarProperties(); 42 | 43 | void load(String pathFile) throws IOException; 44 | 45 | boolean isLoaded(); 46 | 47 | KafkaProperty getKafkaProperties(); 48 | 49 | /** 50 | * Create a {@link PersistentStateStore} if so configured in the radar properties. Notably, the 51 | * {@code persistence_path} must be set. 52 | * @return PersistentStateStore or null if none is configured. 53 | * @throws IOException if the persistence store cannot be reached. 54 | */ 55 | PersistentStateStore getPersistentStateStore() throws IOException; 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/util/PersistentStateStore.java: -------------------------------------------------------------------------------- 1 | package org.radarcns.util; 2 | 3 | import java.io.IOException; 4 | import org.radarcns.kafka.ObservationKey; 5 | 6 | /** 7 | * Store a state for a Kafka consumer. It may not handle maps which have complex objects as keys. 8 | * Use {@link #keyToString(ObservationKey)} and {@link #stringToKey(String)} to use ObservationKey 9 | * as a map key by serializing it to String. 10 | */ 11 | public interface PersistentStateStore { 12 | /** Retrieve a state. The default is returned if no existing state is found. 13 | * 14 | * @param groupId Kafka group ID of a consumer or producer. 15 | * @param clientId Kafka client ID of a consumer or producer. 16 | * @param stateDefault default state if none is found. 17 | * @param type of state to retrieve. 18 | * @throws IOException if the state cannot be deserialized. 19 | */ 20 | T retrieveState(String groupId, String clientId, T stateDefault) throws IOException; 21 | 22 | /** Store a state. 23 | * @param groupId Kafka group ID of a consumer or producer. 24 | * @param clientId Kafka client ID of a consumer or producer. 25 | * @param value state to store. 26 | * @throws IOException if the state cannot be serialized or persisted. 27 | */ 28 | void storeState(String groupId, String clientId, Object value) throws IOException; 29 | 30 | /** 31 | * Uniquely and efficiently serializes an observation key. It can be deserialized with 32 | * {@link #stringToKey(String)}. 33 | * @param key key to serialize 34 | * @return unique serialized form 35 | */ 36 | String keyToString(ObservationKey key); 37 | 38 | /** 39 | * Efficiently serializes an observation key serialized with 40 | * {@link #keyToString(ObservationKey)}. 41 | * 42 | * @param string serialized form 43 | * @return original measurement key 44 | */ 45 | ObservationKey stringToKey(String string); 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/stream/empatica/E4TemperatureStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.stream.empatica; 18 | 19 | import javax.annotation.Nonnull; 20 | import org.apache.kafka.streams.kstream.KStream; 21 | import org.radarcns.config.RadarPropertyHandler.Priority; 22 | import org.radarcns.kafka.AggregateKey; 23 | import org.radarcns.kafka.ObservationKey; 24 | import org.radarcns.passive.empatica.EmpaticaE4Temperature; 25 | import org.radarcns.stream.SensorStreamWorker; 26 | import org.radarcns.stream.StreamDefinition; 27 | import org.radarcns.stream.aggregator.NumericAggregate; 28 | 29 | /** 30 | * Definition of Kafka Stream for aggregating temperature values collected by Empatica E4. 31 | */ 32 | public class E4TemperatureStream extends SensorStreamWorker { 33 | @Override 34 | protected void initialize() { 35 | defineWindowedSensorStream("android_empatica_e4_temperature"); 36 | config.setDefaultPriority(Priority.NORMAL); 37 | } 38 | 39 | @Override 40 | protected KStream implementStream(StreamDefinition definition, 41 | @Nonnull KStream kstream) { 42 | return aggregateNumeric(definition, kstream, "temperature", 43 | EmpaticaE4Temperature.getClassSchema()); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/stream/empatica/E4BatteryLevelStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.stream.empatica; 18 | 19 | import javax.annotation.Nonnull; 20 | import org.apache.kafka.streams.kstream.KStream; 21 | import org.radarcns.config.RadarPropertyHandler.Priority; 22 | import org.radarcns.kafka.AggregateKey; 23 | import org.radarcns.kafka.ObservationKey; 24 | import org.radarcns.passive.empatica.EmpaticaE4BatteryLevel; 25 | import org.radarcns.stream.SensorStreamWorker; 26 | import org.radarcns.stream.StreamDefinition; 27 | import org.radarcns.stream.aggregator.NumericAggregate; 28 | 29 | /** 30 | * Kafka Stream for aggregating data about Empatica E4 battery level. 31 | */ 32 | public class E4BatteryLevelStream extends 33 | SensorStreamWorker { 34 | 35 | @Override 36 | protected void initialize() { 37 | defineWindowedSensorStream("android_empatica_e4_battery_level"); 38 | config.setDefaultPriority(Priority.LOW); 39 | } 40 | 41 | @Override 42 | protected KStream implementStream(StreamDefinition definition, 43 | @Nonnull KStream kstream) { 44 | return aggregateNumeric(definition, kstream, "batteryLevel", 45 | EmpaticaE4BatteryLevel.getClassSchema()); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/org/radarcns/util/RadarUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.util; 18 | 19 | import static org.junit.Assert.assertEquals; 20 | 21 | import org.apache.kafka.streams.kstream.Window; 22 | import org.apache.kafka.streams.kstream.Windowed; 23 | import org.apache.kafka.streams.kstream.internals.TimeWindow; 24 | import org.junit.Before; 25 | import org.junit.Test; 26 | import org.radarcns.kafka.AggregateKey; 27 | import org.radarcns.kafka.ObservationKey; 28 | 29 | public class RadarUtilsTest { 30 | 31 | private RadarUtilities radarUtilities; 32 | 33 | @Before 34 | public void setUp() { 35 | this.radarUtilities = new RadarUtilitiesImpl(); 36 | } 37 | 38 | @Test 39 | public void getWindowed() { 40 | String userId = "userId"; 41 | String sourceId = "sourceId"; 42 | 43 | ObservationKey measurementKey = new ObservationKey(); 44 | measurementKey.setUserId(userId); 45 | measurementKey.setSourceId(sourceId); 46 | 47 | Window window = new TimeWindow(1, 4); 48 | Windowed measurementKeyWindowed = new Windowed<>(measurementKey, window); 49 | 50 | AggregateKey windowedKey = radarUtilities.getWindowed(measurementKeyWindowed); 51 | 52 | assertEquals(windowedKey.getUserId(), userId); 53 | assertEquals(windowedKey.getSourceId(), sourceId); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/stream/empatica/E4BloodVolumePulseStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.stream.empatica; 18 | 19 | import javax.annotation.Nonnull; 20 | import org.apache.kafka.streams.kstream.KStream; 21 | import org.radarcns.config.RadarPropertyHandler.Priority; 22 | import org.radarcns.kafka.AggregateKey; 23 | import org.radarcns.kafka.ObservationKey; 24 | import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; 25 | import org.radarcns.stream.SensorStreamWorker; 26 | import org.radarcns.stream.StreamDefinition; 27 | import org.radarcns.stream.aggregator.NumericAggregate; 28 | 29 | /** 30 | * Kafka Stream for aggregating data about Blood Volume Pulse collected by Empatica E4. 31 | */ 32 | public class E4BloodVolumePulseStream extends 33 | SensorStreamWorker { 34 | @Override 35 | protected void initialize() { 36 | defineWindowedSensorStream("android_empatica_e4_blood_volume_pulse"); 37 | config.setDefaultPriority(Priority.HIGH); 38 | } 39 | 40 | @Override 41 | protected KStream implementStream(StreamDefinition definition, 42 | @Nonnull KStream kstream) { 43 | return aggregateNumeric(definition, kstream, "bloodVolumePulse", 44 | EmpaticaE4BloodVolumePulse.getClassSchema()); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/stream/empatica/E4AccelerationStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.stream.empatica; 18 | 19 | import javax.annotation.Nonnull; 20 | import org.apache.kafka.streams.kstream.KStream; 21 | import org.radarcns.config.RadarPropertyHandler.Priority; 22 | import org.radarcns.kafka.AggregateKey; 23 | import org.radarcns.kafka.ObservationKey; 24 | import org.radarcns.passive.empatica.EmpaticaE4Acceleration; 25 | import org.radarcns.stream.SensorStreamWorker; 26 | import org.radarcns.stream.StreamDefinition; 27 | import org.radarcns.stream.aggregator.AggregateList; 28 | 29 | /** 30 | * Definition of Kafka Stream for aggregating data collected by Empatica E4 Accelerometer sensor. 31 | */ 32 | public class E4AccelerationStream extends 33 | SensorStreamWorker { 34 | 35 | @Override 36 | protected void initialize() { 37 | defineWindowedSensorStream("android_empatica_e4_acceleration"); 38 | config.setDefaultPriority(Priority.HIGH); 39 | } 40 | 41 | @Override 42 | protected KStream implementStream( 43 | StreamDefinition definition, 44 | @Nonnull KStream kstream) { 45 | return aggregateFields(definition, kstream, new String[] {"x", "y", "z"}, 46 | EmpaticaE4Acceleration.getClassSchema()); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/util/Monitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.util; 18 | 19 | import java.util.concurrent.atomic.AtomicInteger; 20 | import org.slf4j.Logger; 21 | 22 | /** 23 | * Monitors a count and buffer variable by printing out their values and resetting them. 24 | */ 25 | public class Monitor implements Runnable { 26 | private final AtomicInteger count; 27 | private final Logger log; 28 | private final String message; 29 | 30 | /** 31 | * Monitor a counter. Log messages with the count and the given message are sent to given 32 | * logger, separated by a space. 33 | * @param log logger to log messages to 34 | * @param message message to append to the current count 35 | */ 36 | public Monitor(Logger log, String message) { 37 | if (log == null) { 38 | throw new IllegalArgumentException("Argument log may not be null"); 39 | } 40 | this.count = new AtomicInteger(0); 41 | this.log = log; 42 | this.message = message; 43 | } 44 | 45 | /** 46 | * Logs the current count and, if applicable buffer size. This resets the current count to 0. 47 | */ 48 | @Override 49 | public void run() { 50 | log.info("{} {}", count.getAndSet(0), message); 51 | } 52 | 53 | /** Increment the count by one. */ 54 | public void increment() { 55 | this.count.incrementAndGet(); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/stream/empatica/E4InterBeatIntervalStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.stream.empatica; 18 | 19 | import javax.annotation.Nonnull; 20 | import org.apache.kafka.streams.kstream.KStream; 21 | import org.radarcns.config.RadarPropertyHandler.Priority; 22 | import org.radarcns.kafka.AggregateKey; 23 | import org.radarcns.kafka.ObservationKey; 24 | import org.radarcns.passive.empatica.EmpaticaE4InterBeatInterval; 25 | import org.radarcns.stream.SensorStreamWorker; 26 | import org.radarcns.stream.StreamDefinition; 27 | import org.radarcns.stream.aggregator.NumericAggregate; 28 | 29 | /** 30 | * Definition of Kafka Stream for aggregating Inter Beat Interval values collected by Empatica E4. 31 | */ 32 | public class E4InterBeatIntervalStream extends 33 | SensorStreamWorker { 34 | 35 | @Override 36 | protected void initialize() { 37 | defineWindowedSensorStream("android_empatica_e4_inter_beat_interval"); 38 | config.setDefaultPriority(Priority.LOW); 39 | } 40 | 41 | @Override 42 | protected KStream implementStream(StreamDefinition definition, 43 | @Nonnull KStream kstream) { 44 | return aggregateNumeric(definition, kstream, "interBeatInterval", 45 | EmpaticaE4InterBeatInterval.getClassSchema()); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/util/serde/JsonSerializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.util.serde; 18 | 19 | import static org.radarcns.util.serde.RadarSerde.GENERIC_WRITER; 20 | 21 | import com.fasterxml.jackson.core.JsonProcessingException; 22 | import com.fasterxml.jackson.databind.ObjectWriter; 23 | import java.util.Map; 24 | import org.apache.kafka.common.serialization.Serializer; 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | public class JsonSerializer implements Serializer { 29 | private static final Logger logger = LoggerFactory.getLogger(JsonSerializer.class); 30 | 31 | private final ObjectWriter writer; 32 | 33 | public JsonSerializer() { 34 | this.writer = GENERIC_WRITER; 35 | } 36 | 37 | public JsonSerializer(Class cls) { 38 | this.writer = GENERIC_WRITER.forType(cls); 39 | } 40 | 41 | @Override 42 | public void configure(Map map, boolean b) { 43 | // no configuration needed 44 | } 45 | 46 | @Override 47 | public byte[] serialize(String topic, T t) { 48 | try { 49 | return writer.writeValueAsBytes(t); 50 | } catch (JsonProcessingException e) { 51 | logger.error("Cannot serialize value {} in topic {}", t, topic, e); 52 | return null; 53 | } 54 | } 55 | 56 | @Override 57 | public void close() { 58 | // noop 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/util/serde/JsonDeserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.util.serde; 18 | 19 | import static org.radarcns.util.serde.RadarSerde.GENERIC_READER; 20 | 21 | import com.fasterxml.jackson.databind.ObjectReader; 22 | import java.io.IOException; 23 | import java.util.Map; 24 | import org.apache.kafka.common.serialization.Deserializer; 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | public class JsonDeserializer implements Deserializer { 29 | private static final Logger logger = LoggerFactory.getLogger(JsonDeserializer.class); 30 | 31 | private final ObjectReader reader; 32 | 33 | public JsonDeserializer(Class deserializedClass) { 34 | reader = GENERIC_READER.forType(deserializedClass); 35 | } 36 | 37 | @Override 38 | @SuppressWarnings("unchecked") 39 | public void configure(Map map, boolean b) { 40 | // no configuration 41 | } 42 | 43 | @Override 44 | public T deserialize(String topic, byte[] bytes) { 45 | if (bytes == null) { 46 | return null; 47 | } 48 | 49 | try { 50 | return reader.readValue(bytes); 51 | } catch (IOException e) { 52 | logger.error("Failed to deserialize value for topic {}", topic, e); 53 | return null; 54 | } 55 | } 56 | 57 | @Override 58 | public void close() { 59 | // noop 60 | } 61 | } -------------------------------------------------------------------------------- /src/test/java/org/radarcns/util/EmailServerRule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.util; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | import javax.mail.MessagingException; 22 | import javax.mail.internet.MimeMessage; 23 | import org.junit.rules.ExternalResource; 24 | import org.subethamail.wiser.Wiser; 25 | import org.subethamail.wiser.WiserMessage; 26 | 27 | public class EmailServerRule extends ExternalResource { 28 | private final int port; 29 | private Wiser emailServer; 30 | 31 | public EmailServerRule() { 32 | this(25251); 33 | } 34 | 35 | public EmailServerRule(int port) { 36 | this.port = port; 37 | } 38 | 39 | @Override 40 | protected void before() { 41 | emailServer = new Wiser(port); 42 | emailServer.setHostname("localhost"); 43 | emailServer.start(); 44 | } 45 | 46 | @Override 47 | protected void after() { 48 | emailServer.stop(); 49 | } 50 | 51 | public List getMessages() throws MessagingException { 52 | List messages = emailServer.getMessages(); 53 | List mimeMessages = new ArrayList<>(messages.size()); 54 | for (WiserMessage message : messages) { 55 | mimeMessages.add(message.getMimeMessage()); 56 | } 57 | return mimeMessages; 58 | } 59 | 60 | public int getPort() { 61 | return port; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/stream/empatica/E4ElectroDermalActivityStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.stream.empatica; 18 | 19 | import javax.annotation.Nonnull; 20 | import org.apache.kafka.streams.kstream.KStream; 21 | import org.radarcns.config.RadarPropertyHandler.Priority; 22 | import org.radarcns.kafka.AggregateKey; 23 | import org.radarcns.kafka.ObservationKey; 24 | import org.radarcns.passive.empatica.EmpaticaE4ElectroDermalActivity; 25 | import org.radarcns.stream.SensorStreamWorker; 26 | import org.radarcns.stream.StreamDefinition; 27 | import org.radarcns.stream.aggregator.NumericAggregate; 28 | 29 | /** 30 | * Kafka Stream for aggregating data about electrodermal activity collected by Empatica E4. 31 | */ 32 | public class E4ElectroDermalActivityStream extends 33 | SensorStreamWorker { 34 | 35 | @Override 36 | protected void initialize() { 37 | defineWindowedSensorStream("android_empatica_e4_electrodermal_activity"); 38 | config.setDefaultPriority(Priority.HIGH); 39 | } 40 | 41 | @Override 42 | protected KStream implementStream(StreamDefinition definition, 43 | @Nonnull KStream kstream) { 44 | return aggregateNumeric(definition, kstream, "electroDermalActivity", 45 | EmpaticaE4ElectroDermalActivity.getClassSchema()); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/util/serde/RadarSerde.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.util.serde; 18 | 19 | import com.fasterxml.jackson.annotation.JsonAutoDetect; 20 | import com.fasterxml.jackson.annotation.PropertyAccessor; 21 | import com.fasterxml.jackson.databind.ObjectMapper; 22 | import com.fasterxml.jackson.databind.ObjectReader; 23 | import com.fasterxml.jackson.databind.ObjectWriter; 24 | import org.apache.kafka.common.serialization.Serde; 25 | import org.apache.kafka.common.serialization.Serdes; 26 | 27 | /** 28 | * It generates the jsonSerializer and jsonDeserializer for the given input class 29 | */ 30 | public class RadarSerde { 31 | private static final ObjectMapper MAPPER = new ObjectMapper(); 32 | static { 33 | MAPPER.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE); 34 | MAPPER.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); 35 | } 36 | static final ObjectWriter GENERIC_WRITER = MAPPER.writer(); 37 | static final ObjectReader GENERIC_READER = MAPPER.reader(); 38 | 39 | private final JsonSerializer jsonSerializer; 40 | private final JsonDeserializer jsonDeserializer; 41 | 42 | public RadarSerde(Class type) { 43 | this.jsonSerializer = new JsonSerializer<>(type); 44 | this.jsonDeserializer = new JsonDeserializer<>(type); 45 | } 46 | 47 | public Serde getSerde() { 48 | return Serdes.serdeFrom(jsonSerializer, jsonDeserializer); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/stream/phone/PhoneUsageAggregationStream.java: -------------------------------------------------------------------------------- 1 | package org.radarcns.stream.phone; 2 | 3 | import java.time.Duration; 4 | import javax.annotation.Nonnull; 5 | import org.apache.kafka.streams.kstream.KStream; 6 | import org.radarcns.config.RadarPropertyHandler.Priority; 7 | import org.radarcns.kafka.AggregateKey; 8 | import org.radarcns.kafka.ObservationKey; 9 | import org.radarcns.passive.phone.PhoneUsageEvent; 10 | import org.radarcns.stream.SensorStreamWorker; 11 | import org.radarcns.stream.StreamDefinition; 12 | import org.radarcns.stream.aggregator.PhoneUsageAggregate; 13 | import org.radarcns.util.serde.RadarSerdes; 14 | 15 | /** 16 | * Created by piotrzakrzewski on 26/07/2017. 17 | */ 18 | public class PhoneUsageAggregationStream extends 19 | SensorStreamWorker { 20 | @Override 21 | protected void initialize() { 22 | defineStream( 23 | "android_phone_usage_event_output", 24 | "android_phone_usage_event_aggregated", 25 | Duration.ofDays(1)); 26 | config.setDefaultPriority(Priority.LOW); 27 | } 28 | 29 | @Override 30 | protected KStream implementStream( 31 | StreamDefinition definition, 32 | @Nonnull KStream kstream) { 33 | return kstream.groupBy(PhoneUsageAggregationStream::temporaryKey) 34 | .windowedBy(definition.getTimeWindows()) 35 | .aggregate( 36 | PhoneUsageCollector::new, 37 | (k, v, valueCollector) -> valueCollector.update(v), 38 | RadarSerdes.materialized(definition.getStateStoreName(), 39 | RadarSerdes.getInstance().getPhoneUsageCollector())) 40 | .toStream() 41 | .map(utilities::phoneCollectorToAvro); 42 | } 43 | 44 | private static TemporaryPackageKey temporaryKey(ObservationKey key, PhoneUsageEvent value) { 45 | return new TemporaryPackageKey(key.getProjectId(), key.getUserId(), key.getSourceId(), 46 | value.getPackageName()); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/stream/empatica/E4HeartRateStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.stream.empatica; 18 | 19 | import static org.radarcns.util.Serialization.floatToDouble; 20 | 21 | import javax.annotation.Nonnull; 22 | import org.apache.kafka.streams.kstream.KStream; 23 | import org.radarcns.config.RadarPropertyHandler.Priority; 24 | import org.radarcns.kafka.AggregateKey; 25 | import org.radarcns.kafka.ObservationKey; 26 | import org.radarcns.passive.empatica.EmpaticaE4InterBeatInterval; 27 | import org.radarcns.stream.SensorStreamWorker; 28 | import org.radarcns.stream.StreamDefinition; 29 | import org.radarcns.stream.aggregator.NumericAggregate; 30 | 31 | /** 32 | * Kafka Stream for computing and aggregating Heart Rate values collected by Empatica E4. 33 | * It is used by converting inter beat interval (input) to heart rate. 34 | */ 35 | public class E4HeartRateStream extends 36 | SensorStreamWorker { 37 | @Override 38 | protected void initialize() { 39 | defineWindowedSensorStream( 40 | "android_empatica_e4_inter_beat_interval", 41 | "android_empatica_e4_heart_rate"); 42 | config.setDefaultPriority(Priority.LOW); 43 | } 44 | 45 | protected KStream implementStream( 46 | StreamDefinition definition, 47 | @Nonnull KStream kstream) { 48 | return aggregateCustomNumeric(definition, kstream, 49 | v -> 60d / floatToDouble(v.getInterBeatInterval()), "heartRate"); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/util/RadarSingletonFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.util; 18 | 19 | import org.radarcns.config.RadarPropertyHandler; 20 | import org.radarcns.config.RadarPropertyHandlerImpl; 21 | 22 | /** 23 | * SingletonFactory of RadarBackend project. This factory composites all singleton objects that need 24 | * to be maintained in this project and provides a gateway to get singleton objects 25 | */ 26 | public final class RadarSingletonFactory { 27 | 28 | private static RadarUtilities utilities; 29 | 30 | private static RadarPropertyHandler propertyHandler; 31 | 32 | private static final Object SYNC_OBJECT = new Object(); 33 | 34 | private RadarSingletonFactory() { 35 | // utility class 36 | } 37 | 38 | /** 39 | * Returns the singleton object of RadarUtilities 40 | * 41 | * @return a RadarUtilities object 42 | */ 43 | public static RadarUtilities getRadarUtilities() { 44 | synchronized (SYNC_OBJECT) { 45 | if (utilities == null) { 46 | utilities = new RadarUtilitiesImpl(); 47 | } 48 | return utilities; 49 | } 50 | } 51 | 52 | /** 53 | * Returns the singleton object of RadarPropertyHandler 54 | * 55 | * @return a RadarPropertyHandler object 56 | */ 57 | public static RadarPropertyHandler getRadarPropertyHandler() { 58 | synchronized (SYNC_OBJECT) { 59 | if (propertyHandler == null) { 60 | propertyHandler = new RadarPropertyHandlerImpl(); 61 | } 62 | return propertyHandler; 63 | } 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/util/RadarUtilities.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.util; 18 | 19 | import org.apache.kafka.streams.KeyValue; 20 | import org.apache.kafka.streams.kstream.Windowed; 21 | import org.radarcns.kafka.AggregateKey; 22 | import org.radarcns.kafka.ObservationKey; 23 | import org.radarcns.stream.aggregator.AggregateList; 24 | import org.radarcns.stream.aggregator.NumericAggregate; 25 | import org.radarcns.stream.aggregator.PhoneUsageAggregate; 26 | import org.radarcns.stream.collector.AggregateListCollector; 27 | import org.radarcns.stream.collector.NumericAggregateCollector; 28 | import org.radarcns.stream.phone.PhoneUsageCollector; 29 | import org.radarcns.stream.phone.TemporaryPackageKey; 30 | 31 | /** 32 | * Interface that facades all utility functions that are required to support RadarBackend features. 33 | */ 34 | public interface RadarUtilities { 35 | 36 | /** 37 | * Creates a AggregateKey for a window of ObservationKey. 38 | * @param window Windowed measurement keys 39 | * @return relevant AggregateKey 40 | */ 41 | AggregateKey getWindowed(Windowed window); 42 | 43 | AggregateKey getWindowedTuple(Windowed window); 44 | 45 | KeyValue listCollectorToAvro( 46 | Windowed window, AggregateListCollector collector); 47 | 48 | KeyValue numericCollectorToAvro( 49 | Windowed window, NumericAggregateCollector collector); 50 | 51 | KeyValue phoneCollectorToAvro( 52 | Windowed window, PhoneUsageCollector collector); 53 | } 54 | -------------------------------------------------------------------------------- /src/test/java/org/radarcns/stream/StreamDefinitionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.stream; 18 | 19 | import static org.junit.Assert.assertEquals; 20 | import static org.junit.Assert.assertFalse; 21 | import static org.junit.Assert.assertTrue; 22 | import static org.radarcns.stream.AbstractStreamWorker.OUTPUT_LABEL; 23 | 24 | import java.util.regex.Pattern; 25 | import org.junit.Test; 26 | import org.radarcns.topic.KafkaTopic; 27 | 28 | public class StreamDefinitionTest { 29 | private static final Pattern TOPIC_PATTERN = Pattern.compile("^[A-Za-z0-9_-]+$"); 30 | 31 | private static final String INPUT = "android_empatica_e4_blood_volume_pulse"; 32 | private static final String OUTPUT = INPUT + OUTPUT_LABEL; 33 | 34 | @Test 35 | public void nameValidation() { 36 | KafkaTopic inputTopic = new KafkaTopic(INPUT); 37 | KafkaTopic outputTopic = new KafkaTopic(OUTPUT); 38 | 39 | StreamDefinition definition = new StreamDefinition(inputTopic, outputTopic); 40 | 41 | assertTrue(TOPIC_PATTERN.matcher(definition.getStateStoreName()).matches()); 42 | assertEquals("From-android_empatica_e4_blood_volume_pulse" 43 | + "-To-android_empatica_e4_blood_volume_pulse_output", 44 | definition.getStateStoreName()); 45 | } 46 | 47 | @Test(expected = IllegalArgumentException.class) 48 | public void faultyNameValidation() { 49 | KafkaTopic inputTopic = new KafkaTopic(INPUT + "$"); 50 | KafkaTopic outputTopic = new KafkaTopic(OUTPUT); 51 | 52 | StreamDefinition definition = new StreamDefinition(inputTopic, outputTopic); 53 | assertFalse(TOPIC_PATTERN.matcher(definition.getStateStoreName()).matches()); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/org/radarcns/config/RadarBackendOptionsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.config; 18 | 19 | import static org.junit.Assert.assertArrayEquals; 20 | import static org.junit.Assert.assertEquals; 21 | import static org.junit.Assert.assertNull; 22 | 23 | import org.apache.commons.cli.ParseException; 24 | import org.junit.Test; 25 | 26 | public class RadarBackendOptionsTest { 27 | @Test 28 | public void empty() throws ParseException { 29 | RadarBackendOptions opts = RadarBackendOptions.parse(new String[] {}); 30 | assertNull(opts.getSubCommand()); 31 | assertNull(opts.getSubCommandArgs()); 32 | assertNull(opts.getPropertyPath()); 33 | } 34 | 35 | @Test 36 | public void withConfig() throws ParseException { 37 | RadarBackendOptions opts = RadarBackendOptions.parse(new String[] {"-c", "cfg"}); 38 | assertNull(opts.getSubCommand()); 39 | assertNull(opts.getSubCommandArgs()); 40 | assertEquals("cfg", opts.getPropertyPath()); 41 | } 42 | 43 | @Test 44 | public void withSubcommand() throws ParseException { 45 | RadarBackendOptions opts = RadarBackendOptions.parse(new String[] {"-c", "cfg", "stream"}); 46 | assertEquals("stream", opts.getSubCommand()); 47 | assertArrayEquals(new String[0], opts.getSubCommandArgs()); 48 | assertEquals("cfg", opts.getPropertyPath()); 49 | } 50 | 51 | @Test 52 | public void withSubcommandArgs() throws ParseException { 53 | RadarBackendOptions opts = RadarBackendOptions.parse( 54 | new String[] {"monitor", "battery"}); 55 | assertEquals("monitor", opts.getSubCommand()); 56 | assertArrayEquals(new String[] {"battery"}, opts.getSubCommandArgs()); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/integrationTest/resources/org/radarcns/kafka/radar.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | released: 2016-11-27 3 | 4 | 5 | #============================== Zookeeper ==============================# 6 | #List of Zookeeper instances 7 | zookeeper: 8 | - host: zookeeper-1 9 | port: 2181 10 | 11 | #================================ Kafka ================================# 12 | #List of Kafka brokers 13 | broker: 14 | - host: kafka-1 15 | port: 9092 16 | - host: kafka-2 17 | port: 9092 18 | - host: kafka-3 19 | port: 9092 20 | 21 | #=========================== Schema Registry ===========================# 22 | #List of Schema Registry instances 23 | schema_registry: 24 | - host: schema-registry-1 25 | port: 8081 26 | protocol: http 27 | 28 | battery_monitor: 29 | notify: 30 | - project_id: test 31 | email_address: 32 | - notifier@email 33 | level: LOW 34 | topics: 35 | - android_empatica_e4_battery_level 36 | 37 | disconnect_monitor: 38 | notify: 39 | - project_id: test 40 | email_address: 41 | - notifier@email 42 | email_host: localhost 43 | email_port: 25 44 | email_user: sender@email 45 | topics: 46 | - android_empatica_e4_temperature 47 | timeout: 10 48 | alert_repeat_interval: 20 49 | alert_repetitions: 2 50 | 51 | extras: 52 | streaming_timeout_ms: 1000 53 | 54 | stream: 55 | threads_per_priority: 56 | low: 1 57 | normal: 2 58 | high: 4 59 | 60 | properties: 61 | max.request.size: 3500042 #Set message.max.bytes for kafka brokers higher than or equal to this value 62 | retries: 15 63 | session.timeout.ms: 20000 64 | cache.max.bytes.buffering: 1 65 | 66 | streams: 67 | - class: org.radarcns.stream.empatica.E4AccelerationStream 68 | - class: org.radarcns.stream.empatica.E4BatteryLevelStream 69 | - class: org.radarcns.stream.empatica.E4BloodVolumePulseStream 70 | - class: org.radarcns.stream.empatica.E4ElectroDermalActivityStream 71 | - class: org.radarcns.stream.empatica.E4HeartRateStream 72 | - class: org.radarcns.stream.empatica.E4InterBeatIntervalStream 73 | - class: org.radarcns.stream.empatica.E4TemperatureStream 74 | - class: org.radarcns.stream.phone.PhoneAccelerationStream 75 | - class: org.radarcns.stream.phone.PhoneBatteryStream 76 | - class: org.radarcns.stream.phone.PhoneUsageStream 77 | - class: org.radarcns.stream.phone.PhoneUsageAggregationStream 78 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/stream/DeviceTimestampExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.stream; 18 | 19 | import org.apache.avro.AvroRuntimeException; 20 | import org.apache.avro.Schema; 21 | import org.apache.avro.generic.IndexedRecord; 22 | import org.apache.kafka.clients.consumer.ConsumerRecord; 23 | import org.apache.kafka.streams.processor.TimestampExtractor; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | /** Custom TimestampExtractor for TimeWindows Streams. */ 28 | public class DeviceTimestampExtractor implements TimestampExtractor { 29 | 30 | private static final Logger log = LoggerFactory.getLogger(DeviceTimestampExtractor.class); 31 | 32 | /** 33 | * Return the timeReceived value converted in long. timeReceived is the timestamp at which the 34 | * device has collected the sample. 35 | * 36 | * @throws RuntimeException if timeReceived is not present inside the analysed record 37 | */ 38 | @Override 39 | public long extract(ConsumerRecord record, long previousTimestamp) { 40 | IndexedRecord value = (IndexedRecord) record.value(); 41 | Schema recordSchema = value.getSchema(); 42 | 43 | try { 44 | Schema.Field field = recordSchema.getField("timeReceived"); 45 | if (value.get(field.pos()) instanceof Double) { 46 | return (long) (1000d * (Double) value.get(field.pos())); 47 | } else { 48 | log.error("timeReceived id not a Double in {}", record); 49 | } 50 | 51 | } catch (AvroRuntimeException e) { 52 | log.error("Cannot extract timeReceived form {}", record, e); 53 | } 54 | 55 | throw new RuntimeException("Impossible to extract timeReceived from " + record); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /gradle/test.gradle: -------------------------------------------------------------------------------- 1 | configurations { 2 | integrationTestCompile.extendsFrom testCompile 3 | integrationTestRuntime.extendsFrom testRuntime 4 | } 5 | 6 | //---------------------------------------------------------------------------// 7 | // Source Set // 8 | //---------------------------------------------------------------------------// 9 | sourceSets { 10 | integrationTest { 11 | java { 12 | compileClasspath += main.output + main.compileClasspath + test.output + test.compileClasspath 13 | runtimeClasspath += main.output + test.output + test.compileClasspath 14 | } 15 | resources { 16 | srcDir 'src/integrationTest/resources' 17 | } 18 | } 19 | } 20 | 21 | dependencies { 22 | // Testing 23 | testCompile group: 'junit', name: 'junit', version: junitVersion 24 | testCompile group: 'org.mockito', name: 'mockito-core', version: mockitoVersion 25 | testCompile group: 'org.hamcrest', name: 'hamcrest-all', version: hamcrestVersion 26 | 27 | // Mock mail server 28 | testCompile group: 'org.subethamail', name: 'subethasmtp', version: subethamailVersion 29 | 30 | integrationTestCompile group: 'org.radarcns', name: 'radar-schemas-tools', version: radarSchemasVersion 31 | testImplementation group: 'log4j', name: 'log4j', version: log4jVersion 32 | testImplementation group: 'org.slf4j', name: 'slf4j-log4j12', version: slf4jVersion 33 | integrationTestImplementation group: 'log4j', name: 'log4j', version: log4jVersion 34 | integrationTestImplementation group: 'org.slf4j', name: 'slf4j-log4j12', version: slf4jVersion 35 | } 36 | 37 | tasks.matching {it instanceof Test}.all { 38 | testLogging { 39 | showStandardStreams = true 40 | showExceptions = true 41 | showCauses = true 42 | showStackTraces = true 43 | exceptionFormat "full" 44 | } 45 | } 46 | 47 | task integrationTest(type: Test) { 48 | description = "Run integration tests (located in src/integrationTest/...)." 49 | testClassesDirs = sourceSets.integrationTest.output.classesDirs 50 | classpath = sourceSets.integrationTest.runtimeClasspath 51 | 52 | testLogging.events "skipped", "failed", "passed" 53 | } 54 | 55 | // Do not do integration test yet, but check if integration tests compile 56 | check.dependsOn integrationTestClasses 57 | 58 | test.testLogging.events "skipped", "failed" 59 | 60 | -------------------------------------------------------------------------------- /src/test/java/org/radarcns/util/PersistentStateStoreTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.util; 18 | 19 | import static org.hamcrest.MatcherAssert.assertThat; 20 | import static org.hamcrest.Matchers.equalTo; 21 | import static org.hamcrest.Matchers.hasEntry; 22 | import static org.hamcrest.core.Is.is; 23 | 24 | import java.io.File; 25 | import java.nio.file.Files; 26 | import java.util.Map; 27 | import org.junit.Rule; 28 | import org.junit.Test; 29 | import org.junit.rules.TemporaryFolder; 30 | import org.radarcns.kafka.ObservationKey; 31 | import org.radarcns.monitor.BatteryLevelMonitor.BatteryLevelState; 32 | 33 | public class PersistentStateStoreTest { 34 | @Rule 35 | public TemporaryFolder folder = new TemporaryFolder(); 36 | 37 | @Test 38 | public void retrieveState() throws Exception { 39 | File base = folder.newFolder(); 40 | YamlPersistentStateStore stateStore = new YamlPersistentStateStore(base); 41 | BatteryLevelState state = new BatteryLevelState(); 42 | ObservationKey key1 = new ObservationKey("test", "a", "b"); 43 | state.updateLevel(stateStore.keyToString(key1), 0.1f); 44 | stateStore.storeState("one", "two", state); 45 | 46 | File outputFile = new File(base, "one_two.yml"); 47 | assertThat(outputFile.exists(), is(true)); 48 | String rawFile = new String(Files.readAllBytes(outputFile.toPath())); 49 | assertThat(rawFile, equalTo("---\nlevels:\n test#a#b: 0.1\n")); 50 | 51 | YamlPersistentStateStore stateStore2 = new YamlPersistentStateStore(base); 52 | BatteryLevelState state2 = stateStore2.retrieveState("one", "two", new BatteryLevelState()); 53 | Map values = state2.getLevels(); 54 | assertThat(values, hasEntry(stateStore.keyToString(key1), 0.1f)); 55 | } 56 | } -------------------------------------------------------------------------------- /src/test/java/org/radarcns/util/serde/RadarSerdesTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.util.serde; 18 | 19 | import static org.junit.Assert.assertEquals; 20 | import static org.junit.Assert.assertNotNull; 21 | 22 | import org.apache.kafka.common.serialization.Serde; 23 | import org.junit.Before; 24 | import org.junit.Test; 25 | import org.radarcns.stream.collector.AggregateListCollector; 26 | import org.radarcns.stream.collector.NumericAggregateCollector; 27 | 28 | /** 29 | * Created by nivethika on 21-12-16. 30 | */ 31 | public class RadarSerdesTest { 32 | private RadarSerdes radarSerdes; 33 | 34 | @Before 35 | public void setUp() { 36 | this.radarSerdes = RadarSerdes.getInstance(); 37 | } 38 | 39 | @Test 40 | public void getDoubleCollector() { 41 | Serde valueCollectorSerde = this.radarSerdes.getNumericAggregateCollector(); 42 | assertNotNull(valueCollectorSerde); 43 | assertNotNull(valueCollectorSerde.serializer()); 44 | assertEquals(valueCollectorSerde.serializer().getClass(), JsonSerializer.class); 45 | assertNotNull(valueCollectorSerde.deserializer()); 46 | assertEquals(valueCollectorSerde.deserializer().getClass(), JsonDeserializer.class); 47 | } 48 | 49 | @Test 50 | public void getDoubleArrayCollector() { 51 | Serde doubleArrayCollector = this.radarSerdes.getAggregateListCollector(); 52 | assertNotNull(doubleArrayCollector); 53 | assertNotNull(doubleArrayCollector.serializer()); 54 | assertEquals(doubleArrayCollector.serializer().getClass(), JsonSerializer.class); 55 | assertNotNull(doubleArrayCollector.deserializer()); 56 | assertEquals(doubleArrayCollector.deserializer().getClass(), JsonDeserializer.class); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/test/java/org/radarcns/util/EmailSenderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.util; 18 | 19 | import static org.junit.Assert.assertEquals; 20 | 21 | import java.io.IOException; 22 | import java.util.Collections; 23 | import java.util.List; 24 | import javax.mail.Address; 25 | import javax.mail.Message.RecipientType; 26 | import javax.mail.MessagingException; 27 | import javax.mail.internet.MimeMessage; 28 | import org.junit.Rule; 29 | import org.junit.Test; 30 | 31 | public class EmailSenderTest { 32 | @Rule 33 | public EmailServerRule emailServer = new EmailServerRule(2525); 34 | 35 | @Test 36 | public void testEmail() throws MessagingException, IOException { 37 | EmailSender sender = new EmailSender("localhost", 2525, "no-reply@radar-cns.org", 38 | Collections.singletonList("test@radar-cns.org")); 39 | 40 | assertEquals(Collections.emptyList(), emailServer.getMessages()); 41 | 42 | sender.sendEmail("hi", "it's me"); 43 | 44 | List messages = emailServer.getMessages(); 45 | assertEquals(1, messages.size()); 46 | MimeMessage mime = messages.get(0); 47 | 48 | assertEquals(1, mime.getFrom().length); 49 | assertEquals("no-reply@radar-cns.org", mime.getFrom()[0].toString()); 50 | 51 | Address[] to = mime.getRecipients(RecipientType.TO); 52 | assertEquals(1, to.length); 53 | assertEquals("test@radar-cns.org", to[0].toString()); 54 | 55 | assertEquals("hi", mime.getSubject()); 56 | assertEquals("it's me", mime.getContent().toString().trim()); 57 | } 58 | 59 | @Test(expected = IOException.class) 60 | public void testEmailNonExisting() throws MessagingException, IOException { 61 | EmailSender sender = new EmailSender("non-existing-host", 2525, "no-reply@radar-cns.org", 62 | Collections.singletonList("test@radar-cns.org")); 63 | } 64 | } -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/util/serde/RadarSerdes.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.util.serde; 18 | 19 | import org.apache.kafka.common.serialization.Serde; 20 | import org.apache.kafka.common.utils.Bytes; 21 | import org.apache.kafka.streams.kstream.Materialized; 22 | import org.apache.kafka.streams.state.WindowStore; 23 | import org.radarcns.stream.collector.AggregateListCollector; 24 | import org.radarcns.stream.collector.NumericAggregateCollector; 25 | import org.radarcns.stream.phone.PhoneUsageCollector; 26 | 27 | /** 28 | * Set of Serde useful for Kafka Streams 29 | */ 30 | public final class RadarSerdes { 31 | private final Serde numericCollector; 32 | private final Serde aggregateListCollector; 33 | private final Serde phoneUsageCollector; 34 | 35 | private static RadarSerdes instance = new RadarSerdes(); 36 | 37 | public static RadarSerdes getInstance() { 38 | return instance; 39 | } 40 | 41 | private RadarSerdes() { 42 | numericCollector = new RadarSerde<>(NumericAggregateCollector.class).getSerde(); 43 | aggregateListCollector = new RadarSerde<>(AggregateListCollector.class).getSerde(); 44 | phoneUsageCollector = new RadarSerde<>(PhoneUsageCollector.class).getSerde(); 45 | } 46 | 47 | public Serde getNumericAggregateCollector() { 48 | return numericCollector; 49 | } 50 | 51 | public Serde getAggregateListCollector() { 52 | return aggregateListCollector; 53 | } 54 | 55 | public Serde getPhoneUsageCollector() { 56 | return phoneUsageCollector; 57 | } 58 | 59 | public static Materialized> materialized(String name, Serde valueSerde) { 60 | Materialized> store = Materialized.as(name); 61 | return store.withValueSerde(valueSerde); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/test/java/org/radarcns/stream/phone/PlayStoreLookupTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.stream.phone; 18 | 19 | import static org.junit.Assert.assertEquals; 20 | 21 | import java.io.IOException; 22 | import java.util.Arrays; 23 | import java.util.Collection; 24 | import org.junit.Test; 25 | import org.junit.runner.RunWith; 26 | import org.junit.runners.Parameterized; 27 | import org.junit.runners.Parameterized.Parameters; 28 | import org.radarcns.stream.phone.PlayStoreLookup.AppCategory; 29 | 30 | @RunWith(Parameterized.class) 31 | public class PlayStoreLookupTest { 32 | @Parameters(name = "{index}: {0}={1}") 33 | public static Collection data() { 34 | return Arrays.asList(new Object[][] { 35 | { "nl.nos.app", "NEWS_AND_MAGAZINES" } 36 | ,{ "nl.thehyve.transmartclient", "MEDICAL" } 37 | ,{ "com.twitter.android", "NEWS_AND_MAGAZINES" } 38 | ,{ "com.facebook.katana", "SOCIAL" } 39 | ,{ "com.nintendo.zara", "GAME_ACTION" } 40 | ,{ "com.duolingo", "EDUCATION" } 41 | ,{ "com.whatsapp", "COMMUNICATION" } 42 | ,{ "com.alibaba.aliexpresshd", "SHOPPING" } 43 | ,{ "com.google.android.wearable.app", "COMMUNICATION" } 44 | ,{ "com.strava", "HEALTH_AND_FITNESS" } 45 | ,{ "com.android.chrome", "COMMUNICATION" } 46 | ,{ "com.google.android.youtube", "VIDEO_PLAYERS" } 47 | ,{ "com.android.systemui", null } 48 | ,{ "abc.abc", null } 49 | }); 50 | } 51 | 52 | private final String fInputPackageName; 53 | private final String fExpectedCategory; 54 | 55 | public PlayStoreLookupTest(String input, String expected) { 56 | fInputPackageName = input; 57 | fExpectedCategory = expected; 58 | } 59 | 60 | @Test 61 | public void fetchCategoryTest() throws IOException { 62 | AppCategory result = PlayStoreLookup.fetchCategory(fInputPackageName); 63 | assertEquals(fExpectedCategory, result.getCategoryName()); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.stream.phone; 18 | 19 | import javax.annotation.Nonnull; 20 | import org.apache.kafka.streams.kstream.KStream; 21 | import org.radarcns.config.RadarPropertyHandler.Priority; 22 | import org.radarcns.kafka.ObservationKey; 23 | import org.radarcns.passive.phone.PhoneUsageEvent; 24 | import org.radarcns.stream.SensorStreamWorker; 25 | import org.radarcns.stream.StreamDefinition; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | public class PhoneUsageStream extends SensorStreamWorker { 30 | private static final Logger logger = LoggerFactory.getLogger(PhoneUsageStream.class); 31 | 32 | // 1 day until an item is refreshed 33 | private static final int CACHE_TIMEOUT = 24 * 3600; 34 | 35 | // Do not cache more than 1 million elements, for memory consumption reasons 36 | private static final int MAX_CACHE_SIZE = 1_000_000; 37 | 38 | private final PlayStoreLookup playStoreLookup; 39 | 40 | public PhoneUsageStream() { 41 | playStoreLookup = new PlayStoreLookup(CACHE_TIMEOUT, MAX_CACHE_SIZE); 42 | } 43 | 44 | @Override 45 | protected void initialize() { 46 | defineSensorStream("android_phone_usage_event"); 47 | config.setDefaultPriority(Priority.LOW); 48 | } 49 | 50 | @Override 51 | protected KStream implementStream(StreamDefinition definition, 52 | @Nonnull KStream kstream) { 53 | return kstream 54 | .mapValues(value -> { 55 | String packageName = value.getPackageName(); 56 | PlayStoreLookup.AppCategory category = playStoreLookup.lookupCategory(packageName); 57 | logger.info("Looked up {}: {}", packageName, category.getCategoryName()); 58 | value.setCategoryName(category.getCategoryName()); 59 | value.setCategoryNameFetchTime(category.getFetchTimeStamp()); 60 | return value; 61 | }); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/producer/MockProducerCommand.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.producer; 18 | 19 | import java.io.File; 20 | import java.io.IOException; 21 | import org.apache.avro.SchemaValidationException; 22 | import org.radarcns.config.ConfigRadar; 23 | import org.radarcns.config.MockConfig; 24 | import org.radarcns.config.RadarBackendOptions; 25 | import org.radarcns.config.RadarPropertyHandler; 26 | import org.radarcns.config.SubCommand; 27 | import org.radarcns.config.YamlConfigLoader; 28 | import org.radarcns.mock.MockProducer; 29 | import org.radarcns.mock.config.BasicMockConfig; 30 | import org.slf4j.Logger; 31 | import org.slf4j.LoggerFactory; 32 | 33 | public class MockProducerCommand implements SubCommand { 34 | private static final Logger logger = LoggerFactory.getLogger(MockProducerCommand.class); 35 | private final MockProducer producer; 36 | 37 | public MockProducerCommand(RadarBackendOptions options, 38 | RadarPropertyHandler radarPropertyHandler) throws IOException { 39 | ConfigRadar radar = radarPropertyHandler.getRadarProperties(); 40 | 41 | BasicMockConfig producerConfig = new BasicMockConfig(); 42 | 43 | File mockFile = options.getMockFile(); 44 | 45 | if (mockFile != null) { 46 | MockConfig mockConfig = new YamlConfigLoader().load(mockFile, MockConfig.class); 47 | producerConfig.setData(mockConfig.getData()); 48 | } else { 49 | producerConfig.setNumberOfDevices(options.getNumMockDevices()); 50 | } 51 | producerConfig.setRestProxy(radar.getRestProxy()); 52 | producerConfig.setSchemaRegistry(radar.getSchemaRegistry().get(0)); 53 | producerConfig.setProducerMode(options.isMockDirect() ? "direct" : "rest"); 54 | producer = new MockProducer(producerConfig); 55 | } 56 | 57 | @Override 58 | public void start() throws IOException { 59 | producer.start(); 60 | } 61 | 62 | @Override 63 | public void shutdown() throws IOException, InterruptedException { 64 | try { 65 | producer.shutdown(); 66 | } catch (SchemaValidationException e) { 67 | logger.error("Data did not match schema", e); 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/config/MonitorConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.config; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | import java.util.List; 21 | 22 | /** 23 | * POJO representing a monitor configuration 24 | */ 25 | public class MonitorConfig { 26 | @JsonProperty("notify") 27 | private List notifyConfig; 28 | 29 | @JsonProperty("email_host") 30 | private String emailHost; 31 | 32 | @JsonProperty("email_port") 33 | private int emailPort; 34 | 35 | @JsonProperty("email_user") 36 | private String emailUser; 37 | 38 | @JsonProperty("log_interval") 39 | private int logInterval = 1000; 40 | 41 | private List topics; 42 | 43 | @JsonProperty("message") 44 | private String message = null; 45 | 46 | public List getNotifyConfig() { 47 | return notifyConfig; 48 | } 49 | 50 | public void setNotifyConfig(List notifyConfig) { 51 | this.notifyConfig = notifyConfig; 52 | } 53 | 54 | public List getTopics() { 55 | return topics; 56 | } 57 | 58 | public void setTopics(List topics) { 59 | this.topics = topics; 60 | } 61 | 62 | public String getEmailHost() { 63 | return emailHost; 64 | } 65 | 66 | public void setEmailHost(String emailHost) { 67 | this.emailHost = emailHost; 68 | } 69 | 70 | public int getEmailPort() { 71 | return emailPort; 72 | } 73 | 74 | public void setEmailPort(int emailPort) { 75 | this.emailPort = emailPort; 76 | } 77 | 78 | public String getEmailUser() { 79 | return emailUser; 80 | } 81 | 82 | public void setEmailUser(String emailUser) { 83 | this.emailUser = emailUser; 84 | } 85 | 86 | public int getLogInterval() { 87 | return logInterval; 88 | } 89 | 90 | public void setLogInterval(int logInterval) { 91 | this.logInterval = logInterval; 92 | } 93 | 94 | public String getMessage() { 95 | return message; 96 | } 97 | 98 | public void setMessage(String message) { 99 | this.message = message; 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/util/RadarThreadFactoryBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.util; 18 | 19 | import java.util.concurrent.ThreadFactory; 20 | import java.util.concurrent.atomic.AtomicLong; 21 | import javax.annotation.Nonnull; 22 | 23 | /** 24 | * Created by Francesco Nobilia on 07/10/2016. 25 | */ 26 | public class RadarThreadFactoryBuilder { 27 | 28 | private String namePrefix = null; 29 | private boolean daemon = false; 30 | private int priority = Thread.NORM_PRIORITY; 31 | 32 | public RadarThreadFactoryBuilder setNamePrefix(@Nonnull String namePrefix) { 33 | if (namePrefix == null) { 34 | throw new IllegalArgumentException("namePrefix cannot be null"); 35 | } 36 | this.namePrefix = namePrefix; 37 | return this; 38 | } 39 | 40 | public RadarThreadFactoryBuilder setDaemon(boolean daemon) { 41 | this.daemon = daemon; 42 | return this; 43 | } 44 | 45 | public RadarThreadFactoryBuilder setPriority(int priority) { 46 | if (priority < Thread.MIN_PRIORITY) { 47 | throw new IllegalArgumentException("Thread priority " + priority 48 | + " must be >= " + Thread.MIN_PRIORITY); 49 | } 50 | 51 | if (priority > Thread.MAX_PRIORITY) { 52 | throw new IllegalArgumentException("Thread priority " + priority 53 | + " must be <= " + Thread.MAX_PRIORITY); 54 | } 55 | 56 | this.priority = priority; 57 | return this; 58 | } 59 | 60 | public ThreadFactory build() { 61 | return build(this); 62 | } 63 | 64 | private static ThreadFactory build(RadarThreadFactoryBuilder builder) { 65 | final String namePrefix = builder.namePrefix; 66 | final boolean daemon = builder.daemon; 67 | final int priority = builder.priority; 68 | 69 | final AtomicLong count = new AtomicLong(0); 70 | 71 | return runnable -> { 72 | Thread thread = new Thread(runnable); 73 | if (namePrefix != null) { 74 | thread.setName(namePrefix + "-" + count.getAndIncrement()); 75 | } 76 | thread.setDaemon(daemon); 77 | thread.setPriority(priority); 78 | 79 | return thread; 80 | }; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/test/java/org/radarcns/stream/SensorStreamWorkerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.stream; 18 | 19 | import static org.mockito.ArgumentMatchers.eq; 20 | import static org.mockito.Mockito.any; 21 | import static org.mockito.Mockito.doCallRealMethod; 22 | import static org.mockito.Mockito.mock; 23 | import static org.mockito.Mockito.times; 24 | import static org.mockito.Mockito.verify; 25 | import static org.mockito.Mockito.when; 26 | 27 | import java.io.IOException; 28 | import java.util.stream.Stream; 29 | import org.apache.kafka.streams.kstream.KStream; 30 | import org.junit.Before; 31 | import org.junit.Test; 32 | import org.radarcns.config.KafkaProperty; 33 | import org.radarcns.config.RadarPropertyHandler; 34 | import org.radarcns.config.SingleStreamConfig; 35 | import org.radarcns.topic.KafkaTopic; 36 | import org.radarcns.util.RadarSingletonFactory; 37 | 38 | /** 39 | * Created by nivethika on 20-12-16. 40 | */ 41 | public class SensorStreamWorkerTest { 42 | private SensorStreamWorker aggregator; 43 | @Before 44 | public void setUp() { 45 | aggregator = mock(SensorStreamWorker.class); 46 | } 47 | 48 | @SuppressWarnings("unchecked") 49 | @Test 50 | public void getBuilder() throws IOException { 51 | String topicName = "TESTTopic"; 52 | StreamDefinition sensorTopic = new StreamDefinition(new KafkaTopic(topicName), new KafkaTopic(topicName + "_output")); 53 | when(aggregator.getStreamDefinitions()).thenReturn(Stream.of(sensorTopic)); 54 | 55 | RadarPropertyHandler propertyHandler = RadarSingletonFactory.getRadarPropertyHandler(); 56 | propertyHandler.load("src/test/resources/config/radar.yml"); 57 | KafkaProperty kafkaProperty = propertyHandler.getKafkaProperties(); 58 | when(aggregator.getStreamProperties(eq(sensorTopic))).thenReturn( 59 | kafkaProperty.getStreamProperties( 60 | "test", new SingleStreamConfig(), DeviceTimestampExtractor.class)); 61 | when(aggregator.implementStream(eq(sensorTopic), any())).thenReturn(mock(KStream.class)); 62 | doCallRealMethod().when(aggregator).createBuilder(sensorTopic); 63 | aggregator.createBuilder(sensorTopic); 64 | 65 | verify(aggregator, times(1)).implementStream(eq(sensorTopic), any()); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/test/java/org/radarcns/stream/phone/PlayStoreCategoryParserTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.stream.phone; 18 | 19 | import static org.junit.Assert.assertEquals; 20 | import static org.junit.Assert.assertNull; 21 | 22 | import java.io.InputStream; 23 | import java.util.zip.GZIPInputStream; 24 | import org.jsoup.Jsoup; 25 | import org.jsoup.nodes.Document; 26 | import org.junit.Test; 27 | import org.radarcns.stream.phone.PlayStoreLookup.AppCategory; 28 | 29 | /** 30 | * Created by joris on 13/07/2017. 31 | */ 32 | public class PlayStoreCategoryParserTest { 33 | private static final String BASE_URL = "https://play.google.com/store/apps/details?id=nl.thehyve.transmartclient"; 34 | 35 | @Test 36 | public void getCategoryFromDocument() throws Exception { 37 | Document doc; 38 | try (InputStream stream = getClass().getResourceAsStream("transmart_app.html.gz"); 39 | InputStream gzipStream = new GZIPInputStream(stream)) { 40 | 41 | doc = Jsoup.parse(gzipStream, "UTF-8", BASE_URL); 42 | } 43 | 44 | AppCategory category = PlayStoreLookup.getCategoryFromDocument( 45 | doc, "nl.thehyve.transmartclient"); 46 | assertEquals("MEDICAL", category.getCategoryName()); 47 | } 48 | 49 | @Test 50 | public void getCategoryFromDocumentNoCategory() throws Exception { 51 | Document doc; 52 | try (InputStream stream = getClass().getResourceAsStream("transmart_app_no_category.html.gz"); 53 | InputStream gzipStream = new GZIPInputStream(stream)) { 54 | 55 | doc = Jsoup.parse(gzipStream, "UTF-8", BASE_URL); 56 | } 57 | 58 | AppCategory category = PlayStoreLookup.getCategoryFromDocument( 59 | doc, "nl.thehyve.transmartclient"); 60 | assertNull(category.getCategoryName()); 61 | } 62 | 63 | @Test 64 | public void getCategoryFromDocumentBroken() throws Exception { 65 | Document doc; 66 | try (InputStream stream = getClass().getResourceAsStream("transmart_app_broken.html.gz"); 67 | InputStream gzipStream = new GZIPInputStream(stream)) { 68 | 69 | doc = Jsoup.parse(gzipStream, "UTF-8", BASE_URL); 70 | } 71 | 72 | AppCategory category = PlayStoreLookup.getCategoryFromDocument( 73 | doc, "nl.thehyve.transmartclient"); 74 | assertNull(category.getCategoryName()); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/stream/KafkaStreamFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.stream; 18 | 19 | import java.util.Arrays; 20 | import java.util.Collection; 21 | import java.util.Collections; 22 | import java.util.HashSet; 23 | import java.util.List; 24 | import java.util.Locale; 25 | import java.util.stream.Stream; 26 | import org.radarcns.config.RadarBackendOptions; 27 | import org.radarcns.config.RadarPropertyHandler; 28 | import org.radarcns.config.SingleStreamConfig; 29 | import org.radarcns.config.SourceStatisticsStreamConfig; 30 | import org.radarcns.config.SubCommand; 31 | import org.slf4j.Logger; 32 | import org.slf4j.LoggerFactory; 33 | 34 | public class KafkaStreamFactory { 35 | private static final Logger logger = LoggerFactory.getLogger( 36 | KafkaStreamFactory.class.getName()); 37 | 38 | private final RadarPropertyHandler radarProperties; 39 | private final RadarBackendOptions options; 40 | 41 | public KafkaStreamFactory(RadarBackendOptions options, 42 | RadarPropertyHandler properties) { 43 | this.options = options; 44 | this.radarProperties = properties; 45 | } 46 | 47 | public StreamMaster createSensorStreams() { 48 | Collection streamTypes; 49 | 50 | String[] args = options.getSubCommandArgs(); 51 | if (args != null && args.length > 0) { 52 | streamTypes = new HashSet<>(Arrays.asList(args)); 53 | } else { 54 | streamTypes = Collections.emptySet(); 55 | } 56 | 57 | return master(radarProperties.getRadarProperties().getStream().getStreamConfigs().stream() 58 | .filter(s -> streamTypes.isEmpty() || streamTypes.stream().anyMatch(n -> 59 | s.getStreamClass().getName().toLowerCase(Locale.US) 60 | .endsWith(n.toLowerCase(Locale.US))))); 61 | } 62 | 63 | private StreamMaster master(Stream configs) { 64 | return new StreamMaster(radarProperties, configs); 65 | } 66 | 67 | public SubCommand createStreamStatistics() { 68 | List configs = radarProperties.getRadarProperties() 69 | .getStream().getSourceStatistics(); 70 | 71 | if (configs == null) { 72 | logger.warn("Statistics monitor is not configured. Cannot start it."); 73 | return master(Stream.empty()); 74 | } 75 | 76 | return master(configs.stream()); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/stream/phone/PhoneUsageCollector.java: -------------------------------------------------------------------------------- 1 | package org.radarcns.stream.phone; 2 | 3 | import java.math.BigDecimal; 4 | import java.math.MathContext; 5 | import org.radarcns.passive.phone.PhoneUsageEvent; 6 | import org.radarcns.passive.phone.UsageEventType; 7 | 8 | /** 9 | * Created by piotrzakrzewski on 27/07/2017. 10 | */ 11 | public class PhoneUsageCollector { 12 | private BigDecimal totalForegroundTime; // total time in seconds 13 | private double lastForegroundEvent; // date in Unix time in seconds 14 | private int timesTurnedOn; 15 | private String categoryName; // optional 16 | private Double categoryNameFetchTime; // optional 17 | 18 | public PhoneUsageCollector() { 19 | totalForegroundTime = BigDecimal.ZERO; 20 | } 21 | 22 | public PhoneUsageCollector update(PhoneUsageEvent event) { 23 | if (event.getCategoryName() != null) { 24 | this.categoryName = event.getCategoryName(); 25 | this.categoryNameFetchTime = event.getCategoryNameFetchTime(); 26 | } 27 | 28 | if (event.getEventType() == UsageEventType.FOREGROUND) { 29 | // Foreground event received 30 | timesTurnedOn++; 31 | lastForegroundEvent = event.getTime(); 32 | } else if (event.getEventType() == UsageEventType.BACKGROUND 33 | && lastForegroundEvent != 0.0) { 34 | // Background event received for an app which was previously on. 35 | totalForegroundTime = totalForegroundTime.add( 36 | BigDecimal.valueOf(event.getTime()) 37 | .subtract(BigDecimal.valueOf(lastForegroundEvent), 38 | MathContext.DECIMAL128)); 39 | lastForegroundEvent = 0.0; 40 | } 41 | // else if eventType is background and it was already in the background, ignore. 42 | // We must have missed an event. 43 | // else if eventType is something else: ignore. 44 | 45 | return this; 46 | } 47 | 48 | public double getTotalForegroundTime() { 49 | return totalForegroundTime.doubleValue(); 50 | } 51 | 52 | public void setTotalForegroundTime(double totalForegroundTime) { 53 | this.totalForegroundTime = BigDecimal.valueOf(totalForegroundTime); 54 | } 55 | 56 | public double getLastForegroundEvent() { 57 | return lastForegroundEvent; 58 | } 59 | 60 | public void setLastForegroundEvent(double lastForegroundEvent) { 61 | this.lastForegroundEvent = lastForegroundEvent; 62 | } 63 | 64 | public int getTimesTurnedOn() { 65 | return timesTurnedOn; 66 | } 67 | 68 | public void setTimesTurnedOn(int timesTurnedOn) { 69 | this.timesTurnedOn = timesTurnedOn; 70 | } 71 | 72 | public String getCategoryName() { 73 | return categoryName; 74 | } 75 | 76 | public void setCategoryName(String categoryName) { 77 | this.categoryName = categoryName; 78 | } 79 | 80 | public Double getCategoryNameFetchTime() { 81 | return categoryNameFetchTime; 82 | } 83 | 84 | public void setCategoryNameFetchTime(Double categoryNameFetchTime) { 85 | this.categoryNameFetchTime = categoryNameFetchTime; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/test/java/org/radarcns/stream/DeviceTimestampExtractorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.stream; 18 | 19 | import static org.junit.Assert.assertEquals; 20 | import static org.junit.Assert.assertNull; 21 | 22 | import org.apache.avro.Schema; 23 | import org.apache.avro.generic.GenericData; 24 | import org.apache.avro.generic.GenericRecord; 25 | import org.apache.kafka.clients.consumer.ConsumerRecord; 26 | import org.junit.Before; 27 | import org.junit.Rule; 28 | import org.junit.Test; 29 | import org.junit.rules.ExpectedException; 30 | 31 | /** 32 | * Created by nivethika on 20-12-16. 33 | */ 34 | public class DeviceTimestampExtractorTest { 35 | 36 | private DeviceTimestampExtractor timestampExtractor; 37 | private String topic; 38 | 39 | @Rule 40 | public ExpectedException exception = ExpectedException.none(); 41 | 42 | @Before 43 | public void setUp() { 44 | this.timestampExtractor = new DeviceTimestampExtractor(); 45 | 46 | this.topic = "TESTTopic"; 47 | } 48 | 49 | @Test 50 | public void extract() { 51 | String userSchema = "{\"namespace\": \"test.radar.backend\", \"type\": \"record\", " 52 | +"\"name\": \"TestTimeExtract\"," 53 | +"\"fields\": [{\"name\": \"timeReceived\", \"type\": \"double\"}]}"; 54 | GenericRecord record = buildIndexedRecord(userSchema); 55 | double timeValue = 40880.051388; 56 | record.put("timeReceived", timeValue); 57 | ConsumerRecord consumerRecord = new ConsumerRecord<>(topic, 3, 30, null, record); 58 | long extracted = this.timestampExtractor.extract(consumerRecord, -1L); 59 | assertEquals((long) (1000d * timeValue), extracted); 60 | } 61 | 62 | @Test 63 | public void extractWithNotDoubleTimeReceived() { 64 | String userSchema = "{\"namespace\": \"test.radar.backend\", \"type\": \"record\", " 65 | +"\"name\": \"TestTimeExtract\"," 66 | +"\"fields\": [{\"name\": \"timeReceived\", \"type\": \"string\"}]}"; 67 | GenericRecord record = buildIndexedRecord(userSchema); 68 | record.put("timeReceived", "timeValue"); 69 | ConsumerRecord consumerRecord = new ConsumerRecord<>(topic, 3, 30, null, record); 70 | 71 | exception.expect(RuntimeException.class); 72 | exception.expectMessage("Impossible to extract timeReceived from"); 73 | long extracted = this.timestampExtractor.extract(consumerRecord, -1L); 74 | assertNull(extracted); 75 | 76 | } 77 | 78 | private static GenericRecord buildIndexedRecord(String userSchema) { 79 | Schema.Parser parser = new Schema.Parser(); 80 | Schema schema = parser.parse(userSchema); 81 | return new GenericData.Record(schema); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/config/RadarBackendOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.config; 18 | 19 | import java.io.File; 20 | import javax.annotation.Nonnull; 21 | import org.apache.commons.cli.CommandLine; 22 | import org.apache.commons.cli.DefaultParser; 23 | import org.apache.commons.cli.Options; 24 | import org.apache.commons.cli.ParseException; 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | public class RadarBackendOptions { 29 | private static final Logger log = LoggerFactory.getLogger(RadarBackendOptions.class); 30 | private final CommandLine cli; 31 | private final String subCommand; 32 | private final String[] subCommandArgs; 33 | public static final Options OPTIONS = new Options() 34 | .addOption("c", "config", true, "Configuration YAML file") 35 | .addOption("d", "devices", true, "Number of devices to use with the mock command.") 36 | .addOption("D", "direct", false, "The mock device will bypass the rest-proxy and use " 37 | + "the Kafka Producer API instead.") 38 | .addOption("f", "file", true, "Read mock data from given configuration file."); 39 | 40 | 41 | /** 42 | * @param cli command line arguments given 43 | */ 44 | public RadarBackendOptions(CommandLine cli) { 45 | log.info("Loading configuration"); 46 | this.cli = cli; 47 | 48 | String[] additionalArgs = this.cli.getArgs(); 49 | 50 | if (additionalArgs.length > 0) { 51 | subCommand = additionalArgs[0]; 52 | subCommandArgs = new String[additionalArgs.length - 1]; 53 | System.arraycopy(additionalArgs, 1, subCommandArgs, 0, subCommandArgs.length); 54 | } else { 55 | subCommand = null; 56 | subCommandArgs = null; 57 | } 58 | } 59 | 60 | public static RadarBackendOptions parse(@Nonnull String[] args) throws ParseException { 61 | CommandLine cli = new DefaultParser().parse(OPTIONS, args); 62 | return new RadarBackendOptions(cli); 63 | } 64 | 65 | public String getPropertyPath() { 66 | return this.cli.getOptionValue("config", null); 67 | } 68 | 69 | public int getNumMockDevices() { 70 | return Integer.parseInt(this.cli.getOptionValue("devices", "1")); 71 | } 72 | 73 | public boolean isMockDirect() { 74 | return this.cli.hasOption("direct"); 75 | } 76 | 77 | public File getMockFile() { 78 | String file = this.cli.getOptionValue("file", null); 79 | if (file == null) { 80 | return null; 81 | } else { 82 | return new File(file); 83 | } 84 | } 85 | 86 | public String getSubCommand() { 87 | return subCommand; 88 | } 89 | 90 | public String[] getSubCommandArgs() { 91 | return subCommandArgs; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /config/pmd/ruleset.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | This ruleset was parsed from the Codacy default codestyle. 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/config/KafkaProperty.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.config; 18 | 19 | import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; 20 | import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde; 21 | import java.util.Properties; 22 | import javax.annotation.Nonnull; 23 | import org.apache.kafka.clients.consumer.ConsumerConfig; 24 | import org.apache.kafka.streams.StreamsConfig; 25 | import org.apache.kafka.streams.errors.LogAndContinueExceptionHandler; 26 | import org.apache.kafka.streams.processor.TimestampExtractor; 27 | 28 | public class KafkaProperty { 29 | 30 | private final ConfigRadar configRadar; 31 | 32 | public KafkaProperty(ConfigRadar configRadar) { 33 | this.configRadar = configRadar; 34 | } 35 | 36 | /** 37 | * @param clientId useful for debugging 38 | * @param singleStreamConfig stream configuration 39 | * @return Properties for a Kafka Stream 40 | */ 41 | public Properties getStreamProperties(@Nonnull String clientId, 42 | SingleStreamConfig singleStreamConfig) { 43 | Properties props = new Properties(); 44 | 45 | StreamConfig streamConfig = configRadar.getStream(); 46 | 47 | props.put(StreamsConfig.APPLICATION_ID_CONFIG, clientId); 48 | props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, configRadar.getBrokerPaths()); 49 | props.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, 50 | configRadar.getSchemaRegistryPaths()); 51 | props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, SpecificAvroSerde.class); 52 | props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, SpecificAvroSerde.class); 53 | props.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, 54 | streamConfig.threadsByPriority(singleStreamConfig.getPriority())); 55 | props.put(StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG, 56 | LogAndContinueExceptionHandler.class.getName()); 57 | props.putAll(configRadar.getStream().getProperties()); 58 | props.putAll(singleStreamConfig.getProperties()); 59 | props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); 60 | 61 | return props; 62 | } 63 | 64 | /** 65 | * @param clientId useful for debugging 66 | * @param singleStreamConfig stream configuration 67 | * @param timestampExtractor custom timestamp extract that overrides the out-of-the-box 68 | * @return Properties for a Kafka Stream 69 | */ 70 | public Properties getStreamProperties(@Nonnull String clientId, 71 | SingleStreamConfig singleStreamConfig, 72 | @Nonnull Class timestampExtractor) { 73 | Properties props = getStreamProperties(clientId, singleStreamConfig); 74 | 75 | props.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, 76 | timestampExtractor.getName()); 77 | 78 | return props; 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.integration; 18 | 19 | import static org.junit.Assert.assertNotNull; 20 | 21 | import java.io.IOException; 22 | import java.util.Collections; 23 | import java.util.Properties; 24 | import org.apache.avro.Schema; 25 | import org.apache.avro.generic.GenericData; 26 | import org.apache.avro.generic.GenericRecord; 27 | import org.apache.kafka.clients.consumer.ConsumerConfig; 28 | import org.apache.kafka.clients.consumer.ConsumerRecord; 29 | import org.apache.kafka.clients.consumer.ConsumerRecords; 30 | import org.radarcns.config.RadarPropertyHandler; 31 | import org.radarcns.monitor.AbstractKafkaMonitor; 32 | import org.slf4j.Logger; 33 | import org.slf4j.LoggerFactory; 34 | 35 | /** 36 | * Consumer for Aggregated Acceleration Stream 37 | */ 38 | public class E4AggregatedAccelerationMonitor extends AbstractKafkaMonitor { 39 | private static final Logger logger = LoggerFactory.getLogger(E4AggregatedAccelerationMonitor.class); 40 | 41 | public E4AggregatedAccelerationMonitor(RadarPropertyHandler radar, String topic, String clientID) throws IOException { 42 | super(radar, Collections.singletonList(topic), "new", clientID, null); 43 | 44 | Properties props = new Properties(); 45 | props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); 46 | props.putAll(radar.getRadarProperties().getStream().getProperties()); 47 | configure(props); 48 | } 49 | 50 | @Override 51 | protected void evaluateRecord(ConsumerRecord records) { 52 | // noop 53 | } 54 | 55 | @Override 56 | protected void evaluateRecords(ConsumerRecords records) { 57 | for (ConsumerRecord record : records) { 58 | 59 | GenericRecord key = record.key(); 60 | if (key == null) { 61 | logger.error("Failed to process record {} without a key.", record); 62 | return; 63 | } 64 | Schema keySchema = key.getSchema(); 65 | if (keySchema.getField("userId") != null 66 | && keySchema.getField("sourceId") != null) { 67 | assertNotNull(key.get("userId")); 68 | assertNotNull(key.get("sourceId")); 69 | } else { 70 | logger.error("Failed to process record {} with wrong key type {}.", 71 | record, key.getSchema()); 72 | return; 73 | } 74 | GenericRecord value = record.value(); 75 | GenericData.Array fields = (GenericData.Array) value.get("fields"); 76 | logger.info("Received [{}, {}, {}] E4 messages", 77 | ((GenericRecord)fields.get(0)).get("count"), ((GenericRecord)fields.get(1)).get("count"), ((GenericRecord)fields.get(2)).get("count")); 78 | 79 | if ((Integer)((GenericRecord)fields.get(0)).get("count") > 100) { 80 | shutdown(); 81 | } 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/util/EmailSender.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.util; 18 | 19 | import java.io.IOException; 20 | import java.util.List; 21 | import java.util.Properties; 22 | import javax.mail.Message; 23 | import javax.mail.MessagingException; 24 | import javax.mail.Session; 25 | import javax.mail.Transport; 26 | import javax.mail.internet.InternetAddress; 27 | import javax.mail.internet.MimeMessage; 28 | 29 | /** 30 | * Sends emails. 31 | */ 32 | public class EmailSender { 33 | private final String from; 34 | private final List to; 35 | private final Session session; 36 | 37 | /** 38 | * Email sender to simple SMTP host. The host must not use authentication 39 | * @param host smtp host 40 | * @param port port that the smtp service is configured on 41 | * @param from MIME From header 42 | * @param to list of recipients in the MIME To header 43 | * @throws IOException if a connection cannot be established with the email provider. 44 | */ 45 | public EmailSender(String host, int port, String from, List to) throws IOException { 46 | this.from = from; 47 | this.to = to; 48 | 49 | Properties properties = new Properties(); 50 | // Get system properties 51 | properties.putAll(System.getProperties()); 52 | 53 | if (host != null) { 54 | // Setup mail server 55 | properties.setProperty("mail.smtp.host", host); 56 | } 57 | if (port > 0) { 58 | properties.setProperty("mail.smtp.port", String.valueOf(port)); 59 | } 60 | 61 | session = Session.getInstance(properties); 62 | try { 63 | Transport transport = session.getTransport("smtp"); 64 | transport.connect(); 65 | if (!transport.isConnected()) { 66 | throw new IOException("Cannot connect to SMTP server " + host + ":" + port); 67 | } 68 | } catch (MessagingException ex) { 69 | throw new IOException("Cannot instantiate SMTP server", ex); 70 | } 71 | } 72 | 73 | /** 74 | * Send an email with given subject and text. The pre-configured From and To headers are used. 75 | * @param subject email subject 76 | * @param text plain text content of the email 77 | * @throws MessagingException if the message could not be sent 78 | */ 79 | public void sendEmail(String subject, String text) throws MessagingException { 80 | // Create a default MimeMessage object. 81 | MimeMessage message = new MimeMessage(session); 82 | 83 | // Set From: header field of the header. 84 | message.setFrom(new InternetAddress(from)); 85 | 86 | for (String recipient : to) { 87 | // Set To: header field of the header. 88 | message.addRecipient(Message.RecipientType.TO, new InternetAddress(recipient)); 89 | } 90 | 91 | // Set Subject: header field 92 | message.setSubject(subject); 93 | 94 | // Now set the actual message 95 | message.setText(text); 96 | 97 | // Send message 98 | Transport.send(message); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /radar.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | released: 2016-11-27 3 | 4 | #============================== Zookeeper ==============================# 5 | #List of Zookeeper instances 6 | zookeeper: 7 | - host: localhost 8 | port: 2181 9 | 10 | #================================ Kafka ================================# 11 | #List of Kafka brokers 12 | broker: 13 | - host: localhost 14 | port: 9092 15 | 16 | #Kafka internal parameters 17 | 18 | #============================ Kafka Streams ============================# 19 | #The number of threads that a stream must be run according is priority 20 | stream: 21 | threads_per_priority: 22 | low: 1 23 | normal: 2 24 | high: 4 25 | 26 | properties: 27 | max.request.size: 3500042 #Set message.max.bytes for kafka brokers higher than or equal to this value 28 | retries: 15 29 | session.timeout.ms: 20000 30 | 31 | streams: 32 | - class: org.radarcns.stream.empatica.E4AccelerationStream 33 | - class: org.radarcns.stream.empatica.E4BatteryLevelStream 34 | - class: org.radarcns.stream.empatica.E4BloodVolumePulseStream 35 | - class: org.radarcns.stream.empatica.E4ElectroDermalActivityStream 36 | - class: org.radarcns.stream.empatica.E4HeartRateStream 37 | - class: org.radarcns.stream.empatica.E4InterBeatIntervalStream 38 | - class: org.radarcns.stream.empatica.E4TemperatureStream 39 | - class: org.radarcns.stream.phone.PhoneAccelerationStream 40 | - class: org.radarcns.stream.phone.PhoneBatteryStream 41 | - class: org.radarcns.stream.phone.PhoneUsageStream 42 | - class: org.radarcns.stream.phone.PhoneUsageAggregationStream 43 | 44 | #====================== Source statistics monitor ======================# 45 | source_statistics: 46 | - name: Empatica E4 47 | topics: 48 | - android_empatica_e4_blood_volume_pulse_1min 49 | output_topic: source_statistics_empatica_e4 50 | - name: Biovotion VSM1 51 | topics: 52 | - android_biovotion_vsm1_acceleration_1min 53 | output_topic: source_statistics_biovotion_vsm1 54 | - name: RADAR pRMT 55 | topics: 56 | - android_phone_acceleration_1min 57 | - android_phone_bluetooth_devices 58 | - android_phone_sms 59 | - android_phone_call 60 | - android_phone_contacts 61 | - android_phone_usage_event 62 | - android_phone_relative_location 63 | output_topic: source_statistics_android_phone 64 | 65 | 66 | #=========================== Schema Registry ===========================# 67 | #List of Schema Registry instances 68 | schema_registry: 69 | - host: localhost 70 | port: 8081 71 | protocol: http 72 | 73 | rest_proxy: 74 | host: radar-test.thehyve.net 75 | port: 8082 76 | protocol: http 77 | 78 | #======================== Battery level monitor ========================# 79 | battery_monitor: 80 | notify: # Each project can have a number of email addresses 81 | - project_id: s1 82 | email_address: 83 | - test@thehyve.nl 84 | - project_id: s2 85 | email_address: 86 | - radar@thehyve.nl 87 | level: LOW 88 | email_host: localhost 89 | email_port: 25 90 | email_user: no-reply@radarcns.org 91 | topics: 92 | - android_empatica_e4_battery_level 93 | 94 | #======================= Disconnection monitor==========================# 95 | disconnect_monitor: 96 | notify: 97 | - project_id: s1 98 | email_address: 99 | - test@thehyve.nl 100 | - project_id: s2 101 | email_address: 102 | - radar@thehyve.nl 103 | email_host: localhost 104 | email_port: 25 105 | email_user: no-reply@radarcns.org 106 | topics: 107 | - android_empatica_e4_temperature 108 | timeout: 1800 # seconds after which a stream is set disconnected 109 | alert_repetitions: 2 # number of additional emails to send after the first 110 | 111 | # persistence_path: /var/lib/radar/data 112 | -------------------------------------------------------------------------------- /gradle/publishing.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'maven-publish' 2 | apply plugin: 'com.jfrog.bintray' 3 | 4 | ext.sharedManifest = manifest { 5 | attributes( 6 | "Implementation-Title": rootProject.name, 7 | "Implementation-Version": version) 8 | } 9 | 10 | //---------------------------------------------------------------------------// 11 | // Packaging // 12 | //---------------------------------------------------------------------------// 13 | 14 | processResources { 15 | expand(version: version) 16 | } 17 | 18 | jar { 19 | manifest { 20 | from sharedManifest 21 | attributes('Main-Class': mainClassName) 22 | } 23 | } 24 | 25 | tasks.withType(Tar){ 26 | compression = Compression.GZIP 27 | extension = 'tar.gz' 28 | } 29 | 30 | // custom tasks for creating source/javadoc jars 31 | task sourcesJar(type: Jar, dependsOn: classes) { 32 | classifier = 'sources' 33 | from sourceSets.main.allSource 34 | manifest.from sharedManifest 35 | } 36 | 37 | task javadocJar(type: Jar, dependsOn: javadoc) { 38 | classifier = 'javadoc' 39 | from javadoc.destinationDir 40 | manifest.from sharedManifest 41 | } 42 | 43 | // add javadoc/source jar tasks as artifacts 44 | artifacts { 45 | archives sourcesJar, javadocJar 46 | } 47 | 48 | publishing { 49 | publications { 50 | mavenJar(MavenPublication) { 51 | from components.java 52 | artifact sourcesJar 53 | artifact javadocJar 54 | pom { 55 | description = moduleDescription 56 | url = githubUrl 57 | 58 | licenses { 59 | license { 60 | name = 'The Apache Software License, Version 2.0' 61 | url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' 62 | distribution = 'repo' 63 | } 64 | } 65 | developers { 66 | developer { 67 | id = 'blootsvoets' 68 | name = 'Joris Borgdorff' 69 | email = 'joris@thehyve.nl' 70 | organization = 'The Hyve' 71 | } 72 | developer { 73 | id = 'nivemaham' 74 | name = 'Nivethika Mahasivam' 75 | email = 'nivethika@thehyve.nl' 76 | organization = 'The Hyve' 77 | } 78 | } 79 | issueManagement { 80 | system = 'GitHub' 81 | url = issueUrl 82 | } 83 | organization { 84 | name = 'RADAR-Base' 85 | url = website 86 | } 87 | scm { 88 | connection = 'scm:git:' + githubUrl 89 | url = githubUrl 90 | } 91 | } 92 | } 93 | } 94 | } 95 | 96 | bintray { 97 | user project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') 98 | key project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') 99 | override false 100 | publications 'mavenJar' 101 | pkg { 102 | repo = project.group 103 | name = rootProject.name 104 | userOrg = 'radar-cns' 105 | desc = moduleDescription 106 | licenses = ['Apache-2.0'] 107 | websiteUrl = website 108 | issueTrackerUrl = issueUrl 109 | vcsUrl = githubUrl 110 | githubRepo = githubRepoName 111 | githubReleaseNotesFile = 'README.md' 112 | version { 113 | name = project.version 114 | desc = moduleDescription 115 | vcsTag = System.getenv('TRAVIS_TAG') 116 | released = new Date() 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/util/RadarUtilitiesImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.util; 18 | 19 | import static org.apache.kafka.streams.KeyValue.pair; 20 | 21 | import java.util.List; 22 | import java.util.stream.Collectors; 23 | import org.apache.kafka.streams.KeyValue; 24 | import org.apache.kafka.streams.kstream.Window; 25 | import org.apache.kafka.streams.kstream.Windowed; 26 | import org.radarcns.kafka.AggregateKey; 27 | import org.radarcns.kafka.ObservationKey; 28 | import org.radarcns.stream.aggregator.AggregateList; 29 | import org.radarcns.stream.aggregator.NumericAggregate; 30 | import org.radarcns.stream.aggregator.PhoneUsageAggregate; 31 | import org.radarcns.stream.collector.AggregateListCollector; 32 | import org.radarcns.stream.collector.NumericAggregateCollector; 33 | import org.radarcns.stream.phone.PhoneUsageCollector; 34 | import org.radarcns.stream.phone.TemporaryPackageKey; 35 | 36 | /** 37 | * Implements {@link RadarUtilities}. 38 | */ 39 | public class RadarUtilitiesImpl implements RadarUtilities { 40 | protected RadarUtilitiesImpl() { 41 | // used for construction from RadarSingletonFactory 42 | } 43 | 44 | @Override 45 | public AggregateKey getWindowed(Windowed window) { 46 | ObservationKey key = window.key(); 47 | Window timeWindow = window.window(); 48 | return new AggregateKey(key.getProjectId(), key.getUserId(), key.getSourceId(), 49 | timeWindow.start() / 1000d, timeWindow.end() / 1000d); 50 | } 51 | 52 | @Override 53 | public AggregateKey getWindowedTuple(Windowed window) { 54 | TemporaryPackageKey key = window.key(); 55 | Window timeWindow = window.window(); 56 | return new AggregateKey(key.getProjectId(), key.getUserId(), key.getSourceId(), 57 | timeWindow.start() / 1000d, timeWindow.end() / 1000d); 58 | } 59 | 60 | @Override 61 | public KeyValue phoneCollectorToAvro( 62 | Windowed window, PhoneUsageCollector collector 63 | ) { 64 | return pair(getWindowedTuple(window) , new PhoneUsageAggregate( 65 | window.key().getPackageName(), 66 | collector.getTotalForegroundTime(), 67 | collector.getTimesTurnedOn(), 68 | collector.getCategoryName(), 69 | collector.getCategoryNameFetchTime() 70 | )); 71 | } 72 | 73 | @Override 74 | public KeyValue listCollectorToAvro(Windowed window, AggregateListCollector collector) { 75 | List fields = collector.getCollectors().stream() 76 | .map(this::numericCollectorToAggregate) 77 | .collect(Collectors.toList()); 78 | 79 | return pair(getWindowed(window), new AggregateList(fields)); 80 | } 81 | 82 | @Override 83 | public KeyValue numericCollectorToAvro( 84 | Windowed window, NumericAggregateCollector collector) { 85 | return pair(getWindowed(window), numericCollectorToAggregate(collector)); 86 | } 87 | 88 | private NumericAggregate numericCollectorToAggregate(NumericAggregateCollector collector) { 89 | return new NumericAggregate(collector.getName(), collector.getMin(), collector.getMax(), 90 | collector.getSum(), collector.getCount(), collector.getMean(), 91 | collector.getQuartile()); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/test/java/org/radarcns/monitor/CombinedKafkaMonitorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.monitor; 18 | 19 | import static org.junit.Assert.assertFalse; 20 | import static org.junit.Assert.assertTrue; 21 | import static org.junit.Assert.fail; 22 | import static org.mockito.Mockito.doThrow; 23 | import static org.mockito.Mockito.mock; 24 | import static org.mockito.Mockito.times; 25 | import static org.mockito.Mockito.verify; 26 | 27 | import java.io.IOException; 28 | import java.time.Duration; 29 | import java.util.concurrent.ExecutorService; 30 | import java.util.concurrent.Executors; 31 | import java.util.concurrent.TimeUnit; 32 | import java.util.stream.Stream; 33 | import org.junit.Test; 34 | 35 | public class CombinedKafkaMonitorTest { 36 | 37 | @Test(expected = IOException.class) 38 | public void testExceptionFlow() throws Exception { 39 | KafkaMonitor kafkaMonitor1 = mock(KafkaMonitor.class); 40 | KafkaMonitor kafkaMonitor2 = mock(KafkaMonitor.class); 41 | 42 | doThrow(new IOException("failed to run!")).when(kafkaMonitor2).start(); 43 | 44 | CombinedKafkaMonitor km = new CombinedKafkaMonitor( Stream.of(kafkaMonitor1, kafkaMonitor2)); 45 | 46 | try { 47 | km.start(); 48 | } catch (IOException ex) { 49 | verify(kafkaMonitor1, times(1)).start(); 50 | verify(kafkaMonitor2, times(1)).start(); 51 | verify(kafkaMonitor1, times(1)).shutdown(); 52 | verify(kafkaMonitor2, times(1)).shutdown(); 53 | assertTrue(km.isShutdown()); 54 | throw ex; 55 | } 56 | } 57 | 58 | @Test 59 | public void testFlow() throws Exception { 60 | KafkaMonitor kafkaMonitor1 = mock(KafkaMonitor.class); 61 | KafkaMonitor kafkaMonitor2 = mock(KafkaMonitor.class); 62 | 63 | CombinedKafkaMonitor km = new CombinedKafkaMonitor(Stream.of(kafkaMonitor1, kafkaMonitor2)); 64 | 65 | ExecutorService executor = Executors.newSingleThreadExecutor(); 66 | executor.submit(() -> { 67 | try { 68 | km.start(); 69 | } catch (IOException | InterruptedException e) { 70 | fail(e.toString()); 71 | } 72 | }); 73 | 74 | Thread.sleep(100L); 75 | 76 | assertFalse(km.isShutdown()); 77 | verify(kafkaMonitor1, times(1)).start(); 78 | verify(kafkaMonitor2, times(1)).start(); 79 | 80 | km.shutdown(); 81 | verify(kafkaMonitor1, times(1)).shutdown(); 82 | verify(kafkaMonitor2, times(1)).shutdown(); 83 | 84 | assertTrue(km.isShutdown()); 85 | executor.shutdown(); 86 | assertTrue(executor.awaitTermination(100, TimeUnit.MILLISECONDS)); 87 | } 88 | 89 | @Test 90 | public void testPollTimeout() { 91 | KafkaMonitor kafkaMonitor1 = mock(KafkaMonitor.class); 92 | KafkaMonitor kafkaMonitor2 = mock(KafkaMonitor.class); 93 | 94 | CombinedKafkaMonitor km = new CombinedKafkaMonitor(Stream.of(kafkaMonitor1, kafkaMonitor2)); 95 | km.setPollTimeout(Duration.ofSeconds(1L)); 96 | 97 | verify(kafkaMonitor1, times(1)).setPollTimeout(Duration.ofSeconds(1L)); 98 | verify(kafkaMonitor2, times(1)).setPollTimeout(Duration.ofSeconds(1L)); 99 | } 100 | 101 | @Test(expected = IllegalArgumentException.class) 102 | public void testEmpty() { 103 | new CombinedKafkaMonitor(Stream.empty()); 104 | } 105 | 106 | @Test(expected = NullPointerException.class) 107 | public void testNull() { 108 | new CombinedKafkaMonitor(null); 109 | } 110 | } -------------------------------------------------------------------------------- /src/main/java/org/radarcns/config/StreamConfig.java: -------------------------------------------------------------------------------- 1 | package org.radarcns.config; 2 | 3 | import com.fasterxml.jackson.annotation.JsonGetter; 4 | import com.fasterxml.jackson.annotation.JsonIgnore; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | import com.fasterxml.jackson.annotation.JsonSetter; 7 | import java.time.Duration; 8 | import java.util.Collections; 9 | import java.util.EnumMap; 10 | import java.util.List; 11 | import java.util.Locale; 12 | import java.util.Map; 13 | import java.util.function.Function; 14 | import java.util.stream.Collectors; 15 | import java.util.stream.Stream; 16 | import org.radarcns.config.RadarPropertyHandler.Priority; 17 | import org.radarcns.stream.TimeWindowMetadata; 18 | 19 | // POJO class 20 | @SuppressWarnings("PMD.ImmutableField") 21 | public class StreamConfig { 22 | private final Map timeWindowCommitInterval = 23 | new EnumMap<>(TimeWindowMetadata.class); 24 | 25 | @JsonIgnore 26 | private final Map priorityThreads; 27 | @JsonProperty("min_commit_interval") 28 | private long minCommitInterval = 10; 29 | @JsonProperty("max_commit_interval") 30 | private long maxCommitInterval = Duration.ofHours(3).getSeconds(); 31 | @JsonProperty("time_window_commit_interval_multiplier") 32 | private float timeWindowCommitIntervalMultiplier = 0.5f; 33 | @JsonProperty 34 | private Map properties; 35 | @JsonProperty("streams") 36 | private List streamConfigs; 37 | 38 | @JsonProperty("source_statistics") 39 | private List sourceStatistics; 40 | 41 | 42 | public StreamConfig() { 43 | priorityThreads = new EnumMap<>(Priority.class); 44 | priorityThreads.put(Priority.LOW, 1); 45 | priorityThreads.put(Priority.NORMAL, 2); 46 | priorityThreads.put(Priority.HIGH, 4); 47 | } 48 | 49 | public long getMinCommitInterval() { 50 | return minCommitInterval; 51 | } 52 | 53 | public long getMaxCommitInterval() { 54 | return maxCommitInterval; 55 | } 56 | 57 | public float getTimeWindowCommitIntervalMultiplier() { 58 | return timeWindowCommitIntervalMultiplier; 59 | } 60 | 61 | public Map getProperties() { 62 | return properties != null ? properties : Collections.emptyMap(); 63 | } 64 | 65 | public List getStreamConfigs() { 66 | return streamConfigs; 67 | } 68 | 69 | @JsonGetter("threads_per_priority") 70 | public Map getThreadsPerPriority() { 71 | return priorityThreads.entrySet().stream() 72 | .collect(Collectors.toMap(e -> e.getKey().getParam(), Map.Entry::getValue)); 73 | } 74 | 75 | @JsonSetter("threads_per_priority") 76 | public void setThreadsPerPriority(Map streamPriority) { 77 | streamPriority.values().forEach(v -> { 78 | if (v == null || v < 1) { 79 | throw new IllegalArgumentException("Stream priorities cannot be smaller than 1"); 80 | } 81 | }); 82 | this.priorityThreads.putAll(streamPriority.entrySet().stream() 83 | .collect(Collectors.toMap( 84 | e -> Priority.valueOf(e.getKey().toUpperCase(Locale.US)), 85 | Map.Entry::getValue))); 86 | } 87 | 88 | @JsonIgnore 89 | public Duration getCommitIntervalForTimeWindow(TimeWindowMetadata w) { 90 | if (timeWindowCommitInterval.isEmpty()) { 91 | timeWindowCommitInterval.putAll(Stream.of(TimeWindowMetadata.values()) 92 | .collect(Collectors.toMap(Function.identity(), 93 | t -> { 94 | long base = (long) (timeWindowCommitIntervalMultiplier 95 | * t.getIntervalInMilliSec() / 1000.0); 96 | return Duration.ofSeconds( 97 | Math.min(maxCommitInterval, 98 | Math.min(base, maxCommitInterval))); 99 | }))); 100 | } 101 | return timeWindowCommitInterval.get(w); 102 | } 103 | 104 | public int threadsByPriority(Priority level) { 105 | return priorityThreads.get(level); 106 | } 107 | 108 | public List getSourceStatistics() { 109 | return sourceStatistics; 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/config/RadarPropertyHandlerImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.config; 18 | 19 | import static org.radarcns.util.Strings.isNullOrEmpty; 20 | 21 | import java.io.File; 22 | import java.io.IOException; 23 | import java.io.InputStream; 24 | import java.net.URISyntaxException; 25 | import java.net.URL; 26 | import java.util.Properties; 27 | import org.radarcns.RadarBackend; 28 | import org.radarcns.util.PersistentStateStore; 29 | import org.radarcns.util.YamlPersistentStateStore; 30 | import org.slf4j.Logger; 31 | import org.slf4j.LoggerFactory; 32 | 33 | /** 34 | * Java Singleton class for handling the yml config file. Implements @link{ RadarPropertyHandler} 35 | */ 36 | public class RadarPropertyHandlerImpl implements RadarPropertyHandler { 37 | private static final String CONFIG_FILE_NAME = "radar.yml"; 38 | private static final Logger log = LoggerFactory.getLogger(RadarPropertyHandlerImpl.class); 39 | 40 | private ConfigRadar properties; 41 | private KafkaProperty kafkaProperty; 42 | 43 | @Override 44 | public ConfigRadar getRadarProperties() { 45 | if (!isLoaded()) { 46 | throw new IllegalStateException( 47 | "Properties cannot be accessed without calling load() first"); 48 | } 49 | return properties; 50 | } 51 | 52 | public boolean isLoaded() { 53 | return properties != null; 54 | } 55 | 56 | @Override 57 | public void load(String pathFile) throws IOException { 58 | if (isLoaded()) { 59 | throw new IllegalStateException("Properties class has been already loaded"); 60 | } 61 | 62 | File file; 63 | 64 | //If pathFile is null 65 | if (isNullOrEmpty(pathFile)) { 66 | file = getDefaultFile(); 67 | log.info("DEFAULT CONFIGURATION: loading config file at {}", file); 68 | } else { 69 | log.info("USER CONFIGURATION: loading config file at {}", pathFile); 70 | file = new File(pathFile); 71 | } 72 | 73 | if (!file.exists()) { 74 | throw new IllegalArgumentException("Config file " + file + " does not exist"); 75 | } 76 | if (!file.isFile()) { 77 | throw new IllegalArgumentException("Config file " + file + " is invalid"); 78 | } 79 | 80 | properties = new YamlConfigLoader().load(file, ConfigRadar.class); 81 | 82 | Properties buildProperties = new Properties(); 83 | try (InputStream in = getClass().getResourceAsStream("/build.properties")) { 84 | if (in != null) { 85 | buildProperties.load(in); 86 | } 87 | } 88 | String version = buildProperties.getProperty("version"); 89 | if (version != null) { 90 | properties.setBuildVersion(version); 91 | } 92 | } 93 | 94 | private File getDefaultFile() throws IOException { 95 | File localFile = new File(CONFIG_FILE_NAME); 96 | if (!localFile.exists()) { 97 | try { 98 | URL codePathUrl = RadarBackend.class.getProtectionDomain().getCodeSource() 99 | .getLocation(); 100 | String codePath = codePathUrl.toURI().getPath(); 101 | String codeDir = codePath.substring(0, codePath.lastIndexOf('/') + 1); 102 | localFile = new File(codeDir, CONFIG_FILE_NAME); 103 | } catch (URISyntaxException ex) { 104 | throw new IOException("Cannot get path of executable", ex); 105 | } 106 | } 107 | return localFile; 108 | } 109 | 110 | 111 | @Override 112 | public KafkaProperty getKafkaProperties() { 113 | if (this.kafkaProperty == null) { 114 | this.kafkaProperty = new KafkaProperty(getRadarProperties()); 115 | } 116 | return this.kafkaProperty; 117 | } 118 | 119 | @Override 120 | public PersistentStateStore getPersistentStateStore() throws IOException { 121 | if (getRadarProperties().getPersistencePath() != null) { 122 | File persistenceDir = new File(getRadarProperties().getPersistencePath()); 123 | return new YamlPersistentStateStore(persistenceDir); 124 | } else { 125 | return null; 126 | } 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /src/integrationTest/docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '2' 3 | 4 | networks: 5 | kafka: 6 | driver: bridge 7 | services: 8 | #---------------------------------------------------------------------------# 9 | # Zookeeper Cluster # 10 | #---------------------------------------------------------------------------# 11 | zookeeper-1: 12 | image: confluentinc/cp-zookeeper:4.1.0 13 | networks: 14 | - kafka 15 | environment: 16 | ZOOKEEPER_SERVER_ID: 1 17 | ZOOKEEPER_CLIENT_PORT: 2181 18 | ZOOKEEPER_TICK_TIME: 1000 19 | ZOOKEEPER_INIT_LIMIT: 5 20 | ZOOKEEPER_SYNC_LIMIT: 2 21 | ZOOKEEPER_SERVERS: zookeeper-1:2888:3888 22 | 23 | #---------------------------------------------------------------------------# 24 | # Kafka Cluster # 25 | #---------------------------------------------------------------------------# 26 | kafka-1: 27 | image: confluentinc/cp-kafka:4.1.0 28 | depends_on: 29 | - zookeeper-1 30 | networks: 31 | - kafka 32 | environment: 33 | KAFKA_BROKER_ID: 1 34 | KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181 35 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:9092 36 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" 37 | KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS: 5000 38 | KAFKA_INTER_BROKER_PROTOCOL_VERSION: 0.11.0 39 | KAFKA_LOG_MESSAGE_FORMAT_VERSION: 0.11.0 40 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 41 | 42 | kafka-2: 43 | image: confluentinc/cp-kafka:4.1.0 44 | depends_on: 45 | - zookeeper-1 46 | networks: 47 | - kafka 48 | environment: 49 | KAFKA_BROKER_ID: 2 50 | KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181 51 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:9092 52 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" 53 | KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS: 5000 54 | KAFKA_INTER_BROKER_PROTOCOL_VERSION: 0.11.0 55 | KAFKA_LOG_MESSAGE_FORMAT_VERSION: 0.11.0 56 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 57 | 58 | kafka-3: 59 | image: confluentinc/cp-kafka:4.1.0 60 | depends_on: 61 | - zookeeper-1 62 | networks: 63 | - kafka 64 | environment: 65 | KAFKA_BROKER_ID: 3 66 | KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181 67 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:9092 68 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" 69 | KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS: 5000 70 | KAFKA_INTER_BROKER_PROTOCOL_VERSION: 0.11.0 71 | KAFKA_LOG_MESSAGE_FORMAT_VERSION: 0.11.0 72 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 73 | 74 | #---------------------------------------------------------------------------# 75 | # Schema Registry # 76 | #---------------------------------------------------------------------------# 77 | schema-registry-1: 78 | image: confluentinc/cp-schema-registry:4.1.0 79 | depends_on: 80 | - zookeeper-1 81 | - kafka-1 82 | - kafka-2 83 | - kafka-3 84 | networks: 85 | - kafka 86 | restart: always 87 | ports: 88 | - "8081:8081" 89 | environment: 90 | SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper-1:2181 91 | SCHEMA_REGISTRY_HOST_NAME: schema-registry-1 92 | SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 93 | KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS: 5000 94 | 95 | #---------------------------------------------------------------------------# 96 | # REST proxy # 97 | #---------------------------------------------------------------------------# 98 | rest-proxy-1: 99 | image: confluentinc/cp-kafka-rest:4.1.0 100 | depends_on: 101 | - kafka-1 102 | - kafka-2 103 | - kafka-3 104 | - schema-registry-1 105 | networks: 106 | - kafka 107 | ports: 108 | - "8082:8082" 109 | environment: 110 | KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper-1:2181 111 | KAFKA_REST_LISTENERS: http://rest-proxy-1:8082 112 | KAFKA_REST_SCHEMA_REGISTRY_URL: http://schema-registry-1:8081 113 | KAFKA_REST_HOST_NAME: rest-proxy-1 114 | KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS: 5000 115 | 116 | #---------------------------------------------------------------------------# 117 | # Integration test # 118 | #---------------------------------------------------------------------------# 119 | integration-test: 120 | build: 121 | context: ../../.. 122 | dockerfile: src/integrationTest/docker/Dockerfile 123 | # Right now, only direct connections to kafka are tested 124 | depends_on: 125 | - kafka-1 126 | - schema-registry-1 127 | networks: 128 | - kafka 129 | command: integrationTest 130 | volumes: 131 | - ../../../build/jacoco:/code/build/jacoco 132 | - ../../../build/reports:/code/build/reports 133 | -------------------------------------------------------------------------------- /src/test/java/org/radarcns/monitor/BatteryLevelMonitorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.monitor; 18 | 19 | import static org.hamcrest.MatcherAssert.assertThat; 20 | import static org.hamcrest.Matchers.hasEntry; 21 | import static org.mockito.ArgumentMatchers.anyString; 22 | import static org.mockito.Mockito.*; 23 | import static org.radarcns.monitor.BatteryLevelMonitor.Status.LOW; 24 | 25 | import java.io.File; 26 | import java.util.Collections; 27 | import java.util.Map; 28 | import javax.mail.MessagingException; 29 | import org.apache.avro.generic.GenericData.Record; 30 | import org.apache.kafka.clients.consumer.ConsumerRecord; 31 | import org.junit.Rule; 32 | import org.junit.Test; 33 | import org.junit.rules.TemporaryFolder; 34 | import org.radarcns.config.ConfigRadar; 35 | import org.radarcns.config.RadarPropertyHandler; 36 | import org.radarcns.kafka.ObservationKey; 37 | import org.radarcns.monitor.BatteryLevelMonitor.BatteryLevelState; 38 | import org.radarcns.passive.empatica.EmpaticaE4BatteryLevel; 39 | import org.radarcns.util.EmailSender; 40 | import org.radarcns.util.EmailSenders; 41 | import org.radarcns.util.YamlPersistentStateStore; 42 | 43 | public class BatteryLevelMonitorTest { 44 | 45 | @Rule 46 | public TemporaryFolder folder = new TemporaryFolder(); 47 | 48 | private long offset; 49 | private long timeReceived; 50 | private int timesSent; 51 | private EmailSenders senders; 52 | private EmailSender sender; 53 | 54 | private static final String PROJECT_ID = "test"; 55 | 56 | @Test 57 | public void evaluateRecord() throws Exception { 58 | offset = 1000L; 59 | timeReceived = 2000L; 60 | timesSent = 0; 61 | sender = mock(EmailSender.class); 62 | 63 | senders = new EmailSenders(Collections.singletonMap(PROJECT_ID, sender)); 64 | 65 | ConfigRadar config = KafkaMonitorFactoryTest 66 | .getBatteryMonitorConfig(25252, folder); 67 | RadarPropertyHandler properties = KafkaMonitorFactoryTest 68 | .getRadarPropertyHandler(config, folder); 69 | 70 | BatteryLevelMonitor monitor = new BatteryLevelMonitor(properties, 71 | Collections.singletonList("mytopic"), senders, LOW, 10L); 72 | 73 | sendMessage(monitor, 1.0f, false); 74 | sendMessage(monitor, 1.0f, false); 75 | sendMessage(monitor, 0.1f, true); 76 | sendMessage(monitor, 0.1f, false); 77 | sendMessage(monitor, 0.3f, false); 78 | sendMessage(monitor, 0.4f, false); 79 | sendMessage(monitor, 0.01f, true); 80 | sendMessage(monitor, 0.01f, false); 81 | sendMessage(monitor, 0.1f, false); 82 | sendMessage(monitor, 0.1f, false); 83 | sendMessage(monitor, 0.01f, true); 84 | sendMessage(monitor, 1f, false); 85 | } 86 | 87 | private void sendMessage(BatteryLevelMonitor monitor, float batteryLevel, boolean sentMessage) 88 | throws MessagingException { 89 | Record key = new Record(ObservationKey.getClassSchema()); 90 | key.put("projectId", PROJECT_ID); 91 | key.put("sourceId", "1"); 92 | key.put("userId", "me"); 93 | 94 | Record value = new Record(EmpaticaE4BatteryLevel.getClassSchema()); 95 | value.put("time", timeReceived); 96 | value.put("timeReceived", timeReceived++); 97 | value.put("batteryLevel", batteryLevel); 98 | monitor.evaluateRecord(new ConsumerRecord<>("mytopic", 0, offset++, key, value)); 99 | 100 | if (sentMessage) { 101 | timesSent++; 102 | } 103 | verify(sender, times(timesSent)).sendEmail(anyString(), anyString()); 104 | } 105 | 106 | @Test 107 | public void retrieveState() throws Exception { 108 | File base = folder.newFolder(); 109 | YamlPersistentStateStore stateStore = new YamlPersistentStateStore(base); 110 | BatteryLevelState state = new BatteryLevelState(); 111 | ObservationKey key1 = new ObservationKey("test", "a", "b"); 112 | String keyString = stateStore.keyToString(key1); 113 | state.updateLevel(keyString, 0.1f); 114 | stateStore.storeState("one", "two", state); 115 | 116 | YamlPersistentStateStore stateStore2 = new YamlPersistentStateStore(base); 117 | BatteryLevelState state2 = stateStore2.retrieveState("one", "two", new BatteryLevelState()); 118 | Map values = state2.getLevels(); 119 | assertThat(values, hasEntry(keyString, 0.1f)); 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/monitor/KafkaMonitorFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.monitor; 18 | 19 | import java.io.IOException; 20 | import java.util.Arrays; 21 | import java.util.Collection; 22 | import java.util.Collections; 23 | import java.util.Locale; 24 | import java.util.stream.Stream; 25 | import org.radarcns.config.BatteryMonitorConfig; 26 | import org.radarcns.config.DisconnectMonitorConfig; 27 | import org.radarcns.config.MonitorConfig; 28 | import org.radarcns.config.RadarBackendOptions; 29 | import org.radarcns.config.RadarPropertyHandler; 30 | import org.radarcns.util.EmailSenders; 31 | import org.slf4j.Logger; 32 | import org.slf4j.LoggerFactory; 33 | 34 | public class KafkaMonitorFactory { 35 | 36 | private static final Logger logger = LoggerFactory.getLogger(KafkaMonitorFactory.class); 37 | 38 | private final RadarPropertyHandler properties; 39 | private final RadarBackendOptions options; 40 | 41 | public KafkaMonitorFactory(RadarBackendOptions options, 42 | RadarPropertyHandler properties) { 43 | this.options = options; 44 | this.properties = properties; 45 | } 46 | 47 | public KafkaMonitor createMonitor() throws IOException { 48 | KafkaMonitor monitor; 49 | String[] args = options.getSubCommandArgs(); 50 | String commandType; 51 | if (args == null || args.length == 0) { 52 | commandType = "all"; 53 | } else { 54 | commandType = args[0]; 55 | } 56 | 57 | switch (commandType) { 58 | case "battery": 59 | monitor = createBatteryLevelMonitor(); 60 | break; 61 | case "disconnect": 62 | monitor = createDisconnectMonitor(); 63 | break; 64 | case "all": 65 | monitor = new CombinedKafkaMonitor( 66 | Stream.of(createDisconnectMonitor(), createBatteryLevelMonitor())); 67 | break; 68 | default: 69 | throw new IllegalArgumentException("Cannot create unknown monitor " + commandType); 70 | } 71 | if (monitor == null) { 72 | throw new IllegalArgumentException("Monitor " + commandType + " is not configured."); 73 | } 74 | return monitor; 75 | } 76 | 77 | private KafkaMonitor createBatteryLevelMonitor() throws IOException { 78 | BatteryMonitorConfig config = properties.getRadarProperties().getBatteryMonitor(); 79 | 80 | if (config == null) { 81 | logger.warn("Battery level monitor is not configured. Cannot start it."); 82 | return null; 83 | } 84 | 85 | BatteryLevelMonitor.Status minLevel = BatteryLevelMonitor.Status.CRITICAL; 86 | EmailSenders senders = getSenders(config); 87 | Collection topics = getTopics(config, "android_empatica_e4_battery_level"); 88 | 89 | if (config.getLevel() != null) { 90 | String level = config.getLevel().toUpperCase(Locale.US); 91 | try { 92 | minLevel = BatteryLevelMonitor.Status.valueOf(level); 93 | } catch (IllegalArgumentException ex) { 94 | logger.warn("Minimum battery level {} is not recognized. " 95 | + "Choose from {} instead. Using CRITICAL.", 96 | level, Arrays.toString(BatteryLevelMonitor.Status.values())); 97 | } 98 | } 99 | long logInterval = config.getLogInterval(); 100 | 101 | return new BatteryLevelMonitor(properties, topics, senders, minLevel, logInterval); 102 | } 103 | 104 | private KafkaMonitor createDisconnectMonitor() 105 | throws IOException { 106 | DisconnectMonitorConfig config = properties.getRadarProperties().getDisconnectMonitor(); 107 | if (config == null) { 108 | logger.warn("Disconnect monitor is not configured. Cannot start it."); 109 | return null; 110 | } 111 | EmailSenders senders = getSenders(config); 112 | Collection topics = getTopics(config, "android_empatica_e4_temperature"); 113 | return new DisconnectMonitor(properties, topics, "disconnect_monitor", senders); 114 | } 115 | 116 | 117 | private EmailSenders getSenders(MonitorConfig config) throws IOException { 118 | if (config != null && config.getNotifyConfig() != null) { 119 | return EmailSenders.parseConfig(config); 120 | } 121 | return null; 122 | } 123 | 124 | private Collection getTopics(MonitorConfig config, String defaultTopic) { 125 | if (config != null && config.getTopics() != null) { 126 | return config.getTopics(); 127 | } else { 128 | return Collections.singleton(defaultTopic); 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/test/java/org/radarcns/config/RadarPropertyHandlerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.config; 18 | 19 | import static org.hamcrest.MatcherAssert.assertThat; 20 | import static org.hamcrest.Matchers.hasEntry; 21 | import static org.junit.Assert.assertNotNull; 22 | import static org.junit.Assert.assertNull; 23 | 24 | import com.fasterxml.jackson.databind.JsonMappingException; 25 | import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException; 26 | import java.lang.reflect.Field; 27 | import org.junit.Before; 28 | import org.junit.Rule; 29 | import org.junit.Test; 30 | import org.junit.rules.ExpectedException; 31 | 32 | /** 33 | * Created by nivethika on 19-12-16. 34 | */ 35 | public class RadarPropertyHandlerTest { 36 | 37 | private RadarPropertyHandler propertyHandler ; 38 | 39 | @Before 40 | public void setUp() { 41 | 42 | this.propertyHandler = new RadarPropertyHandlerImpl(); 43 | 44 | } 45 | 46 | @Rule 47 | public ExpectedException exception = ExpectedException.none(); 48 | 49 | @Test 50 | public void getInstanceEmptyProperties() throws NoSuchFieldException, IllegalAccessException, SecurityException { 51 | Field properties = RadarPropertyHandlerImpl.class.getDeclaredField("properties"); 52 | properties.setAccessible(true); 53 | properties.set(this.propertyHandler,null); 54 | exception.expect(IllegalStateException.class); 55 | exception.expectMessage("Properties cannot be accessed without calling load() first"); 56 | propertyHandler.getRadarProperties(); 57 | } 58 | 59 | @Test 60 | public void loadWithInvalidFilePath() throws Exception { 61 | Field properties = RadarPropertyHandlerImpl.class.getDeclaredField("properties"); 62 | properties.setAccessible(true); 63 | properties.set(this.propertyHandler, null); 64 | exception.expect(IllegalArgumentException.class); 65 | exception.expectMessage("Config file /usr is invalid"); 66 | String invalidPath = "/usr/"; 67 | propertyHandler.load(invalidPath); 68 | } 69 | 70 | @Test 71 | public void load() throws Exception { 72 | Field propertiess = RadarPropertyHandlerImpl.class.getDeclaredField("properties"); 73 | propertiess.setAccessible(true); 74 | propertiess.set(this.propertyHandler,null); 75 | propertyHandler.load("src/test/resources/config/radar.yml"); 76 | 77 | ConfigRadar properties = propertyHandler.getRadarProperties(); 78 | assertNotNull(properties.getBroker()); 79 | assertNotNull(properties.getBrokerPaths()); 80 | assertNotNull(properties.getReleased()); 81 | assertNotNull(properties.getSchemaRegistry()); 82 | assertNotNull(properties.getSchemaRegistryPaths()); 83 | assertNotNull(properties.getZookeeper()); 84 | assertNotNull(properties.getZookeeperPaths()); 85 | assertNotNull(properties.getVersion()); 86 | assertThat(properties.getExtras(), hasEntry("somethingother", "bla")); 87 | } 88 | 89 | @Test 90 | public void loadInvalidYaml() throws Exception { 91 | exception.expect(UnrecognizedPropertyException.class); 92 | propertyHandler.load("src/test/resources/config/invalidradar.yml"); 93 | } 94 | 95 | @Test 96 | public void loadInvalidStreamPriority() throws Exception { 97 | exception.expect(JsonMappingException.class); 98 | propertyHandler.load("src/test/resources/config/invalid_stream_priority.yml"); 99 | } 100 | 101 | @Test 102 | public void loadWithInstance() throws Exception { 103 | 104 | exception.expect(IllegalStateException.class); 105 | exception.expectMessage("Properties class has been already loaded"); 106 | propertyHandler.load("radar.yml"); 107 | propertyHandler.load("again.yml"); 108 | ConfigRadar propertiesS = propertyHandler.getRadarProperties(); 109 | assertNotNull(propertiesS); 110 | } 111 | 112 | @Test 113 | public void getKafkaPropertiesBeforeLoad() throws IllegalAccessException, NoSuchFieldException { 114 | Field properties = RadarPropertyHandlerImpl.class.getDeclaredField("properties"); 115 | properties.setAccessible(true); 116 | properties.set(this.propertyHandler,null); 117 | exception.expect(IllegalStateException.class); 118 | exception.expectMessage("Properties cannot be accessed without calling load() first"); 119 | KafkaProperty property =propertyHandler.getKafkaProperties(); 120 | assertNull(property); 121 | } 122 | 123 | @Test 124 | public void getKafkaProperties() throws Exception 125 | { 126 | Field properties = RadarPropertyHandlerImpl.class.getDeclaredField("properties"); 127 | properties.setAccessible(true); 128 | properties.set(propertyHandler,null); 129 | propertyHandler.load("radar.yml"); 130 | KafkaProperty property =propertyHandler.getKafkaProperties(); 131 | assertNotNull(property); 132 | } 133 | 134 | } 135 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/monitor/CombinedKafkaMonitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.monitor; 18 | 19 | import java.io.IOException; 20 | import java.time.Duration; 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | import java.util.Objects; 24 | import java.util.concurrent.ExecutorService; 25 | import java.util.concurrent.Executors; 26 | import java.util.concurrent.TimeUnit; 27 | import java.util.concurrent.atomic.AtomicBoolean; 28 | import java.util.stream.Collectors; 29 | import java.util.stream.Stream; 30 | import org.slf4j.Logger; 31 | import org.slf4j.LoggerFactory; 32 | 33 | /** 34 | * Runs multiple monitors, each in its own thread. 35 | */ 36 | public class CombinedKafkaMonitor implements KafkaMonitor { 37 | private static final Logger logger = LoggerFactory.getLogger(CombinedKafkaMonitor.class); 38 | 39 | private final List monitors; 40 | private final AtomicBoolean done; 41 | 42 | private ExecutorService executor; 43 | private IOException ioException; 44 | private InterruptedException interruptedException; 45 | 46 | public CombinedKafkaMonitor(Stream monitors) { 47 | this.monitors = Objects.requireNonNull(monitors) 48 | .filter(Objects::nonNull) 49 | .collect(Collectors.toList()); 50 | 51 | if (this.monitors.isEmpty()) { 52 | throw new IllegalArgumentException("Monitor collection may not be empty"); 53 | } 54 | this.done = new AtomicBoolean(false); 55 | this.executor = null; 56 | this.ioException = null; 57 | this.interruptedException = null; 58 | } 59 | 60 | @Override 61 | public Duration getPollTimeout() { 62 | return monitors.get(0).getPollTimeout(); 63 | } 64 | 65 | @Override 66 | public void setPollTimeout(Duration pollTimeout) { 67 | for (KafkaMonitor monitor : monitors) { 68 | monitor.setPollTimeout(pollTimeout); 69 | } 70 | } 71 | 72 | @Override 73 | public void start() throws IOException, InterruptedException { 74 | synchronized (this) { 75 | if (executor != null) { 76 | throw new IllegalStateException("Cannot start monitor twice"); 77 | } 78 | executor = Executors.newFixedThreadPool(monitors.size()); 79 | } 80 | 81 | for (KafkaMonitor monitor : monitors) { 82 | executor.submit(() -> { 83 | try { 84 | monitor.start(); 85 | } catch (IOException ex) { 86 | setIoException(ex); 87 | } catch (InterruptedException ex) { 88 | setInterruptedException(ex); 89 | } 90 | }); 91 | } 92 | 93 | executor.shutdown(); 94 | executor.awaitTermination(366_000, TimeUnit.DAYS); // > 1000 years... 95 | 96 | if (getIoException() != null) { 97 | throw getIoException(); 98 | } 99 | if (getInterruptedException() != null) { 100 | throw getInterruptedException(); 101 | } 102 | } 103 | 104 | private synchronized IOException getIoException() { 105 | return ioException; 106 | } 107 | 108 | private synchronized void setIoException(IOException ioException) { 109 | this.ioException = ioException; 110 | initiateShutdownIgnoreException(); 111 | } 112 | 113 | private synchronized InterruptedException getInterruptedException() { 114 | return interruptedException; 115 | } 116 | 117 | private synchronized void setInterruptedException(InterruptedException interruptedException) { 118 | this.interruptedException = interruptedException; 119 | initiateShutdownIgnoreException(); 120 | } 121 | 122 | private void initiateShutdownIgnoreException() { 123 | try { 124 | initiateShutdown(); 125 | } catch (InterruptedException ex) { 126 | logger.info("Ignoring additional InterruptedException", ex); 127 | } catch (IOException ex) { 128 | logger.info("Ignoring additional IOException", ex); 129 | } 130 | } 131 | 132 | private void initiateShutdown() throws IOException, InterruptedException { 133 | if (!done.getAndSet(true)) { 134 | for (KafkaMonitor monitor : monitors) { 135 | monitor.shutdown(); 136 | } 137 | } 138 | } 139 | 140 | @Override 141 | public void shutdown() throws IOException, InterruptedException { 142 | synchronized (this) { 143 | if (executor == null) { 144 | return; 145 | } 146 | } 147 | initiateShutdown(); 148 | executor.awaitTermination(30, TimeUnit.SECONDS); 149 | executor.shutdownNow(); 150 | } 151 | 152 | @Override 153 | public boolean isShutdown() { 154 | return done.get(); 155 | } 156 | 157 | public List getMonitors() { 158 | return new ArrayList<>(monitors); 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/config/ConfigRadar.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns.config; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | import java.util.Date; 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | /** 25 | * POJO representing the yml file 26 | */ 27 | public class ConfigRadar { 28 | private Date released; 29 | private String version; 30 | private List zookeeper; 31 | private List broker; 32 | @JsonProperty("schema_registry") 33 | private List schemaRegistry; 34 | @JsonProperty("rest_proxy") 35 | private ServerConfig restProxy; 36 | @JsonProperty("battery_monitor") 37 | private BatteryMonitorConfig batteryMonitor; 38 | @JsonProperty("disconnect_monitor") 39 | private DisconnectMonitorConfig disconnectMonitor; 40 | @JsonProperty("statistics_monitors") 41 | private List statisticsMonitors; 42 | @JsonProperty("stream") 43 | private StreamConfig stream; 44 | @JsonProperty("persistence_path") 45 | private String persistencePath; 46 | private Map extras; 47 | 48 | @JsonProperty("build_version") 49 | private String buildVersion; 50 | 51 | public Date getReleased() { 52 | return released; 53 | } 54 | 55 | public void setReleased(Date released) { 56 | this.released = released; 57 | } 58 | 59 | public String getVersion() { 60 | return version; 61 | } 62 | 63 | public void setVersion(String version) { 64 | this.version = version; 65 | } 66 | 67 | public List getZookeeper() { 68 | return zookeeper; 69 | } 70 | 71 | public void setZookeeper(List zookeeper) { 72 | this.zookeeper = zookeeper; 73 | } 74 | 75 | public List getBroker() { 76 | return broker; 77 | } 78 | 79 | public void setBroker(List broker) { 80 | this.broker = broker; 81 | } 82 | 83 | public List getSchemaRegistry() { 84 | return schemaRegistry; 85 | } 86 | 87 | public void setSchemaRegistry(List schemaRegistry) { 88 | this.schemaRegistry = schemaRegistry; 89 | } 90 | 91 | public ServerConfig getRestProxy() { 92 | return restProxy; 93 | } 94 | 95 | public void setRestProxy(ServerConfig restProxy) { 96 | this.restProxy = restProxy; 97 | } 98 | 99 | public String getZookeeperPaths() { 100 | if (zookeeper == null) { 101 | throw new IllegalStateException("'zookeeper' is not configured"); 102 | } 103 | return ServerConfig.getPaths(zookeeper); 104 | } 105 | 106 | public String getBrokerPaths() { 107 | if (broker == null) { 108 | throw new IllegalStateException("Kafka 'broker' is not configured"); 109 | } 110 | return ServerConfig.getPaths(broker); 111 | } 112 | 113 | public String getSchemaRegistryPaths() { 114 | if (schemaRegistry == null) { 115 | throw new IllegalStateException("'schema_registry' is not configured"); 116 | } 117 | 118 | return ServerConfig.getPaths(schemaRegistry); 119 | } 120 | 121 | public String getRestProxyPath() { 122 | if (restProxy == null) { 123 | throw new IllegalStateException("'rest_proxy' is not configured"); 124 | } 125 | 126 | return restProxy.getPath(); 127 | } 128 | 129 | public BatteryMonitorConfig getBatteryMonitor() { 130 | return batteryMonitor; 131 | } 132 | 133 | public void setBatteryMonitor(BatteryMonitorConfig batteryMonitor) { 134 | this.batteryMonitor = batteryMonitor; 135 | } 136 | 137 | public DisconnectMonitorConfig getDisconnectMonitor() { 138 | return disconnectMonitor; 139 | } 140 | 141 | public void setDisconnectMonitor(DisconnectMonitorConfig disconnectMonitor) { 142 | this.disconnectMonitor = disconnectMonitor; 143 | } 144 | 145 | public String getPersistencePath() { 146 | return persistencePath; 147 | } 148 | 149 | public void setPersistencePath(String persistencePath) { 150 | this.persistencePath = persistencePath; 151 | } 152 | 153 | public Map getExtras() { 154 | return extras; 155 | } 156 | 157 | public void setExtras(Map extras) { 158 | this.extras = extras; 159 | } 160 | 161 | @Override 162 | public String toString() { 163 | return new YamlConfigLoader().prettyString(this); 164 | } 165 | 166 | public String getBuildVersion() { 167 | return buildVersion; 168 | } 169 | 170 | public void setBuildVersion(String buildVersion) { 171 | this.buildVersion = buildVersion; 172 | } 173 | 174 | public List getStatisticsMonitors() { 175 | return statisticsMonitors; 176 | } 177 | 178 | public void setStatisticsMonitors(List statisticsMonitors) { 179 | this.statisticsMonitors = statisticsMonitors; 180 | } 181 | 182 | public StreamConfig getStream() { 183 | return stream; 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /src/main/java/org/radarcns/RadarBackend.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 King's College London and The Hyve 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.radarcns; 18 | 19 | import java.io.IOException; 20 | import java.util.Arrays; 21 | import javax.annotation.Nonnull; 22 | import org.apache.commons.cli.ParseException; 23 | import org.radarcns.config.RadarBackendOptions; 24 | import org.radarcns.config.RadarPropertyHandler; 25 | import org.radarcns.config.SubCommand; 26 | import org.radarcns.monitor.KafkaMonitorFactory; 27 | import org.radarcns.producer.MockProducerCommand; 28 | import org.radarcns.stream.KafkaStreamFactory; 29 | import org.radarcns.util.RadarSingletonFactory; 30 | import org.slf4j.Logger; 31 | import org.slf4j.LoggerFactory; 32 | 33 | /** 34 | * Core class that initialises configurations and then start all needed Kafka streams 35 | */ 36 | public final class RadarBackend { 37 | private static final Logger log = LoggerFactory.getLogger(RadarBackend.class); 38 | private final RadarBackendOptions options; 39 | private final RadarPropertyHandler radarPropertyHandler; 40 | private SubCommand command; 41 | 42 | public RadarBackend(@Nonnull RadarBackendOptions options) throws IOException { 43 | this(options, createPropertyHandler(options)); 44 | } 45 | 46 | public RadarBackend(@Nonnull RadarBackendOptions options, 47 | @Nonnull RadarPropertyHandler properties) { 48 | this.options = options; 49 | this.radarPropertyHandler = properties; 50 | 51 | log.info("Configuration successfully updated"); 52 | log.info("radar.yml configuration: {}", radarPropertyHandler.getRadarProperties()); 53 | } 54 | 55 | private static RadarPropertyHandler createPropertyHandler(@Nonnull RadarBackendOptions options) 56 | throws IOException { 57 | RadarPropertyHandler properties = RadarSingletonFactory.getRadarPropertyHandler(); 58 | properties.load(options.getPropertyPath()); 59 | return properties; 60 | } 61 | 62 | public SubCommand createCommand() throws IOException { 63 | String subCommand = options.getSubCommand(); 64 | if (subCommand == null) { 65 | subCommand = "stream"; 66 | } 67 | switch (subCommand) { 68 | case "stream": 69 | return new KafkaStreamFactory(options, radarPropertyHandler) 70 | .createSensorStreams(); 71 | case "statistics": 72 | return new KafkaStreamFactory(options, radarPropertyHandler) 73 | .createStreamStatistics(); 74 | case "monitor": 75 | return new KafkaMonitorFactory(options, radarPropertyHandler).createMonitor(); 76 | case "mock": 77 | return new MockProducerCommand(options, radarPropertyHandler); 78 | default: 79 | throw new IllegalArgumentException("Unknown subcommand " 80 | + options.getSubCommand()); 81 | } 82 | } 83 | 84 | /** 85 | * It starts streams and sets a ShutdownHook to close streams while closing the application 86 | */ 87 | public void application() { 88 | try { 89 | start(); 90 | } catch (IOException ex) { 91 | log.error("FATAL ERROR! The current instance cannot start", ex); 92 | System.exit(1); 93 | } catch (InterruptedException ex) { 94 | log.error("The current instance was interrupted", ex); 95 | } 96 | 97 | Runtime.getRuntime().addShutdownHook(new Thread(() -> { 98 | try { 99 | shutdown(); 100 | } catch (Exception ex) { 101 | log.error("Impossible to finalise the shutdown hook", ex); 102 | } 103 | })); 104 | } 105 | 106 | /** 107 | * Start here all needed StreamMaster 108 | * 109 | * @throws IOException if the command failed to start up 110 | * @throws InterruptedException if the command was interrupted 111 | */ 112 | public void start() throws IOException, InterruptedException { 113 | log.info("STARTING"); 114 | 115 | command = createCommand(); 116 | command.start(); 117 | 118 | log.info("STARTED"); 119 | } 120 | 121 | /** 122 | * Stop here all commands started inside {@link #start()}. 123 | * 124 | * @throws IOException if the command failed to shut down 125 | * @throws InterruptedException if the command was interrupted before completely shutting down 126 | */ 127 | public void shutdown() throws InterruptedException, IOException { 128 | log.info("SHUTTING DOWN"); 129 | 130 | command.shutdown(); 131 | 132 | log.info("FINISHED"); 133 | } 134 | 135 | public static void main(String[] args) { 136 | try { 137 | RadarBackendOptions options = RadarBackendOptions.parse(args); 138 | RadarBackend backend = new RadarBackend(options); 139 | backend.application(); 140 | } catch (ParseException ex) { 141 | log.error("Cannot parse arguments {}. Valid options are:\n{}", 142 | Arrays.toString(args), RadarBackendOptions.OPTIONS); 143 | System.exit(1); 144 | } catch (Exception ex) { 145 | log.error("Failed to run command", ex); 146 | System.exit(1); 147 | } 148 | } 149 | } 150 | --------------------------------------------------------------------------------