├── conf ├── exdemon-env.sh └── log4j.properties ├── src ├── test │ ├── resources │ │ ├── .gitignore │ │ └── log4j.properties │ └── java │ │ └── ch │ │ └── cern │ │ ├── test │ │ └── Utils.java │ │ ├── exdemon │ │ ├── components │ │ │ ├── source │ │ │ │ └── types │ │ │ │ │ ├── TestComponentsSource.java │ │ │ │ │ └── ZookeeperComponentsSourceIntegrationTest.java │ │ │ └── ConfigurationResultTest.java │ │ ├── metrics │ │ │ ├── JSONMetric.java │ │ │ └── defined │ │ │ │ ├── equation │ │ │ │ └── var │ │ │ │ │ ├── TestVariableStatus.java │ │ │ │ │ └── agg │ │ │ │ │ ├── AvgAgregationTest.java │ │ │ │ │ ├── CountAgregationTest.java │ │ │ │ │ └── AggregationValuesTest.java │ │ │ │ └── DefinedMetricIDTest.java │ │ ├── monitor │ │ │ ├── analysis │ │ │ │ ├── types │ │ │ │ │ ├── AlwaysTrueAnalysisTest.java │ │ │ │ │ └── htm │ │ │ │ │ │ └── ResultsToFileWriter.java │ │ │ │ ├── StringAnalysisTest.java │ │ │ │ ├── BooleanAnalysisTest.java │ │ │ │ ├── NumericAnalysisTest.java │ │ │ │ └── AnalysisTest.java │ │ │ └── trigger │ │ │ │ └── action │ │ │ │ └── ActionTest.java │ │ └── DriverTest.java │ │ ├── spark │ │ ├── status │ │ │ ├── TestStatus.java │ │ │ └── storage │ │ │ │ └── manager │ │ │ │ ├── ToStringPatternStatusKeyFilterTest.java │ │ │ │ ├── IDStatusKeyFilterTest.java │ │ │ │ └── ExpireStatusValueFilterTest.java │ │ ├── Batches.java │ │ └── TestInputStream.java │ │ └── utils │ │ ├── StringUtilsTest.java │ │ └── TimeUtilsTest.java └── main │ └── java │ └── ch │ └── cern │ ├── spark │ ├── status │ │ ├── StatusKey.java │ │ ├── IDStatusKey.java │ │ ├── HasStatus.java │ │ ├── storage │ │ │ ├── ClassNameAlias.java │ │ │ ├── StatusSerializer.java │ │ │ ├── manager │ │ │ │ ├── ExpireStatusValueFilter.java │ │ │ │ ├── IDStatusKeyFilter.java │ │ │ │ ├── ClassNameStatusKeyFilter.java │ │ │ │ ├── ValueSizeFilter.java │ │ │ │ └── ToStringPatternStatusKeyFilter.java │ │ │ ├── types │ │ │ │ └── ConsumerRecordSer.java │ │ │ └── JavaStatusSerializer.java │ │ ├── RemoveAndValue.java │ │ ├── StateDStream.java │ │ ├── StatusImpl.java │ │ └── StatusValue.java │ ├── SparkConf.java │ └── utils │ │ └── StructTypeUtils.java │ ├── Taggable.java │ ├── exdemon │ ├── struct │ │ ├── Driver.java │ │ └── Metric.java │ ├── monitor │ │ ├── trigger │ │ │ ├── action │ │ │ │ ├── template │ │ │ │ │ └── ValueSupplier.java │ │ │ │ ├── actuator │ │ │ │ │ ├── Actuators.java │ │ │ │ │ ├── Actuator.java │ │ │ │ │ └── RunActuatorsF.java │ │ │ │ ├── silence │ │ │ │ │ ├── Silences.java │ │ │ │ │ └── SilencesFilterF.java │ │ │ │ └── Action.java │ │ │ ├── TriggerStatus.java │ │ │ ├── TriggerStatusKey.java │ │ │ ├── types │ │ │ │ └── StatusesTrigger.java │ │ │ └── ComputeTriggerKeysF.java │ │ ├── analysis │ │ │ ├── results │ │ │ │ └── sink │ │ │ │ │ ├── AnalysisResultsSink.java │ │ │ │ │ └── types │ │ │ │ │ └── ElasticAnalysisResultsSink.java │ │ │ ├── types │ │ │ │ ├── AlwaysTrueAnalysis.java │ │ │ │ ├── NoneAnalysis.java │ │ │ │ └── htm │ │ │ │ │ ├── AnomaliesResults.java │ │ │ │ │ └── PersistableKryoSerializer.java │ │ │ ├── NumericAnalysis.java │ │ │ ├── StringAnalysis.java │ │ │ ├── BooleanAnalysis.java │ │ │ └── Analysis.java │ │ ├── MonitorStatusKey.java │ │ ├── ComputeMonitorKeysF.java │ │ └── UpdateMonitorStatusesF.java │ ├── metrics │ │ ├── Sink.java │ │ ├── defined │ │ │ ├── equation │ │ │ │ ├── ComputationException.java │ │ │ │ ├── ValueComputable.java │ │ │ │ ├── functions │ │ │ │ │ ├── num │ │ │ │ │ │ ├── MinusFunc.java │ │ │ │ │ │ ├── AddFunc.java │ │ │ │ │ │ ├── DivFunc.java │ │ │ │ │ │ ├── MulFunc.java │ │ │ │ │ │ ├── SubFunc.java │ │ │ │ │ │ ├── PowFunc.java │ │ │ │ │ │ ├── NumericFunction.java │ │ │ │ │ │ ├── BiNumericFunction.java │ │ │ │ │ │ ├── GTFunc.java │ │ │ │ │ │ ├── LTFunc.java │ │ │ │ │ │ ├── AbsFunc.java │ │ │ │ │ │ ├── CosFunc.java │ │ │ │ │ │ ├── SinFunc.java │ │ │ │ │ │ ├── TanFunc.java │ │ │ │ │ │ ├── SqrtFunc.java │ │ │ │ │ │ └── IfFloatFunc.java │ │ │ │ │ ├── FunctionCaller.java │ │ │ │ │ ├── bool │ │ │ │ │ │ ├── NotFunc.java │ │ │ │ │ │ ├── EqualFunc.java │ │ │ │ │ │ ├── NotEqualFunc.java │ │ │ │ │ │ ├── AndFunc.java │ │ │ │ │ │ ├── OrFunc.java │ │ │ │ │ │ └── IfBoolFunc.java │ │ │ │ │ └── string │ │ │ │ │ │ ├── StringFunction.java │ │ │ │ │ │ ├── BiStringFunction.java │ │ │ │ │ │ ├── TrimFunc.java │ │ │ │ │ │ ├── ConcatFunc.java │ │ │ │ │ │ └── IfStringFunc.java │ │ │ │ ├── var │ │ │ │ │ ├── VariableStatus.java │ │ │ │ │ ├── VariableCreationResult.java │ │ │ │ │ ├── agg │ │ │ │ │ │ ├── CountAgregation.java │ │ │ │ │ │ ├── CountFloatsAgregation.java │ │ │ │ │ │ ├── CountBoolsAgregation.java │ │ │ │ │ │ ├── CountStringsAgregation.java │ │ │ │ │ │ ├── CountTruesAgregation.java │ │ │ │ │ │ ├── CountFalsesAgregation.java │ │ │ │ │ │ ├── MaxAggregation.java │ │ │ │ │ │ ├── MinAggregation.java │ │ │ │ │ │ ├── SumAggregation.java │ │ │ │ │ │ ├── AvgAggregation.java │ │ │ │ │ │ ├── DiffAggregation.java │ │ │ │ │ │ ├── SuccessiveCountAgregation.java │ │ │ │ │ │ └── LastValueAggregation.java │ │ │ │ │ ├── VariableStatuses.java │ │ │ │ │ └── PropertiesVariable.java │ │ │ │ └── Equation.java │ │ │ ├── DefinedMetricStatuskey.java │ │ │ ├── ComputeDefinedMetricKeysF.java │ │ │ ├── ComputeBatchDefineMetricsF.java │ │ │ └── DefinedMetrics.java │ │ ├── predictor │ │ │ └── Prediction.java │ │ ├── DatedValue.java │ │ ├── schema │ │ │ ├── MetricSchemas.java │ │ │ └── MetricSchemasF.java │ │ ├── value │ │ │ ├── StringValue.java │ │ │ ├── AggregatedValue.java │ │ │ ├── PropertiesValue.java │ │ │ ├── ExceptionValue.java │ │ │ ├── BooleanValue.java │ │ │ └── FloatValue.java │ │ ├── filter │ │ │ ├── EqualMetricPredicate.java │ │ │ └── NotEqualMetricPredicate.java │ │ ├── source │ │ │ └── MetricsSource.java │ │ └── Metric.java │ ├── components │ │ ├── RegisterComponentType.java │ │ └── ComponentType.java │ ├── http │ │ └── JsonPOSTRequest.java │ ├── SparkKryoRegistrator.java │ └── json │ │ └── JSONParser.java │ ├── utils │ ├── LimitedQueue.java │ ├── ByteArray.java │ ├── Pair.java │ ├── IntEvictingQueue.java │ ├── IString.java │ └── ExceptionsCache.java │ ├── util │ └── function │ │ ├── AndPredicate.java │ │ └── OrPredicate.java │ └── properties │ └── ConfigurationException.java ├── doc ├── img │ ├── dataflow.png │ ├── analysis │ │ ├── recent.png │ │ ├── percentile.png │ │ ├── seasonal.png │ │ └── fixed-thresholds.png │ ├── trigger │ │ ├── constant.png │ │ └── percentage.png │ └── example-monitored-metric.png └── users-manual │ ├── metrics.md │ ├── metric-sources.md │ ├── silences.md │ ├── running.md │ └── metrics-filter.md ├── lib ├── http-appender-log4j-0.0.1.jar ├── elasticsearch-hadoop-6.2.2.jar └── zookeepercli-1.0.12-1.x86_64.rpm ├── .gitignore ├── graphics └── icons │ ├── border-error-icon.png │ ├── border-info-icon.png │ ├── empty-error-icon.png │ ├── empty-info-icon.png │ ├── filled-error-icon.png │ ├── filled-info-icon.png │ ├── border-success-icon.png │ ├── border-warning-icon.png │ ├── empty-success-icon.png │ ├── empty-warning-icon.png │ ├── filled-success-icon.png │ └── filled-warning-icon.png ├── .travis.yml ├── koji.conf ├── bin ├── kafka-topic-mng ├── restart ├── stop ├── status-manager └── status └── Dockerfile /conf/exdemon-env.sh: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/resources/.gitignore: -------------------------------------------------------------------------------- 1 | /write_mounts_outputs.csv 2 | -------------------------------------------------------------------------------- /doc/img/dataflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/doc/img/dataflow.png -------------------------------------------------------------------------------- /doc/img/analysis/recent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/doc/img/analysis/recent.png -------------------------------------------------------------------------------- /doc/img/trigger/constant.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/doc/img/trigger/constant.png -------------------------------------------------------------------------------- /doc/img/analysis/percentile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/doc/img/analysis/percentile.png -------------------------------------------------------------------------------- /doc/img/analysis/seasonal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/doc/img/analysis/seasonal.png -------------------------------------------------------------------------------- /doc/img/trigger/percentage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/doc/img/trigger/percentage.png -------------------------------------------------------------------------------- /lib/http-appender-log4j-0.0.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/lib/http-appender-log4j-0.0.1.jar -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.project 2 | /.classpath 3 | /target/ 4 | /.settings/ 5 | /.cache-main 6 | /.cache-tests 7 | 8 | /api/venv/ -------------------------------------------------------------------------------- /doc/img/example-monitored-metric.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/doc/img/example-monitored-metric.png -------------------------------------------------------------------------------- /graphics/icons/border-error-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/graphics/icons/border-error-icon.png -------------------------------------------------------------------------------- /graphics/icons/border-info-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/graphics/icons/border-info-icon.png -------------------------------------------------------------------------------- /graphics/icons/empty-error-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/graphics/icons/empty-error-icon.png -------------------------------------------------------------------------------- /graphics/icons/empty-info-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/graphics/icons/empty-info-icon.png -------------------------------------------------------------------------------- /graphics/icons/filled-error-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/graphics/icons/filled-error-icon.png -------------------------------------------------------------------------------- /graphics/icons/filled-info-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/graphics/icons/filled-info-icon.png -------------------------------------------------------------------------------- /lib/elasticsearch-hadoop-6.2.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/lib/elasticsearch-hadoop-6.2.2.jar -------------------------------------------------------------------------------- /lib/zookeepercli-1.0.12-1.x86_64.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/lib/zookeepercli-1.0.12-1.x86_64.rpm -------------------------------------------------------------------------------- /doc/img/analysis/fixed-thresholds.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/doc/img/analysis/fixed-thresholds.png -------------------------------------------------------------------------------- /graphics/icons/border-success-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/graphics/icons/border-success-icon.png -------------------------------------------------------------------------------- /graphics/icons/border-warning-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/graphics/icons/border-warning-icon.png -------------------------------------------------------------------------------- /graphics/icons/empty-success-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/graphics/icons/empty-success-icon.png -------------------------------------------------------------------------------- /graphics/icons/empty-warning-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/graphics/icons/empty-warning-icon.png -------------------------------------------------------------------------------- /graphics/icons/filled-success-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/graphics/icons/filled-success-icon.png -------------------------------------------------------------------------------- /graphics/icons/filled-warning-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlanza1/ExDeMon/HEAD/graphics/icons/filled-warning-icon.png -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | 3 | jdk: 4 | - oraclejdk8 5 | 6 | notifications: 7 | email: 8 | on_success: change 9 | on_failure: change -------------------------------------------------------------------------------- /koji.conf: -------------------------------------------------------------------------------- 1 | [koji] 2 | server = https://kojihub.cern.ch/kojihub 3 | weburl = https://koji.cern.ch/ 4 | topurl = https://koji.cern.ch/kojifiles 5 | krb_canon_host = no 6 | krb_rdns = False -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/status/StatusKey.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status; 2 | 3 | import java.io.Serializable; 4 | 5 | public interface StatusKey extends Serializable { 6 | 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/Taggable.java: -------------------------------------------------------------------------------- 1 | package ch.cern; 2 | 3 | import java.util.Map; 4 | 5 | public interface Taggable { 6 | 7 | public Map getTags(); 8 | 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/status/IDStatusKey.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status; 2 | 3 | public interface IDStatusKey extends StatusKey { 4 | 5 | public String getID(); 6 | 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/struct/Driver.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.struct; 2 | 3 | public class Driver { 4 | 5 | public static void main(String[] args) { 6 | 7 | } 8 | 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/trigger/action/template/ValueSupplier.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.trigger.action.template; 2 | 3 | public interface ValueSupplier { 4 | 5 | Object get(String key); 6 | 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/status/HasStatus.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status; 2 | 3 | public interface HasStatus { 4 | 5 | public void load(StatusValue store); 6 | 7 | public StatusValue save(); 8 | 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/Sink.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics; 2 | 3 | import org.apache.spark.streaming.api.java.JavaDStream; 4 | 5 | public interface Sink { 6 | 7 | public void sink(JavaDStream outputStream); 8 | 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/status/storage/ClassNameAlias.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status.storage; 2 | 3 | import java.lang.annotation.Retention; 4 | import java.lang.annotation.RetentionPolicy; 5 | 6 | @Retention(RetentionPolicy.RUNTIME) 7 | public @interface ClassNameAlias { 8 | 9 | String value(); 10 | 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/components/RegisterComponentType.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.components; 2 | 3 | import java.lang.annotation.Retention; 4 | import java.lang.annotation.RetentionPolicy; 5 | 6 | @Retention(RetentionPolicy.RUNTIME) 7 | public @interface RegisterComponentType { 8 | 9 | String value(); 10 | 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/ComputationException.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation; 2 | 3 | public class ComputationException extends Exception { 4 | 5 | private static final long serialVersionUID = 1671640460677970099L; 6 | 7 | public ComputationException(String message) { 8 | super(message); 9 | } 10 | 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/components/ComponentType.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.components; 2 | 3 | import java.lang.annotation.Retention; 4 | import java.lang.annotation.RetentionPolicy; 5 | 6 | import ch.cern.exdemon.components.Component.Type; 7 | 8 | @Retention(RetentionPolicy.RUNTIME) 9 | public @interface ComponentType { 10 | 11 | Type value(); 12 | 13 | } 14 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/test/Utils.java: -------------------------------------------------------------------------------- 1 | package ch.cern.test; 2 | 3 | import static org.junit.Assert.fail; 4 | 5 | import java.util.Optional; 6 | 7 | public class Utils { 8 | 9 | public static void assertNotPresent(Optional optional) { 10 | if(optional.isPresent()) 11 | fail(); 12 | } 13 | 14 | public static void assertPresent(Optional optional) { 15 | if(!optional.isPresent()) 16 | fail(); 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/trigger/action/actuator/Actuators.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.trigger.action.actuator; 2 | 3 | import org.apache.spark.streaming.api.java.JavaDStream; 4 | 5 | import ch.cern.exdemon.monitor.trigger.action.Action; 6 | import ch.cern.properties.Properties; 7 | 8 | public class Actuators { 9 | 10 | public static void run(JavaDStream actions, Properties propertiesSourceProps) { 11 | actions.foreachRDD(rdd -> rdd.foreach(new RunActuatorsF(propertiesSourceProps))); 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/trigger/action/silence/Silences.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.trigger.action.silence; 2 | 3 | import org.apache.spark.streaming.api.java.JavaDStream; 4 | 5 | import ch.cern.exdemon.monitor.trigger.action.Action; 6 | import ch.cern.properties.Properties; 7 | 8 | public class Silences { 9 | 10 | public static JavaDStream apply(JavaDStream actions, Properties componentsSourceProps) { 11 | return actions.filter(new SilencesFilterF(componentsSourceProps)); 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/ValueComputable.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation; 2 | 3 | import java.time.Instant; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.var.VariableStatuses; 6 | import ch.cern.exdemon.metrics.value.Value; 7 | 8 | public interface ValueComputable { 9 | 10 | public Value compute(VariableStatuses store, Instant time); 11 | 12 | /** 13 | * Serves the check casting (generic types are not checked when parsing equation) 14 | */ 15 | public Class returnType(); 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/predictor/Prediction.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.predictor; 2 | 3 | public class Prediction { 4 | 5 | private float value; 6 | private float standardDeviation; 7 | 8 | public Prediction(float value, float standardDeviation) { 9 | this.value = value; 10 | this.standardDeviation = standardDeviation; 11 | } 12 | 13 | public float getValue() { 14 | return value; 15 | } 16 | 17 | public float getStandardDeviation() { 18 | return standardDeviation; 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/utils/LimitedQueue.java: -------------------------------------------------------------------------------- 1 | package ch.cern.utils; 2 | 3 | import java.util.LinkedList; 4 | 5 | public class LimitedQueue extends LinkedList { 6 | 7 | private static final long serialVersionUID = 8496130829423632184L; 8 | 9 | private int limit; 10 | 11 | public LimitedQueue(int limit) { 12 | this.limit = limit; 13 | } 14 | 15 | @Override 16 | public boolean add(E o) { 17 | boolean added = super.add(o); 18 | while (added && size() > limit) { 19 | super.remove(); 20 | } 21 | return added; 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/status/storage/StatusSerializer.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status.storage; 2 | 3 | import java.io.IOException; 4 | import java.io.Serializable; 5 | 6 | import ch.cern.spark.status.StatusKey; 7 | import ch.cern.spark.status.StatusValue; 8 | 9 | public interface StatusSerializer extends Serializable{ 10 | 11 | public StatusKey toKey(byte[] bytes) throws IOException; 12 | 13 | public byte[] fromKey(StatusKey key) throws IOException; 14 | 15 | public StatusValue toValue(byte[] bytes) throws IOException; 16 | 17 | public byte[] fromValue(StatusValue value) throws IOException; 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/num/MinusFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.num; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | 7 | public class MinusFunc extends NumericFunction { 8 | 9 | public static String REPRESENTATION = "-"; 10 | 11 | public MinusFunc(ValueComputable... v) throws ParseException { 12 | super(REPRESENTATION, v); 13 | 14 | operationInTheMiddleForToString(); 15 | } 16 | 17 | @Override 18 | public float compute(float value) { 19 | return -value; 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/analysis/results/sink/AnalysisResultsSink.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.analysis.results.sink; 2 | 3 | import ch.cern.exdemon.components.Component; 4 | import ch.cern.exdemon.components.ComponentType; 5 | import ch.cern.exdemon.components.Component.Type; 6 | import ch.cern.exdemon.metrics.Sink; 7 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult; 8 | 9 | @ComponentType(Type.ANALYSIS_RESULTS_SINK) 10 | public abstract class AnalysisResultsSink extends Component implements Sink{ 11 | 12 | private static final long serialVersionUID = -2336360271932362626L; 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/util/function/AndPredicate.java: -------------------------------------------------------------------------------- 1 | package ch.cern.util.function; 2 | 3 | import java.util.function.Predicate; 4 | 5 | import lombok.Getter; 6 | 7 | public class AndPredicate implements Predicate{ 8 | 9 | @Getter 10 | private Predicate pred1, pred2; 11 | 12 | public AndPredicate(Predicate pred1, Predicate pred2) { 13 | this.pred1 = pred1; 14 | this.pred2 = pred2; 15 | } 16 | 17 | @Override 18 | public boolean test(T t) { 19 | return pred1.test(t) && pred2.test(t); 20 | } 21 | 22 | @Override 23 | public String toString() { 24 | return "(" + pred1 + " & " + pred2 + ")"; 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/util/function/OrPredicate.java: -------------------------------------------------------------------------------- 1 | package ch.cern.util.function; 2 | 3 | import java.util.function.Predicate; 4 | 5 | import lombok.Getter; 6 | 7 | public class OrPredicate implements Predicate{ 8 | 9 | @Getter 10 | private Predicate pred1, pred2; 11 | 12 | public OrPredicate(Predicate pred1, Predicate pred2) { 13 | this.pred1 = pred1; 14 | this.pred2 = pred2; 15 | } 16 | 17 | @Override 18 | public boolean test(T t) { 19 | return pred1.test(t) || pred2.test(t); 20 | } 21 | 22 | @Override 23 | public String toString() { 24 | return "(" + pred1 + " | " + pred2 + ")"; 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/utils/ByteArray.java: -------------------------------------------------------------------------------- 1 | package ch.cern.utils; 2 | 3 | import java.io.Serializable; 4 | 5 | import lombok.EqualsAndHashCode; 6 | 7 | @EqualsAndHashCode(callSuper=false) 8 | public class ByteArray implements Serializable { 9 | 10 | private static final long serialVersionUID = -337388419815576498L; 11 | 12 | private byte[] values; 13 | 14 | public ByteArray(byte[] values) { 15 | super(); 16 | this.values = values; 17 | } 18 | 19 | public byte[] get() { 20 | return values; 21 | } 22 | 23 | @Override 24 | public String toString() { 25 | return values != null ? new String(values) : "null"; 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/exdemon/components/source/types/TestComponentsSource.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.components.source.types; 2 | 3 | import ch.cern.exdemon.components.RegisterComponentType; 4 | import ch.cern.exdemon.components.source.ComponentsSource; 5 | import ch.cern.properties.Properties; 6 | 7 | @RegisterComponentType("test") 8 | public class TestComponentsSource extends ComponentsSource { 9 | 10 | private static final long serialVersionUID = -5721933057258619601L; 11 | 12 | public void add(Type componentType, String id, Properties properties) { 13 | register(componentType, id, properties); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/num/AddFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.num; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | 7 | public class AddFunc extends BiNumericFunction { 8 | 9 | public static String REPRESENTATION = "+"; 10 | 11 | public AddFunc(ValueComputable... arguments) throws ParseException { 12 | super(REPRESENTATION, arguments); 13 | 14 | operationInTheMiddleForToString(); 15 | } 16 | 17 | @Override 18 | public float compute(float value1, float value2) { 19 | return value1 + value2; 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/num/DivFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.num; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | 7 | public class DivFunc extends BiNumericFunction{ 8 | 9 | public static String REPRESENTATION = "/"; 10 | 11 | public DivFunc(ValueComputable... arguments) throws ParseException { 12 | super(REPRESENTATION, arguments); 13 | 14 | operationInTheMiddleForToString(); 15 | } 16 | 17 | @Override 18 | public float compute(float value1, float value2) { 19 | return value1 / value2; 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/num/MulFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.num; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | 7 | public class MulFunc extends BiNumericFunction{ 8 | 9 | public static String REPRESENTATION = "*"; 10 | 11 | public MulFunc(ValueComputable... arguments) throws ParseException { 12 | super(REPRESENTATION, arguments); 13 | 14 | operationInTheMiddleForToString(); 15 | } 16 | 17 | @Override 18 | public float compute(float value1, float value2) { 19 | return value1 * value2; 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/num/SubFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.num; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | 7 | public class SubFunc extends BiNumericFunction{ 8 | 9 | public static String REPRESENTATION = "-"; 10 | 11 | public SubFunc(ValueComputable... arguments) throws ParseException { 12 | super(REPRESENTATION, arguments); 13 | 14 | operationInTheMiddleForToString(); 15 | } 16 | 17 | @Override 18 | public float compute(float value1, float value2) { 19 | return value1 - value2; 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /doc/users-manual/metrics.md: -------------------------------------------------------------------------------- 1 | # Metrics 2 | 3 | Metric schemas produces metrics. A metric in ExDeMon has the following characteristics: 4 | 5 | * Contains a set of key-value pairs which are considered attributes. Key and value are always strings. These attributes should serve to differentiate between metrics. These attributes are set by the metric source and can change when defining new metrics. 6 | * It is marked with a time stamp. Time stamp is extracted by metric schemas. 7 | * Contains a single value. Value could be of type float, string or boolean. Type is determined by the metric schema. When defining new metrics, aggregation operations or equation defined in value determine the type of the new metric. -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/num/PowFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.num; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | 7 | public class PowFunc extends BiNumericFunction{ 8 | 9 | public static String REPRESENTATION = "^"; 10 | 11 | public PowFunc(ValueComputable... arguments) throws ParseException { 12 | super(REPRESENTATION, arguments); 13 | 14 | operationInTheMiddleForToString(); 15 | } 16 | 17 | @Override 18 | public float compute(float value1, float value2) { 19 | return (float) Math.pow(value1, value2); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/utils/Pair.java: -------------------------------------------------------------------------------- 1 | package ch.cern.utils; 2 | 3 | import java.io.Serializable; 4 | 5 | import lombok.EqualsAndHashCode; 6 | import lombok.ToString; 7 | 8 | @ToString 9 | @EqualsAndHashCode(callSuper=false) 10 | public class Pair implements Serializable{ 11 | 12 | private static final long serialVersionUID = 4231657422199019843L; 13 | 14 | public K first; 15 | 16 | public V second; 17 | 18 | public Pair(K first, V second){ 19 | this.first = first; 20 | this.second = second; 21 | } 22 | 23 | public K first() { 24 | return first; 25 | } 26 | 27 | public V second() { 28 | return second; 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/var/VariableStatus.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var; 2 | 3 | import java.io.Serializable; 4 | import java.time.Instant; 5 | 6 | import lombok.EqualsAndHashCode; 7 | import lombok.Getter; 8 | import lombok.Setter; 9 | import lombok.ToString; 10 | 11 | @ToString 12 | @EqualsAndHashCode 13 | public abstract class VariableStatus implements Serializable { 14 | 15 | private static final long serialVersionUID = -1238303955426246795L; 16 | 17 | @Getter 18 | @Setter 19 | private Instant lastUpdateMetricTime; 20 | 21 | public VariableStatus() { 22 | lastUpdateMetricTime = Instant.EPOCH; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/exdemon/metrics/JSONMetric.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics; 2 | 3 | import java.time.Instant; 4 | 5 | @SuppressWarnings("unused") 6 | public class JSONMetric { 7 | 8 | private String CLUSTER, HOSTNAME, METRIC, KEY_TO_REMOVE; 9 | 10 | private Instant TIMESTAMP; 11 | 12 | private float VALUE; 13 | 14 | public JSONMetric(Metric metric) { 15 | CLUSTER = metric.getAttributes().get("CLUSTER"); 16 | HOSTNAME = metric.getAttributes().get("HOSTNAME"); 17 | METRIC = metric.getAttributes().get("METRIC"); 18 | KEY_TO_REMOVE = metric.getAttributes().get("KEY_TO_REMOVE"); 19 | 20 | TIMESTAMP = metric.getTimestamp(); 21 | 22 | VALUE = metric.getValue().getAsFloat().get(); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/FunctionCaller.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions; 2 | 3 | import java.text.ParseException; 4 | import java.util.Map; 5 | 6 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 7 | import ch.cern.exdemon.metrics.value.Value; 8 | 9 | public interface FunctionCaller { 10 | 11 | public String getFunctionRepresentation(); 12 | 13 | public Class[] getArgumentTypes(); 14 | 15 | public ValueComputable call(ValueComputable... arguments) throws ParseException; 16 | 17 | public default void register(Map functions) { 18 | functions.put(getFunctionRepresentation(), this); 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /bin/kafka-topic-mng: -------------------------------------------------------------------------------- 1 | [root@monit-kafkax-dev-e0d4b64eae ~]# 2 | 3 | export KAFKA_OPTS=-Djava.security.auth.login.config=/opt/kafka/config/server_jaas.conf 4 | 5 | /opt/kafka/bin/kafka-topics.sh \ 6 | --zookeeper monit-zookeeper-dev-ab5bcc5450:2181,monit-zookeeper-dev-af7ea987bd:2181,monit-zookeeper-dev-c50f381e22:2181/kafkax/monit \ 7 | --delete \ 8 | --topic metrics_monitor_checkpoint_qa 9 | 10 | /opt/kafka/bin/kafka-topics.sh \ 11 | --zookeeper monit-zookeeper-dev-ab5bcc5450:2181,monit-zookeeper-dev-af7ea987bd:2181,monit-zookeeper-dev-c50f381e22:2181/kafkax/monit \ 12 | --config cleanup.policy=compact \ 13 | --config segment.ms=240000 \ 14 | --topic metrics_monitor_checkpoint_qa \ 15 | --create \ 16 | --partitions 10 \ 17 | --replication-factor 3 -------------------------------------------------------------------------------- /src/test/java/ch/cern/exdemon/monitor/analysis/types/AlwaysTrueAnalysisTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.analysis.types; 2 | 3 | import static ch.cern.exdemon.metrics.MetricTest.Metric; 4 | import static org.junit.Assert.assertEquals; 5 | 6 | import org.junit.Test; 7 | 8 | import ch.cern.exdemon.metrics.Metric; 9 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult.Status;; 10 | 11 | public class AlwaysTrueAnalysisTest { 12 | 13 | @Test 14 | public void analysis() { 15 | 16 | AlwaysTrueAnalysis analysis = new AlwaysTrueAnalysis(); 17 | 18 | Metric metric = Metric(0, true); 19 | assertEquals(Status.OK, analysis.apply(metric).getStatus()); 20 | 21 | metric = Metric(0, false); 22 | assertEquals(Status.ERROR, analysis.apply(metric).getStatus()); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/SparkConf.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark; 2 | 3 | import ch.cern.properties.Properties; 4 | 5 | public class SparkConf extends org.apache.spark.SparkConf { 6 | 7 | private static final long serialVersionUID = 2168292945784881986L; 8 | 9 | public SparkConf(){ 10 | super(); 11 | } 12 | 13 | public void runLocallyIfMasterIsNotConfigured() { 14 | if(!contains("spark.master")) { 15 | setMaster("local[1]"); 16 | set("spark.driver.host", "localhost"); 17 | } 18 | } 19 | 20 | public void addProperties(Properties properties, String prefix) { 21 | properties.stringPropertyNames().stream() 22 | .filter(key -> key.startsWith(prefix)) 23 | .forEach(key -> set(key, properties.getProperty(key))); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/utils/IntEvictingQueue.java: -------------------------------------------------------------------------------- 1 | package ch.cern.utils; 2 | 3 | public class IntEvictingQueue { 4 | 5 | private int[] values; 6 | 7 | private int head; 8 | 9 | private int size; 10 | 11 | public IntEvictingQueue(int limit) { 12 | values = new int[limit]; 13 | head = 0; 14 | size = 0; 15 | } 16 | 17 | public void add(int newValue) { 18 | head++; 19 | if(head == values.length) 20 | head = 0; 21 | 22 | values[head] = newValue; 23 | 24 | if(size < values.length) 25 | size++; 26 | } 27 | 28 | public int size() { 29 | return size; 30 | } 31 | 32 | public int get(int i) { 33 | if(i >= size || i < 0) 34 | throw new IndexOutOfBoundsException(i + " is >= size (" + size + ") or < 0"); 35 | 36 | return values[(size - head + 1 + i) % values.length]; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/status/RemoveAndValue.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status; 2 | 3 | import java.io.Serializable; 4 | 5 | import org.apache.spark.api.java.Optional; 6 | 7 | public class RemoveAndValue implements Serializable{ 8 | 9 | private static final long serialVersionUID = 625633246879682831L; 10 | 11 | private K key; 12 | 13 | private V value; 14 | 15 | public RemoveAndValue(K key, Optional value) { 16 | this.key = key; 17 | this.value = value.isPresent() ? value.get() : null; 18 | } 19 | 20 | public boolean isRemoveAction() { 21 | return key != null; 22 | } 23 | 24 | public K getKey() { 25 | return key; 26 | } 27 | 28 | public Optional getValue() { 29 | return Optional.fromNullable(value); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/exdemon/components/ConfigurationResultTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.components; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | 5 | import org.junit.Test; 6 | 7 | public class ConfigurationResultTest { 8 | 9 | @Test 10 | public void merge() { 11 | ConfigurationResult configResult1 = ConfigurationResult.SUCCESSFUL(); 12 | 13 | ConfigurationResult configResult2 = ConfigurationResult.SUCCESSFUL(); 14 | configResult2.withError("e", "message_e"); 15 | configResult2.withWarning("w", "message_w"); 16 | 17 | configResult1.merge("a", configResult2); 18 | 19 | assertEquals("message_e", configResult1.getErrors().get(0).getMessage()); 20 | assertEquals("message_w", configResult1.getWarnings().get(0).getMessage()); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/var/VariableCreationResult.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var; 2 | 3 | import java.util.Optional; 4 | 5 | import ch.cern.exdemon.components.ConfigurationResult; 6 | import lombok.Getter; 7 | 8 | public class VariableCreationResult { 9 | 10 | private Variable variable = null; 11 | 12 | @Getter 13 | private ConfigurationResult configResult; 14 | 15 | public VariableCreationResult(Variable variable, ConfigurationResult configResult) { 16 | super(); 17 | 18 | this.configResult = configResult; 19 | 20 | if(configResult.getErrors().isEmpty()) 21 | this.variable = variable; 22 | } 23 | 24 | public Optional getVariable(){ 25 | return Optional.ofNullable(variable); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/trigger/TriggerStatus.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.trigger; 2 | 3 | import java.time.Instant; 4 | 5 | import ch.cern.spark.status.StatusValue; 6 | import ch.cern.spark.status.storage.ClassNameAlias; 7 | import lombok.EqualsAndHashCode; 8 | import lombok.Getter; 9 | import lombok.Setter; 10 | 11 | @ClassNameAlias("trigger-status") 12 | @EqualsAndHashCode(callSuper=false) 13 | public class TriggerStatus extends StatusValue { 14 | 15 | private static final long serialVersionUID = -753687009841163354L; 16 | 17 | @Getter @Setter 18 | private Instant lastRaised; 19 | 20 | @Getter @Setter 21 | private StatusValue activeStatus; 22 | 23 | @Getter @Setter 24 | private StatusValue sailentStatus; 25 | 26 | public TriggerStatus() { 27 | lastRaised = Instant.EPOCH; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /bin/restart: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2018, CERN 4 | # This software is distributed under the terms of the GNU General Public 5 | # Licence version 3 (GPL Version 3), copied verbatim in the file "LICENSE". 6 | # In applying this license, CERN does not waive the privileges and immunities 7 | # granted to it by virtue of its status as Intergovernmental Organization 8 | # or submit itself to any jurisdiction. 9 | 10 | # Reference: http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in 11 | SOURCE="${BASH_SOURCE[0]}" 12 | while [ -h "$SOURCE" ]; do 13 | DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" 14 | SOURCE="$(readlink "$SOURCE")" 15 | [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" 16 | done 17 | SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" 18 | 19 | HOME=$SCRIPT_DIR/.. 20 | 21 | $HOME/bin/stop 22 | 23 | $HOME/bin/start 24 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/DatedValue.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics; 2 | 3 | import java.io.Serializable; 4 | import java.time.Instant; 5 | 6 | import ch.cern.exdemon.metrics.value.Value; 7 | import lombok.EqualsAndHashCode; 8 | import lombok.Getter; 9 | import lombok.ToString; 10 | 11 | @EqualsAndHashCode(callSuper=false) 12 | @ToString 13 | public class DatedValue implements Serializable, Comparable { 14 | 15 | private static final long serialVersionUID = 3930338572646527289L; 16 | 17 | @Getter 18 | private Instant time; 19 | 20 | @Getter 21 | private Value value; 22 | 23 | public DatedValue(Instant time, Value value) { 24 | this.time = time; 25 | this.value = value; 26 | } 27 | 28 | @Override 29 | public int compareTo(DatedValue other) { 30 | return time.compareTo(other.time); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/trigger/action/actuator/Actuator.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.trigger.action.actuator; 2 | 3 | import ch.cern.exdemon.components.Component; 4 | import ch.cern.exdemon.components.ComponentType; 5 | import ch.cern.exdemon.components.Component.Type; 6 | import ch.cern.exdemon.monitor.trigger.action.Action; 7 | import lombok.Getter; 8 | import lombok.Setter; 9 | 10 | @ComponentType(Type.ACTUATOR) 11 | public abstract class Actuator extends Component { 12 | 13 | private static final long serialVersionUID = 8984201586179047078L; 14 | 15 | @Getter @Setter 16 | private String id; 17 | 18 | protected boolean shouldBeProcess(Action action) { 19 | return action.getActuatorIDs().contains(id) || action.getActuatorIDs().contains("ALL"); 20 | } 21 | 22 | protected abstract void run(Action action) throws Exception; 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/analysis/types/AlwaysTrueAnalysis.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.analysis.types; 2 | 3 | import java.time.Instant; 4 | 5 | import ch.cern.exdemon.components.RegisterComponentType; 6 | import ch.cern.exdemon.monitor.analysis.BooleanAnalysis; 7 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult; 8 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult.Status; 9 | 10 | @RegisterComponentType("true") 11 | public class AlwaysTrueAnalysis extends BooleanAnalysis { 12 | 13 | private static final long serialVersionUID = -4512609434171905694L; 14 | 15 | @Override 16 | public AnalysisResult process(Instant timestamp, boolean value) { 17 | if(value) 18 | return AnalysisResult.buildWithStatus(Status.OK, "Metric is true."); 19 | else 20 | return AnalysisResult.buildWithStatus(Status.ERROR, "Metric is false."); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/status/StateDStream.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status; 2 | 3 | import org.apache.spark.streaming.api.java.JavaDStream; 4 | import org.apache.spark.streaming.api.java.JavaMapWithStateDStream; 5 | import org.apache.spark.streaming.api.java.JavaPairDStream; 6 | 7 | public class StateDStream { 8 | 9 | private JavaMapWithStateDStream, S, RemoveAndValue> stream; 10 | 11 | public StateDStream(JavaMapWithStateDStream, S, RemoveAndValue> statusStream) { 12 | this.stream = statusStream; 13 | } 14 | 15 | public JavaDStream values() { 16 | return stream.filter(av -> av.getValue().isPresent()).map(av -> av.getValue().get()); 17 | } 18 | 19 | public JavaPairDStream statuses() { 20 | return stream.stateSnapshots(); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /doc/users-manual/metric-sources.md: -------------------------------------------------------------------------------- 1 | # Metric sources 2 | 3 | Metric sources produces JSON documents. These JSON documents are later interpreted by metric schemas to be parsed to metrics. 4 | 5 | Optionally, generated stream can be repartitioned with "partitions" parameter. By default, no repartitioning is done. 6 | 7 | ``` 8 | metrics.source..partitions = 9 | ``` 10 | 11 | ## Kafka metric source 12 | 13 | It expects documents as JSON. 14 | 15 | Configuration: 16 | 17 | ``` 18 | metrics.source..type = kafka 19 | metrics.source..topics = 20 | metrics.source..consumer.bootstrap.servers = 21 | metrics.source..consumer.group.id = 22 | # All these parameters (source.consumer.) will by passed to the consumer 23 | metrics.source..consumer. = 24 | ``` -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/MonitorStatusKey.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor; 2 | 3 | import java.util.Map; 4 | 5 | import ch.cern.spark.status.IDStatusKey; 6 | import ch.cern.spark.status.storage.ClassNameAlias; 7 | import lombok.EqualsAndHashCode; 8 | import lombok.Getter; 9 | import lombok.ToString; 10 | 11 | @ClassNameAlias("monitor-key") 12 | @ToString 13 | @EqualsAndHashCode(callSuper=false) 14 | public class MonitorStatusKey implements IDStatusKey{ 15 | 16 | private static final long serialVersionUID = -7288054142035095969L; 17 | 18 | private String id; 19 | 20 | @Getter 21 | private Map metric_attributes; 22 | 23 | public MonitorStatusKey(String monitorID, Map metricIDs){ 24 | this.id = monitorID; 25 | this.metric_attributes = metricIDs; 26 | } 27 | 28 | public String getID() { 29 | return id; 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/DefinedMetricStatuskey.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined; 2 | 3 | import java.util.Map; 4 | 5 | import ch.cern.spark.status.IDStatusKey; 6 | import ch.cern.spark.status.storage.ClassNameAlias; 7 | import lombok.EqualsAndHashCode; 8 | import lombok.Getter; 9 | import lombok.ToString; 10 | 11 | @ClassNameAlias("defined-metric-key") 12 | @ToString 13 | @EqualsAndHashCode(callSuper=false) 14 | public class DefinedMetricStatuskey implements IDStatusKey { 15 | 16 | private static final long serialVersionUID = 2992810464329306975L; 17 | 18 | private String id; 19 | 20 | @Getter 21 | private Map metric_attributes; 22 | 23 | public DefinedMetricStatuskey(String definedMetricId, Map metric_attributes) { 24 | this.id = definedMetricId; 25 | this.metric_attributes = metric_attributes; 26 | } 27 | 28 | public String getID() { 29 | return id; 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Set everything to be logged to the console 2 | log4j.rootCategory=OFF, console 3 | 4 | log4j.appender.console=org.apache.log4j.ConsoleAppender 5 | log4j.appender.console.target=System.err 6 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 8 | 9 | log4j.appender.file=org.apache.log4j.RollingFileAppender 10 | log4j.appender.file.File=log/spark-job.log 11 | log4j.appender.file.MaxFileSize=10MB 12 | log4j.appender.file.MaxBackupIndex=10 13 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 14 | log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 15 | 16 | # Settings to quiet third party logs that are too verbose 17 | log4j.logger.org.eclipse.jetty=WARN 18 | log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO 19 | log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO 20 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/properties/ConfigurationException.java: -------------------------------------------------------------------------------- 1 | package ch.cern.properties; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | 6 | public class ConfigurationException extends Exception { 7 | 8 | private static final long serialVersionUID = -2597184362736139519L; 9 | 10 | @Getter 11 | @Setter 12 | private String parameter; 13 | 14 | public ConfigurationException(Exception e) { 15 | super(e); 16 | } 17 | 18 | public ConfigurationException(String parameter, Exception e) { 19 | super(e); 20 | 21 | this.parameter = parameter; 22 | } 23 | 24 | public ConfigurationException(String parameter, String msg) { 25 | super(msg); 26 | 27 | this.parameter = parameter; 28 | } 29 | 30 | @Override 31 | public String toString() { 32 | if(parameter != null) 33 | return parameter + ": " + getMessage(); 34 | else 35 | return getMessage(); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM cern/cc7-base 2 | 3 | # Dockerfile metadata 4 | MAINTAINER daniel.lanza@cern.ch 5 | LABEL description="Testbed for ExDeMon project" 6 | 7 | # Install Apache Maven (for packing projects) 8 | RUN curl https://repos.fedorapeople.org/repos/dchen/apache-maven/epel-apache-maven.repo > /etc/yum.repos.d/epel-apache-maven.repo 9 | RUN sed -i s/\$releasever/7/g /etc/yum.repos.d/epel-apache-maven.repo 10 | RUN yum install -y which java apache-maven 11 | 12 | # Run Maven to download dependencies 13 | ENV SHARED_DIR="/tmp/repository/" 14 | COPY ./pom.xml $SHARED_DIR/ 15 | RUN (cd $SHARED_DIR/; mvn versions:set -DnewVersion=1-DOCKER_GENERATION) 16 | RUN (cd $SHARED_DIR/; mvn package clean) 17 | 18 | # Install make and rpmbuild (for building RPMs) 19 | RUN yum install -y make rpm-build 20 | RUN mkdir -p ~/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS} 21 | 22 | # Install Koji (for releasing RPM) 23 | RUN yum --disablerepo=extras -y install koji 24 | COPY ./koji.conf /root/.koji/config 25 | 26 | CMD /usr/bin/sleep infinity 27 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/analysis/NumericAnalysis.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.analysis; 2 | 3 | import java.time.Instant; 4 | 5 | import ch.cern.exdemon.metrics.Metric; 6 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult; 7 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult.Status; 8 | 9 | public abstract class NumericAnalysis extends Analysis { 10 | 11 | private static final long serialVersionUID = -1822474093334300773L; 12 | 13 | @Override 14 | public AnalysisResult process(Metric metric) { 15 | if(!metric.getValue().getAsFloat().isPresent()) { 16 | AnalysisResult result = AnalysisResult.buildWithStatus(Status.EXCEPTION, "Current analysis requires metrics of float type."); 17 | result.setAnalyzedMetric(metric); 18 | 19 | return result; 20 | } 21 | 22 | return process(metric.getTimestamp(), metric.getValue().getAsFloat().get()); 23 | } 24 | 25 | public abstract AnalysisResult process(Instant timestamp, double value); 26 | 27 | } 28 | 29 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/analysis/StringAnalysis.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.analysis; 2 | 3 | import java.time.Instant; 4 | 5 | import ch.cern.exdemon.metrics.Metric; 6 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult; 7 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult.Status; 8 | 9 | public abstract class StringAnalysis extends Analysis { 10 | 11 | private static final long serialVersionUID = -1822474093334300773L; 12 | 13 | @Override 14 | public AnalysisResult process(Metric metric) { 15 | if(!metric.getValue().getAsString().isPresent()) { 16 | AnalysisResult result = AnalysisResult.buildWithStatus(Status.EXCEPTION, "Current analysis requires metrics of string type."); 17 | result.setAnalyzedMetric(metric); 18 | 19 | return result; 20 | } 21 | 22 | return process(metric.getTimestamp(), metric.getValue().getAsString().get()); 23 | } 24 | 25 | public abstract AnalysisResult process(Instant timestamp, String value); 26 | 27 | } 28 | 29 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/analysis/BooleanAnalysis.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.analysis; 2 | 3 | import java.time.Instant; 4 | 5 | import ch.cern.exdemon.metrics.Metric; 6 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult; 7 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult.Status; 8 | 9 | public abstract class BooleanAnalysis extends Analysis { 10 | 11 | private static final long serialVersionUID = -1822474093334300773L; 12 | 13 | @Override 14 | public AnalysisResult process(Metric metric) { 15 | if(!metric.getValue().getAsBoolean().isPresent()) { 16 | AnalysisResult result = AnalysisResult.buildWithStatus(Status.EXCEPTION, "Current analysis requires metrics of boolean type."); 17 | result.setAnalyzedMetric(metric); 18 | 19 | return result; 20 | } 21 | 22 | return process(metric.getTimestamp(), metric.getValue().getAsBoolean().get()); 23 | } 24 | 25 | public abstract AnalysisResult process(Instant timestamp, boolean value); 26 | 27 | } 28 | 29 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/analysis/types/NoneAnalysis.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.analysis.types; 2 | 3 | import ch.cern.exdemon.components.ConfigurationResult; 4 | import ch.cern.exdemon.components.RegisterComponentType; 5 | import ch.cern.exdemon.metrics.Metric; 6 | import ch.cern.exdemon.monitor.analysis.Analysis; 7 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult; 8 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult.Status; 9 | import ch.cern.properties.Properties; 10 | 11 | @RegisterComponentType("none") 12 | public class NoneAnalysis extends Analysis { 13 | 14 | private static final long serialVersionUID = 335288998662554717L; 15 | 16 | @Override 17 | protected ConfigurationResult config(Properties properties) { 18 | return super.config(properties).merge(null, properties.warningsIfNotAllPropertiesUsed()); 19 | } 20 | 21 | @Override 22 | protected AnalysisResult process(Metric metric) { 23 | return AnalysisResult.buildWithStatus(Status.OK, "No analysis"); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/exdemon/monitor/trigger/action/ActionTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.trigger.action; 2 | 3 | import java.time.Instant; 4 | import java.util.HashMap; 5 | import java.util.HashSet; 6 | 7 | import ch.cern.exdemon.metrics.Metric; 8 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult; 9 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult.Status; 10 | 11 | public class ActionTest { 12 | 13 | public static Action DUMMY = null; 14 | static { 15 | DUMMY = new Action( 16 | "dummyMonitorID", 17 | "dummyTriggerID", 18 | new HashMap<>(), 19 | "dummy reason", 20 | new HashSet<>(), 21 | new HashMap<>(), 22 | new AnalysisResult()); 23 | DUMMY.getTriggeringResult().setAnalyzedMetric(new Metric(Instant.now(), 1234f, new HashMap<>())); 24 | DUMMY.getTriggeringResult().setStatus(Status.OK, "dummy"); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/spark/status/TestStatus.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status; 2 | 3 | import ch.cern.spark.status.storage.ClassNameAlias; 4 | 5 | @ClassNameAlias("test-status") 6 | public class TestStatus extends StatusValue { 7 | 8 | private static final long serialVersionUID = 2051099937848011407L; 9 | 10 | private long number = 10; 11 | 12 | public TestStatus(int i) { 13 | number = i; 14 | } 15 | 16 | @Override 17 | public int hashCode() { 18 | final int prime = 31; 19 | int result = 1; 20 | result = prime * result + (int) (number ^ (number >>> 32)); 21 | return result; 22 | } 23 | 24 | @Override 25 | public boolean equals(Object obj) { 26 | if (this == obj) 27 | return true; 28 | if (obj == null) 29 | return false; 30 | if (getClass() != obj.getClass()) 31 | return false; 32 | TestStatus other = (TestStatus) obj; 33 | if (number != other.number) 34 | return false; 35 | return true; 36 | } 37 | 38 | @Override 39 | public String toString() { 40 | return "TestStatus [number=" + number + "]"; 41 | } 42 | 43 | } -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/bool/NotFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.bool; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.Function; 7 | import ch.cern.exdemon.metrics.value.BooleanValue; 8 | import ch.cern.exdemon.metrics.value.Value; 9 | 10 | public class NotFunc extends Function{ 11 | 12 | public static String REPRESENTATION = "!"; 13 | 14 | public static Class[] argumentTypes = types(BooleanValue.class); 15 | 16 | public NotFunc(ValueComputable... arguments) throws ParseException { 17 | super(REPRESENTATION, argumentTypes, arguments); 18 | 19 | operationInTheMiddleForToString(); 20 | } 21 | 22 | @Override 23 | public Class returnType() { 24 | return BooleanValue.class; 25 | } 26 | 27 | @Override 28 | protected Value compute(Value... values) { 29 | return new BooleanValue(!values[0].getAsBoolean().get()); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/bool/EqualFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.bool; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.Function; 7 | import ch.cern.exdemon.metrics.value.BooleanValue; 8 | import ch.cern.exdemon.metrics.value.Value; 9 | 10 | public class EqualFunc extends Function { 11 | 12 | public static String REPRESENTATION = "=="; 13 | 14 | public static Class[] argumentTypes = types(Value.class, Value.class); 15 | 16 | public EqualFunc(ValueComputable... arguments) throws ParseException { 17 | super(REPRESENTATION, argumentTypes, arguments); 18 | 19 | operationInTheMiddleForToString(); 20 | } 21 | 22 | @Override 23 | public Class returnType() { 24 | return BooleanValue.class; 25 | } 26 | 27 | @Override 28 | protected Value compute(Value... values) { 29 | return new BooleanValue(values[0].equals(values[1])); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/num/NumericFunction.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.num; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.Function; 7 | import ch.cern.exdemon.metrics.value.FloatValue; 8 | import ch.cern.exdemon.metrics.value.Value; 9 | 10 | public abstract class NumericFunction extends Function { 11 | 12 | public static Class[] argumentTypes = types(FloatValue.class); 13 | 14 | public NumericFunction(String representation, ValueComputable... argument) 15 | throws ParseException { 16 | super(representation, argumentTypes, argument); 17 | } 18 | 19 | @Override 20 | protected Value compute(Value... values) { 21 | return new FloatValue(compute(values[0].getAsFloat().get())); 22 | } 23 | 24 | @Override 25 | public Class returnType() { 26 | return FloatValue.class; 27 | } 28 | 29 | public abstract float compute(float value); 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/status/storage/manager/ExpireStatusValueFilter.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status.storage.manager; 2 | 3 | import java.time.Duration; 4 | import java.time.Instant; 5 | 6 | import org.apache.spark.api.java.function.Function; 7 | 8 | import ch.cern.spark.status.StatusKey; 9 | import ch.cern.spark.status.StatusValue; 10 | import lombok.EqualsAndHashCode; 11 | import lombok.ToString; 12 | import scala.Tuple2; 13 | 14 | @ToString 15 | @EqualsAndHashCode(callSuper=false) 16 | public class ExpireStatusValueFilter implements Function, Boolean> { 17 | 18 | private static final long serialVersionUID = 8935468162326857198L; 19 | 20 | private Instant oldest; 21 | 22 | public ExpireStatusValueFilter(Duration expired_period) { 23 | this.oldest = Instant.now().minus(expired_period); 24 | } 25 | 26 | @Override 27 | public Boolean call(Tuple2 tuple) throws Exception { 28 | return tuple._2 == null || tuple._2.getStatus_update_time() < oldest.toEpochMilli(); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/bool/NotEqualFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.bool; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.Function; 7 | import ch.cern.exdemon.metrics.value.BooleanValue; 8 | import ch.cern.exdemon.metrics.value.Value; 9 | 10 | public class NotEqualFunc extends Function { 11 | 12 | public static String REPRESENTATION = "!="; 13 | 14 | public static Class[] argumentTypes = types(Value.class, Value.class); 15 | 16 | public NotEqualFunc(ValueComputable... arguments) throws ParseException { 17 | super(REPRESENTATION, argumentTypes, arguments); 18 | 19 | operationInTheMiddleForToString(); 20 | } 21 | 22 | @Override 23 | public Class returnType() { 24 | return BooleanValue.class; 25 | } 26 | 27 | @Override 28 | protected Value compute(Value... values) { 29 | return new BooleanValue(!values[0].equals(values[1])); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/string/StringFunction.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.string; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.Function; 7 | import ch.cern.exdemon.metrics.value.StringValue; 8 | import ch.cern.exdemon.metrics.value.Value; 9 | 10 | public abstract class StringFunction extends Function{ 11 | 12 | public static Class[] argumentTypes = types(StringValue.class); 13 | 14 | public StringFunction(String representation, ValueComputable... arguments) 15 | throws ParseException { 16 | super(representation, argumentTypes, arguments); 17 | } 18 | 19 | @Override 20 | protected Value compute(Value... values) { 21 | return new StringValue(compute(values[0].getAsString().get())); 22 | } 23 | 24 | @Override 25 | public Class returnType() { 26 | return StringValue.class; 27 | } 28 | 29 | public abstract String compute(String value); 30 | 31 | } 32 | -------------------------------------------------------------------------------- /doc/users-manual/silences.md: -------------------------------------------------------------------------------- 1 | # Silences 2 | 3 | Actions generated by [triggers](monitor-triggers.md) can be ignored by using silences. 4 | 5 | Silences are components like monitors that can be added and removed dynamically. 6 | 7 | Configuration: 8 | 9 | ``` 10 | from = (e.g: 2018-02-16 11:00:02) 11 | to = (e.g: 2018-04-16 11:00:02) 12 | duration = (if configured, "from" parameter must be set and "to" parameter is ignored) 13 | filter.expr = (as for metrics filter) 14 | filter.attribute. = 15 | filter.attribute. = 16 | ``` 17 | 18 | Silences will be active during the specified period. 19 | If no period is configured, it will be always active till silence is removed. 20 | 21 | During active period, actions that match filter will be dropped. 22 | 23 | An example to silence all actions from a "http\_errors" monitor for the next to hour: 24 | 25 | ``` 26 | # "from" should configure current time 27 | from = 2018-08-01 19:40:02 28 | duration = 2h 29 | filter.attribute.$monitor = http_errors 30 | ``` 31 | 32 | 33 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/bool/AndFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.bool; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.Function; 7 | import ch.cern.exdemon.metrics.value.BooleanValue; 8 | import ch.cern.exdemon.metrics.value.Value; 9 | 10 | public class AndFunc extends Function { 11 | 12 | public static String REPRESENTATION = "&&"; 13 | 14 | public static Class[] argumentTypes = types(BooleanValue.class, BooleanValue.class); 15 | 16 | public AndFunc(ValueComputable... arguments) throws ParseException { 17 | super(REPRESENTATION, argumentTypes, arguments); 18 | 19 | operationInTheMiddleForToString(); 20 | } 21 | 22 | @Override 23 | public Class returnType() { 24 | return BooleanValue.class; 25 | } 26 | 27 | @Override 28 | protected Value compute(Value... values) { 29 | return new BooleanValue(values[0].getAsBoolean().get() && values[1].getAsBoolean().get()); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/bool/OrFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.bool; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.Function; 7 | import ch.cern.exdemon.metrics.value.BooleanValue; 8 | import ch.cern.exdemon.metrics.value.Value; 9 | 10 | public class OrFunc extends Function { 11 | 12 | public static String REPRESENTATION = "||"; 13 | 14 | public static Class[] argumentTypes = types(BooleanValue.class, BooleanValue.class); 15 | 16 | public OrFunc(ValueComputable... arguments) throws ParseException { 17 | super(REPRESENTATION, argumentTypes, arguments); 18 | 19 | operationInTheMiddleForToString(); 20 | } 21 | 22 | @Override 23 | protected Value compute(Value... values) { 24 | return new BooleanValue(values[0].getAsBoolean().get() || values[1].getAsBoolean().get()); 25 | } 26 | 27 | @Override 28 | public Class returnType() { 29 | return BooleanValue.class; 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/utils/IString.java: -------------------------------------------------------------------------------- 1 | package ch.cern.utils; 2 | 3 | public class IString { 4 | 5 | private String string; 6 | 7 | private int index; 8 | 9 | public IString(String input) { 10 | this.string = input; 11 | index = 0; 12 | } 13 | 14 | public boolean isLastChar() { 15 | return string.length() - 1 == index; 16 | } 17 | 18 | public char getChar() { 19 | return string.charAt(index); 20 | } 21 | 22 | public Character nextChar() { 23 | if(index + 1 < string.length()) { 24 | index++; 25 | 26 | return string.charAt(index); 27 | }else 28 | return null; 29 | } 30 | 31 | public boolean startsWith(String prefix) { 32 | return string.startsWith(prefix, index); 33 | } 34 | 35 | public IString moveIndex(int diff) { 36 | index += diff; 37 | 38 | return this; 39 | } 40 | 41 | public boolean isFinished() { 42 | return index + 1 >= string.length(); 43 | } 44 | 45 | public String getString() { 46 | return string; 47 | } 48 | 49 | public int getIndex() { 50 | return index; 51 | } 52 | 53 | @Override 54 | public String toString() { 55 | return string + " pos:" + index; 56 | } 57 | 58 | } -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/num/BiNumericFunction.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.num; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.Function; 7 | import ch.cern.exdemon.metrics.value.FloatValue; 8 | import ch.cern.exdemon.metrics.value.Value; 9 | 10 | public abstract class BiNumericFunction extends Function{ 11 | 12 | public static Class[] argumentTypes = types(FloatValue.class, FloatValue.class); 13 | 14 | public BiNumericFunction(String representation, ValueComputable... arguments) 15 | throws ParseException { 16 | super(representation, argumentTypes, arguments); 17 | } 18 | 19 | @Override 20 | protected Value compute(Value... values) { 21 | return new FloatValue(compute(values[0].getAsFloat().get(), values[1].getAsFloat().get())); 22 | } 23 | 24 | @Override 25 | public Class returnType() { 26 | return FloatValue.class; 27 | } 28 | 29 | public abstract float compute(float value1, float value2); 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/exdemon/metrics/defined/equation/var/TestVariableStatus.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var; 2 | 3 | import ch.cern.spark.status.storage.ClassNameAlias; 4 | 5 | @ClassNameAlias("test-variable-status") 6 | public class TestVariableStatus extends VariableStatus { 7 | 8 | private static final long serialVersionUID = 2051099937848011407L; 9 | 10 | private long number = 10; 11 | 12 | public TestVariableStatus(int i) { 13 | number = i; 14 | } 15 | 16 | @Override 17 | public int hashCode() { 18 | final int prime = 31; 19 | int result = 1; 20 | result = prime * result + (int) (number ^ (number >>> 32)); 21 | return result; 22 | } 23 | 24 | @Override 25 | public boolean equals(Object obj) { 26 | if (this == obj) 27 | return true; 28 | if (obj == null) 29 | return false; 30 | if (getClass() != obj.getClass()) 31 | return false; 32 | TestVariableStatus other = (TestVariableStatus) obj; 33 | if (number != other.number) 34 | return false; 35 | return true; 36 | } 37 | 38 | @Override 39 | public String toString() { 40 | return "TestStatus [number=" + number + "]"; 41 | } 42 | 43 | } -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/analysis/Analysis.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.analysis; 2 | 3 | import java.util.function.Function; 4 | 5 | import ch.cern.exdemon.components.Component; 6 | import ch.cern.exdemon.components.ComponentType; 7 | import ch.cern.exdemon.components.Component.Type; 8 | import ch.cern.exdemon.metrics.Metric; 9 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult; 10 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult.Status; 11 | 12 | @ComponentType(Type.ANAYLSIS) 13 | public abstract class Analysis extends Component implements Function { 14 | 15 | private static final long serialVersionUID = -1822474093334300773L; 16 | 17 | public final AnalysisResult apply(Metric metric) { 18 | if(metric.getValue().getAsException().isPresent()) { 19 | AnalysisResult result = AnalysisResult.buildWithStatus(Status.EXCEPTION, "Metric of type excpetion."); 20 | result.setAnalyzedMetric(metric); 21 | 22 | return result; 23 | } 24 | 25 | return process(metric); 26 | } 27 | 28 | protected abstract AnalysisResult process(Metric metric); 29 | 30 | } 31 | 32 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/string/BiStringFunction.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.string; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.Function; 7 | import ch.cern.exdemon.metrics.value.StringValue; 8 | import ch.cern.exdemon.metrics.value.Value; 9 | 10 | public abstract class BiStringFunction extends Function { 11 | 12 | public static Class[] argumentTypes = types(StringValue.class, StringValue.class); 13 | 14 | public BiStringFunction(String representation, ValueComputable... arguments) 15 | throws ParseException { 16 | super(representation, argumentTypes, arguments); 17 | } 18 | 19 | @Override 20 | protected Value compute(Value... values) { 21 | return new StringValue(compute(values[0].getAsString().get(), values[1].getAsString().get())); 22 | } 23 | 24 | @Override 25 | public Class returnType() { 26 | return StringValue.class; 27 | } 28 | 29 | public abstract String compute(String value1, String value2); 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/num/GTFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.num; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.Function; 7 | import ch.cern.exdemon.metrics.value.BooleanValue; 8 | import ch.cern.exdemon.metrics.value.FloatValue; 9 | import ch.cern.exdemon.metrics.value.Value; 10 | 11 | public class GTFunc extends Function { 12 | 13 | public static String REPRESENTATION = "<"; 14 | 15 | public static Class[] argumentTypes = types(FloatValue.class, FloatValue.class); 16 | 17 | public GTFunc(ValueComputable... arguments) 18 | throws ParseException { 19 | super(REPRESENTATION, argumentTypes, arguments); 20 | 21 | operationInTheMiddleForToString(); 22 | } 23 | 24 | @Override 25 | public Class returnType() { 26 | return BooleanValue.class; 27 | } 28 | 29 | @Override 30 | protected Value compute(Value... values) { 31 | return new BooleanValue(values[0].getAsFloat().get() < values[1].getAsFloat().get()); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/num/LTFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.num; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.Function; 7 | import ch.cern.exdemon.metrics.value.BooleanValue; 8 | import ch.cern.exdemon.metrics.value.FloatValue; 9 | import ch.cern.exdemon.metrics.value.Value; 10 | 11 | public class LTFunc extends Function { 12 | 13 | public static String REPRESENTATION = ">"; 14 | 15 | public static Class[] argumentTypes = types(FloatValue.class, FloatValue.class); 16 | 17 | public LTFunc(ValueComputable... arguments) 18 | throws ParseException { 19 | super(REPRESENTATION, argumentTypes, arguments); 20 | 21 | operationInTheMiddleForToString(); 22 | } 23 | 24 | @Override 25 | public Class returnType() { 26 | return BooleanValue.class; 27 | } 28 | 29 | @Override 30 | protected Value compute(Value... values) { 31 | return new BooleanValue(values[0].getAsFloat().get() > values[1].getAsFloat().get()); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/status/StatusImpl.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status; 2 | 3 | import org.apache.spark.streaming.State; 4 | import org.apache.spark.streaming.Time; 5 | 6 | public class StatusImpl extends State { 7 | 8 | private State state; 9 | 10 | private Time time; 11 | 12 | private boolean removed; 13 | 14 | public StatusImpl(State state, Time time) { 15 | this.state = state; 16 | this.time = time; 17 | 18 | this.removed = false; 19 | } 20 | 21 | @Override 22 | public boolean exists() { 23 | return state.exists(); 24 | } 25 | 26 | @Override 27 | public S get() { 28 | return state.get(); 29 | } 30 | 31 | @Override 32 | public void remove() { 33 | state.remove(); 34 | removed = true; 35 | } 36 | 37 | @Override 38 | public boolean isTimingOut() { 39 | return state.isTimingOut(); 40 | } 41 | 42 | @Override 43 | public void update(S newState) { 44 | newState.update(state, time); 45 | } 46 | 47 | public boolean isRemoved() { 48 | return removed; 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/num/AbsFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.num; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.FunctionCaller; 7 | import ch.cern.exdemon.metrics.value.Value; 8 | 9 | public class AbsFunc extends NumericFunction { 10 | 11 | public static String REPRESENTATION = "abs"; 12 | 13 | public AbsFunc(ValueComputable... v) throws ParseException { 14 | super(REPRESENTATION, v); 15 | } 16 | 17 | @Override 18 | public float compute(float value) { 19 | return (float) Math.abs(value); 20 | } 21 | 22 | public static class Caller implements FunctionCaller{ 23 | 24 | @Override 25 | public String getFunctionRepresentation() { 26 | return REPRESENTATION; 27 | } 28 | 29 | @Override 30 | public Class[] getArgumentTypes() { 31 | return argumentTypes; 32 | } 33 | 34 | @Override 35 | public ValueComputable call(ValueComputable... arguments) throws ParseException { 36 | return new AbsFunc(arguments); 37 | } 38 | 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/string/TrimFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.string; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.FunctionCaller; 7 | import ch.cern.exdemon.metrics.value.Value; 8 | 9 | public class TrimFunc extends StringFunction { 10 | 11 | public static String REPRESENTATION = "trim"; 12 | 13 | public TrimFunc(ValueComputable... v) throws ParseException { 14 | super(REPRESENTATION, v); 15 | } 16 | 17 | @Override 18 | public String compute(String value) { 19 | return value.trim(); 20 | } 21 | 22 | public static class Caller implements FunctionCaller{ 23 | 24 | @Override 25 | public String getFunctionRepresentation() { 26 | return REPRESENTATION; 27 | } 28 | 29 | @Override 30 | public Class[] getArgumentTypes() { 31 | return argumentTypes; 32 | } 33 | 34 | @Override 35 | public ValueComputable call(ValueComputable... arguments) throws ParseException { 36 | return new TrimFunc(arguments); 37 | } 38 | 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/schema/MetricSchemas.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.schema; 2 | 3 | import org.apache.log4j.Logger; 4 | import org.apache.spark.streaming.api.java.JavaDStream; 5 | 6 | import ch.cern.exdemon.metrics.Metric; 7 | import ch.cern.properties.Properties; 8 | 9 | public class MetricSchemas { 10 | 11 | private transient final static Logger LOG = Logger.getLogger(MetricSchemas.class.getName()); 12 | 13 | private static final int JSON_MAX_SIZE = 64000; 14 | 15 | public static JavaDStream generate(JavaDStream jsons, Properties propertiesSourceProps, String sourceId) { 16 | jsons = jsons.filter(string -> { 17 | if(string.length() > JSON_MAX_SIZE) { 18 | LOG.warn("Event dropped because exceeds max size ("+JSON_MAX_SIZE+"): " + string.substring(0, 10000) + "..."); 19 | 20 | return false; 21 | } 22 | 23 | return true; 24 | }); 25 | 26 | return jsons.flatMap(new MetricSchemasF(propertiesSourceProps, sourceId)); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/num/CosFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.num; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.FunctionCaller; 7 | import ch.cern.exdemon.metrics.value.Value; 8 | 9 | public class CosFunc extends NumericFunction { 10 | 11 | public static String REPRESENTATION = "cos"; 12 | 13 | public CosFunc(ValueComputable... v) throws ParseException { 14 | super(REPRESENTATION, v); 15 | } 16 | 17 | @Override 18 | public float compute(float value) { 19 | return (float) Math.cos(Math.toRadians(value)); 20 | } 21 | 22 | public static class Caller implements FunctionCaller{ 23 | 24 | @Override 25 | public String getFunctionRepresentation() { 26 | return REPRESENTATION; 27 | } 28 | 29 | @Override 30 | public Class[] getArgumentTypes() { 31 | return argumentTypes; 32 | } 33 | 34 | @Override 35 | public ValueComputable call(ValueComputable... arguments) throws ParseException { 36 | return new CosFunc(arguments); 37 | } 38 | 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/num/SinFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.num; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.FunctionCaller; 7 | import ch.cern.exdemon.metrics.value.Value; 8 | 9 | public class SinFunc extends NumericFunction { 10 | 11 | public static String REPRESENTATION = "sin"; 12 | 13 | public SinFunc(ValueComputable... v) throws ParseException { 14 | super(REPRESENTATION, v); 15 | } 16 | 17 | @Override 18 | public float compute(float value) { 19 | return (float) Math.sin(Math.toRadians(value)); 20 | } 21 | 22 | public static class Caller implements FunctionCaller{ 23 | 24 | @Override 25 | public String getFunctionRepresentation() { 26 | return REPRESENTATION; 27 | } 28 | 29 | @Override 30 | public Class[] getArgumentTypes() { 31 | return argumentTypes; 32 | } 33 | 34 | @Override 35 | public ValueComputable call(ValueComputable... arguments) throws ParseException { 36 | return new SinFunc(arguments); 37 | } 38 | 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/num/TanFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.num; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.FunctionCaller; 7 | import ch.cern.exdemon.metrics.value.Value; 8 | 9 | public class TanFunc extends NumericFunction { 10 | 11 | public static String REPRESENTATION = "tan"; 12 | 13 | public TanFunc(ValueComputable... v) throws ParseException { 14 | super(REPRESENTATION, v); 15 | } 16 | 17 | @Override 18 | public float compute(float value) { 19 | return (float) Math.tan(Math.toRadians(value)); 20 | } 21 | 22 | public static class Caller implements FunctionCaller{ 23 | 24 | @Override 25 | public String getFunctionRepresentation() { 26 | return REPRESENTATION; 27 | } 28 | 29 | @Override 30 | public Class[] getArgumentTypes() { 31 | return argumentTypes; 32 | } 33 | 34 | @Override 35 | public ValueComputable call(ValueComputable... arguments) throws ParseException { 36 | return new TanFunc(arguments); 37 | } 38 | 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/num/SqrtFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.num; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.FunctionCaller; 7 | import ch.cern.exdemon.metrics.value.Value; 8 | 9 | public class SqrtFunc extends NumericFunction { 10 | 11 | public static String REPRESENTATION = "sqrt"; 12 | 13 | public SqrtFunc(ValueComputable... arguments) throws ParseException { 14 | super(REPRESENTATION, arguments); 15 | } 16 | 17 | @Override 18 | public float compute(float value) { 19 | return (float) Math.sqrt(value); 20 | } 21 | 22 | public static class Caller implements FunctionCaller{ 23 | 24 | @Override 25 | public String getFunctionRepresentation() { 26 | return REPRESENTATION; 27 | } 28 | 29 | @Override 30 | public Class[] getArgumentTypes() { 31 | return argumentTypes; 32 | } 33 | 34 | @Override 35 | public ValueComputable call(ValueComputable... arguments) throws ParseException { 36 | return new SqrtFunc(arguments); 37 | } 38 | 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/string/ConcatFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.string; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.FunctionCaller; 7 | import ch.cern.exdemon.metrics.value.Value; 8 | 9 | public class ConcatFunc extends BiStringFunction { 10 | 11 | public static String REPRESENTATION = "concat"; 12 | 13 | public ConcatFunc(ValueComputable... arguments) throws ParseException { 14 | super(REPRESENTATION, arguments); 15 | } 16 | 17 | @Override 18 | public String compute(String value1, String value2) { 19 | return value1.concat(value2); 20 | } 21 | 22 | public static class Caller implements FunctionCaller{ 23 | 24 | @Override 25 | public String getFunctionRepresentation() { 26 | return REPRESENTATION; 27 | } 28 | 29 | @Override 30 | public Class[] getArgumentTypes() { 31 | return argumentTypes; 32 | } 33 | 34 | @Override 35 | public ValueComputable call(ValueComputable... arguments) throws ParseException { 36 | return new ConcatFunc(arguments); 37 | } 38 | 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/status/storage/types/ConsumerRecordSer.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status.storage.types; 2 | 3 | import java.io.Serializable; 4 | 5 | import org.apache.kafka.clients.consumer.ConsumerRecord; 6 | import org.apache.kafka.common.utils.Bytes; 7 | 8 | import ch.cern.utils.ByteArray; 9 | import lombok.ToString; 10 | 11 | @ToString 12 | public class ConsumerRecordSer implements Serializable{ 13 | 14 | private static final long serialVersionUID = -8815018248610333648L; 15 | 16 | private long offset; 17 | private ByteArray key; 18 | private ByteArray value; 19 | 20 | public ConsumerRecordSer(ConsumerRecord consumerRecord) { 21 | this.offset = consumerRecord.offset(); 22 | 23 | if(consumerRecord.key() != null) 24 | this.key = new ByteArray(consumerRecord.key().get()); 25 | 26 | if(consumerRecord.value() != null) 27 | this.value = new ByteArray(consumerRecord.value().get()); 28 | } 29 | 30 | public long offset() { 31 | return offset; 32 | } 33 | 34 | public ByteArray key() { 35 | return key; 36 | } 37 | 38 | public ByteArray value() { 39 | return value; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/status/storage/manager/IDStatusKeyFilter.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status.storage.manager; 2 | 3 | import org.apache.spark.api.java.function.Function; 4 | 5 | import ch.cern.spark.status.IDStatusKey; 6 | import ch.cern.spark.status.StatusKey; 7 | import ch.cern.spark.status.StatusValue; 8 | import lombok.EqualsAndHashCode; 9 | import lombok.ToString; 10 | import scala.Tuple2; 11 | 12 | @ToString 13 | @EqualsAndHashCode(callSuper=false) 14 | public class IDStatusKeyFilter implements Function, Boolean> { 15 | 16 | private static final long serialVersionUID = -8299090750744907416L; 17 | 18 | private String filter_by_id; 19 | 20 | public IDStatusKeyFilter(String filter_by_id) { 21 | this.filter_by_id = filter_by_id; 22 | } 23 | 24 | @Override 25 | public Boolean call(Tuple2 tuple) throws Exception { 26 | if(filter_by_id == null) 27 | return true; 28 | 29 | StatusKey key = tuple._1; 30 | 31 | if(!(key instanceof IDStatusKey)) 32 | return false; 33 | 34 | IDStatusKey idKey = (IDStatusKey) key; 35 | 36 | return idKey.getID().equals(filter_by_id); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/value/StringValue.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.value; 2 | 3 | import java.time.Instant; 4 | import java.util.Optional; 5 | 6 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 7 | import ch.cern.exdemon.metrics.defined.equation.var.VariableStatuses; 8 | import lombok.EqualsAndHashCode; 9 | import lombok.NonNull; 10 | 11 | @EqualsAndHashCode(callSuper=false) 12 | public class StringValue extends Value implements ValueComputable{ 13 | 14 | private static final long serialVersionUID = 6026199196915653369L; 15 | 16 | private String str; 17 | 18 | public StringValue(@NonNull String value){ 19 | this.str = value; 20 | } 21 | 22 | @Override 23 | public Optional getAsString() { 24 | return Optional.of(str); 25 | } 26 | 27 | @Override 28 | public StringValue compute(VariableStatuses store, Instant time) { 29 | return new StringValue(str); 30 | } 31 | 32 | @Override 33 | public Class returnType() { 34 | return StringValue.class; 35 | } 36 | 37 | @Override 38 | public String toString() { 39 | return "\"" + str + "\""; 40 | } 41 | 42 | @Override 43 | public String getSource() { 44 | if(source == null) 45 | return toString(); 46 | else 47 | return super.toString(); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/trigger/TriggerStatusKey.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.trigger; 2 | 3 | import java.util.Map; 4 | 5 | import ch.cern.spark.status.IDStatusKey; 6 | import ch.cern.spark.status.storage.ClassNameAlias; 7 | import lombok.EqualsAndHashCode; 8 | import lombok.Getter; 9 | import lombok.NonNull; 10 | import lombok.ToString; 11 | 12 | @ClassNameAlias("trigger-key") 13 | @ToString 14 | @EqualsAndHashCode(callSuper=false) 15 | public class TriggerStatusKey implements IDStatusKey{ 16 | 17 | private static final long serialVersionUID = -4289498306145284346L; 18 | 19 | @NonNull 20 | private String id; 21 | 22 | @Getter @NonNull 23 | private String monitor_id; 24 | 25 | @Getter @NonNull 26 | private Map metric_attributes; 27 | 28 | public TriggerStatusKey(@NonNull String monitorID, @NonNull String triggerID, @NonNull Map metric_attributes){ 29 | this.monitor_id = monitorID; 30 | this.id = triggerID; 31 | this.metric_attributes = metric_attributes; 32 | } 33 | 34 | public String getNotificatorID() { 35 | return id; 36 | } 37 | 38 | @Override 39 | public String getID() { 40 | return monitor_id + ":" + id; 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/value/AggregatedValue.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.value; 2 | 3 | import java.time.Instant; 4 | import java.util.Optional; 5 | 6 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 7 | import ch.cern.exdemon.metrics.defined.equation.var.VariableStatuses; 8 | import lombok.EqualsAndHashCode; 9 | import lombok.NonNull; 10 | 11 | @EqualsAndHashCode(callSuper=false) 12 | public class AggregatedValue extends Value implements ValueComputable{ 13 | 14 | private static final long serialVersionUID = 6026199196915653369L; 15 | 16 | private Value agg; 17 | 18 | public AggregatedValue(@NonNull Value value){ 19 | this.agg = value; 20 | } 21 | 22 | @Override 23 | public AggregatedValue compute(VariableStatuses store, Instant time) { 24 | return new AggregatedValue(agg); 25 | } 26 | 27 | @Override 28 | public Optional getAsAggregated() { 29 | return Optional.of(this.agg); 30 | } 31 | @Override 32 | public Class returnType() { 33 | return AggregatedValue.class; 34 | } 35 | 36 | @Override 37 | public String toString() { 38 | return agg.toString() + " (agg)"; 39 | } 40 | 41 | @Override 42 | public String getSource() { 43 | if(source == null) 44 | return toString(); 45 | else 46 | return super.toString(); 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/utils/ExceptionsCache.java: -------------------------------------------------------------------------------- 1 | package ch.cern.utils; 2 | 3 | import java.time.Duration; 4 | import java.time.Instant; 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | 8 | public class ExceptionsCache { 9 | 10 | private Map, Instant> raisedAts; 11 | 12 | private Duration period; 13 | 14 | public ExceptionsCache(Duration period) { 15 | this.raisedAts = new HashMap<>(); 16 | this.period = period; 17 | } 18 | 19 | public boolean wasRecentlyRaised(String id, Exception exception) { 20 | if(exception == null || exception.getMessage() == null) 21 | return false; 22 | 23 | Pair key = new Pair<>(id, exception.getMessage().hashCode()); 24 | 25 | Instant raisedAt = raisedAts.get(key); 26 | 27 | if(raisedAt == null) 28 | return false; 29 | else 30 | return raisedAt.plus(period).isAfter(Instant.now()); 31 | } 32 | 33 | public void raised(String id, Exception exception) { 34 | if(exception == null || exception.getMessage() == null) 35 | return; 36 | 37 | Pair key = new Pair<>(id, exception.getMessage().hashCode()); 38 | 39 | raisedAts.put(key, Instant.now()); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/var/agg/CountAgregation.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var.agg; 2 | 3 | import java.time.Instant; 4 | import java.util.Collection; 5 | 6 | import ch.cern.exdemon.components.RegisterComponentType; 7 | import ch.cern.exdemon.metrics.DatedValue; 8 | import ch.cern.exdemon.metrics.value.FloatValue; 9 | import ch.cern.exdemon.metrics.value.Value; 10 | 11 | @RegisterComponentType("count") 12 | public class CountAgregation extends Aggregation { 13 | 14 | private static final long serialVersionUID = 8274088208286521725L; 15 | 16 | @Override 17 | public Class inputType() { 18 | return Value.class; 19 | } 20 | 21 | @Override 22 | public Value aggregate(Collection values, Instant time) { 23 | return new FloatValue(values.stream() 24 | .mapToDouble(v -> v.getValue().getAsAggregated().isPresent() ? 25 | v.getValue().getAsAggregated().get().getAsFloat().get() : 26 | 1) 27 | .sum()); 28 | } 29 | 30 | @Override 31 | public Class returnType() { 32 | return FloatValue.class; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/filter/EqualMetricPredicate.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.filter; 2 | 3 | import java.io.Serializable; 4 | import java.text.ParseException; 5 | import java.util.Map; 6 | import java.util.function.Predicate; 7 | import java.util.regex.Pattern; 8 | import java.util.regex.PatternSyntaxException; 9 | 10 | import lombok.Getter; 11 | 12 | public class EqualMetricPredicate implements Predicate>, Serializable { 13 | 14 | private static final long serialVersionUID = 99926521342965096L; 15 | 16 | @Getter 17 | private String key; 18 | 19 | @Getter 20 | private Pattern value; 21 | 22 | public EqualMetricPredicate(String key, String value) throws ParseException { 23 | this.key = key; 24 | 25 | try { 26 | this.value = Pattern.compile(value); 27 | }catch(PatternSyntaxException e) { 28 | throw new ParseException(e.getDescription(), 0); 29 | } 30 | } 31 | 32 | @Override 33 | public boolean test(Map attributes) { 34 | Predicate> exist = metric -> attributes.containsKey(key); 35 | Predicate> match = metric -> value.matcher(attributes.get(key)).matches(); 36 | 37 | return exist.and(match).test(attributes); 38 | } 39 | 40 | @Override 41 | public String toString() { 42 | return key + " == \"" + value + "\""; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/analysis/types/htm/AnomaliesResults.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.analysis.types.htm; 2 | 3 | import java.io.Serializable; 4 | 5 | public class AnomaliesResults implements Serializable{ 6 | 7 | private static final long serialVersionUID = 2504261847208235950L; 8 | private double score; 9 | private double errThreshold; 10 | private double warnThreshold; 11 | 12 | public AnomaliesResults(double score, double errorThreshold, double warningThreshold){ 13 | setAnomalyLikelihoodScore(score); 14 | setErrThreshold(errorThreshold); 15 | setWarnThreshold(warningThreshold); 16 | } 17 | 18 | public double getAnomalyLikelihoodScore() { 19 | return score; 20 | } 21 | 22 | public void setAnomalyLikelihoodScore(double score) { 23 | this.score = score; 24 | } 25 | 26 | public double getErrThreshold() { 27 | return errThreshold; 28 | } 29 | 30 | public void setErrThreshold(double errThreshold) { 31 | this.errThreshold = errThreshold; 32 | } 33 | 34 | public double getWarnThreshold() { 35 | return warnThreshold; 36 | } 37 | 38 | public void setWarnThreshold(double warnThreshold) { 39 | this.warnThreshold = warnThreshold; 40 | } 41 | 42 | public boolean isError() { 43 | return score > getErrThreshold(); 44 | } 45 | 46 | public boolean isWarning() { 47 | return score > getWarnThreshold(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /bin/stop: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2018, CERN 4 | # This software is distributed under the terms of the GNU General Public 5 | # Licence version 3 (GPL Version 3), copied verbatim in the file "LICENSE". 6 | # In applying this license, CERN does not waive the privileges and immunities 7 | # granted to it by virtue of its status as Intergovernmental Organization 8 | # or submit itself to any jurisdiction. 9 | 10 | # Reference: http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in 11 | SOURCE="${BASH_SOURCE[0]}" 12 | while [ -h "$SOURCE" ]; do 13 | DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" 14 | SOURCE="$(readlink "$SOURCE")" 15 | [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" 16 | done 17 | SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" 18 | 19 | HOME=$SCRIPT_DIR/.. 20 | 21 | if [ -e $HOME/conf/exdemon-env.sh ] ; then 22 | . $HOME/conf/exdemon-env.sh 23 | fi 24 | 25 | YARN_CMD=yarn 26 | if [ ! -z "${HADOOP_YARN_HOME}" ]; then 27 | YARN_CMD=$HADOOP_YARN_HOME/bin/$YARN_CMD 28 | fi 29 | 30 | ZOOCNX=`sed -n 's/^spark.streaming.listener.connection_string[ ]=//p' $HOME/conf/exdemon.properties` 31 | ZOOPTH=`sed -n 's/^spark.streaming.listener.path[ ]=//p' $HOME/conf/exdemon.properties` 32 | 33 | APPID=`zookeepercli -servers $ZOOCNX -c get $ZOOPTH/id 2>/dev/null | grep application_` 34 | 35 | $YARN_CMD application -kill $APPID 36 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/value/PropertiesValue.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.value; 2 | 3 | import java.time.Instant; 4 | import java.util.Optional; 5 | 6 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 7 | import ch.cern.exdemon.metrics.defined.equation.var.VariableStatuses; 8 | import ch.cern.properties.Properties; 9 | import lombok.EqualsAndHashCode; 10 | import lombok.NonNull; 11 | 12 | @EqualsAndHashCode(callSuper=false) 13 | public class PropertiesValue extends Value implements ValueComputable{ 14 | 15 | private static final long serialVersionUID = 3844172330757307935L; 16 | 17 | private String properties_name; 18 | 19 | private Properties props; 20 | 21 | public PropertiesValue(String name, @NonNull Properties props) { 22 | this.properties_name = name; 23 | this.props = props; 24 | } 25 | 26 | @Override 27 | public Optional getAsProperties() { 28 | return Optional.of(props); 29 | } 30 | 31 | @Override 32 | public Value compute(VariableStatuses store, Instant time) { 33 | return this; 34 | } 35 | 36 | @Override 37 | public Class returnType() { 38 | return PropertiesValue.class; 39 | } 40 | 41 | @Override 42 | public String toString() { 43 | return "props(" + properties_name + ")"; 44 | } 45 | 46 | public String getName() { 47 | return properties_name; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/var/agg/CountFloatsAgregation.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var.agg; 2 | 3 | import java.time.Instant; 4 | import java.util.Collection; 5 | 6 | import ch.cern.exdemon.components.RegisterComponentType; 7 | import ch.cern.exdemon.metrics.DatedValue; 8 | import ch.cern.exdemon.metrics.value.FloatValue; 9 | import ch.cern.exdemon.metrics.value.Value; 10 | 11 | @RegisterComponentType("count_floats") 12 | public class CountFloatsAgregation extends Aggregation { 13 | 14 | private static final long serialVersionUID = 8274088208286521725L; 15 | 16 | @Override 17 | public Class inputType() { 18 | return FloatValue.class; 19 | } 20 | 21 | @Override 22 | public Value aggregate(Collection values, Instant time) { 23 | return new FloatValue(values.stream() 24 | .mapToDouble(v -> v.getValue().getAsAggregated().isPresent() ? 25 | v.getValue().getAsAggregated().get().getAsFloat().get() : 26 | 1) 27 | .sum()); 28 | } 29 | 30 | @Override 31 | public Class returnType() { 32 | return FloatValue.class; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/value/ExceptionValue.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.value; 2 | 3 | import java.time.Instant; 4 | import java.util.Optional; 5 | 6 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 7 | import ch.cern.exdemon.metrics.defined.equation.var.VariableStatuses; 8 | import lombok.EqualsAndHashCode; 9 | import lombok.NonNull; 10 | 11 | @EqualsAndHashCode(callSuper=false) 12 | public class ExceptionValue extends Value implements ValueComputable{ 13 | 14 | private static final long serialVersionUID = 8938782791564766439L; 15 | 16 | private String exception_message; 17 | 18 | public ExceptionValue(@NonNull String message) { 19 | this.exception_message = message; 20 | } 21 | 22 | @Override 23 | public Optional getAsException() { 24 | return Optional.of(exception_message); 25 | } 26 | 27 | @Override 28 | public ExceptionValue compute(VariableStatuses store, Instant time) { 29 | return new ExceptionValue(exception_message); 30 | } 31 | 32 | @Override 33 | public Class returnType() { 34 | return ExceptionValue.class; 35 | } 36 | 37 | @Override 38 | public String toString() { 39 | return "{Error: " + exception_message + "}"; 40 | } 41 | 42 | @Override 43 | public String getSource() { 44 | if(source == null) 45 | return toString(); 46 | else 47 | return super.toString(); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/Equation.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation; 2 | 3 | import java.text.ParseException; 4 | import java.time.Instant; 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | 8 | import ch.cern.exdemon.metrics.defined.equation.var.Variable; 9 | import ch.cern.exdemon.metrics.defined.equation.var.VariableStatuses; 10 | import ch.cern.exdemon.metrics.value.Value; 11 | import ch.cern.properties.ConfigurationException; 12 | import ch.cern.properties.Properties; 13 | import lombok.EqualsAndHashCode; 14 | import lombok.Getter; 15 | import lombok.ToString; 16 | 17 | @ToString 18 | @EqualsAndHashCode(callSuper=false) 19 | public class Equation implements ValueComputable{ 20 | 21 | private static EquationParser parser = new EquationParser(); 22 | 23 | private ValueComputable formula; 24 | 25 | @Getter 26 | private Map variables = new HashMap<>(); 27 | 28 | public Equation(String equationString, Properties variablesProperties) throws ParseException, ConfigurationException { 29 | this.formula = parser.parse(equationString, variablesProperties, variables); 30 | } 31 | 32 | @Override 33 | public Value compute(VariableStatuses stores, Instant time) { 34 | return formula.compute(stores, time); 35 | } 36 | 37 | @Override 38 | public Class returnType() { 39 | return Value.class; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/status/storage/manager/ClassNameStatusKeyFilter.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status.storage.manager; 2 | 3 | import org.apache.spark.api.java.function.Function; 4 | 5 | import ch.cern.spark.status.StatusKey; 6 | import ch.cern.spark.status.StatusValue; 7 | import ch.cern.spark.status.storage.ClassNameAlias; 8 | import lombok.EqualsAndHashCode; 9 | import lombok.ToString; 10 | import scala.Tuple2; 11 | 12 | @ToString 13 | @EqualsAndHashCode(callSuper=false) 14 | public class ClassNameStatusKeyFilter implements Function, Boolean> { 15 | 16 | private static final long serialVersionUID = -5039261513238813096L; 17 | private String filter_by_fqcn; 18 | 19 | public ClassNameStatusKeyFilter(String filter_by_fqcn) { 20 | this.filter_by_fqcn = filter_by_fqcn; 21 | } 22 | 23 | @Override 24 | public Boolean call(Tuple2 tuple) throws Exception { 25 | if(filter_by_fqcn == null) 26 | return true; 27 | 28 | StatusKey key = tuple._1; 29 | 30 | String name = key.getClass().getName(); 31 | if(name.equals(filter_by_fqcn)) 32 | return true; 33 | 34 | ClassNameAlias aliasAnno = key.getClass().getAnnotation(ClassNameAlias.class); 35 | 36 | return aliasAnno != null && aliasAnno.value().equals(filter_by_fqcn); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/status/storage/manager/ValueSizeFilter.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status.storage.manager; 2 | 3 | import org.apache.spark.api.java.function.Function; 4 | 5 | import ch.cern.spark.status.StatusKey; 6 | import ch.cern.spark.status.StatusValue; 7 | import ch.cern.spark.status.storage.StatusSerializer; 8 | import lombok.EqualsAndHashCode; 9 | import lombok.ToString; 10 | import scala.Tuple2; 11 | 12 | @ToString 13 | @EqualsAndHashCode(callSuper=false) 14 | public class ValueSizeFilter implements Function, Boolean> { 15 | 16 | private static final long serialVersionUID = -6806354277199482268L; 17 | 18 | private long min_value_size; 19 | 20 | private StatusSerializer serializer; 21 | 22 | public ValueSizeFilter(long min_value_size, StatusSerializer serializer) { 23 | this.min_value_size = min_value_size; 24 | this.serializer = serializer; 25 | } 26 | 27 | @Override 28 | public Boolean call(Tuple2 tuple) throws Exception { 29 | StatusValue value = tuple._2; 30 | 31 | if(value == null) 32 | return true; 33 | 34 | byte[] serialized = serializer.fromValue(value); 35 | 36 | if(serialized == null) 37 | return true; 38 | 39 | return serialized.length >= min_value_size; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/value/BooleanValue.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.value; 2 | 3 | import java.time.Instant; 4 | import java.util.Optional; 5 | 6 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 7 | import ch.cern.exdemon.metrics.defined.equation.var.VariableStatuses; 8 | import lombok.EqualsAndHashCode; 9 | 10 | @EqualsAndHashCode(callSuper=false) 11 | public class BooleanValue extends Value implements ValueComputable{ 12 | 13 | private static final long serialVersionUID = 6026199196915653369L; 14 | 15 | private boolean bool; 16 | 17 | public BooleanValue(boolean value){ 18 | this.bool = value; 19 | } 20 | 21 | @Override 22 | public Optional getAsBoolean() { 23 | return Optional.of(bool); 24 | } 25 | 26 | public static BooleanValue from(String value_string) { 27 | return new BooleanValue(Boolean.parseBoolean(value_string)); 28 | } 29 | 30 | @Override 31 | public BooleanValue compute(VariableStatuses store, Instant time) { 32 | return new BooleanValue(bool); 33 | } 34 | 35 | @Override 36 | public Class returnType() { 37 | return BooleanValue.class; 38 | } 39 | 40 | @Override 41 | public String toString() { 42 | return Boolean.toString(bool); 43 | } 44 | 45 | @Override 46 | public String getSource() { 47 | if(source == null) 48 | return toString(); 49 | else 50 | return super.toString(); 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/exdemon/metrics/defined/DefinedMetricIDTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | import static org.junit.Assert.assertNotEquals; 5 | 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | 9 | import org.junit.Test; 10 | 11 | public class DefinedMetricIDTest { 12 | 13 | @Test 14 | public void equality() { 15 | Map ids1 = new HashMap<>(); 16 | ids1.put("a", "a1"); 17 | ids1.put("b", "b1"); 18 | DefinedMetricStatuskey id1 = new DefinedMetricStatuskey("id1", ids1); 19 | 20 | Map ids2 = new HashMap<>(); 21 | ids2.put("a", "a1"); 22 | ids2.put("b", "b1"); 23 | DefinedMetricStatuskey id2 = new DefinedMetricStatuskey("id1", ids2); 24 | 25 | assertEquals(id1, id2); 26 | assertEquals(id1.hashCode(), id2.hashCode()); 27 | } 28 | 29 | @Test 30 | public void inequality() { 31 | Map ids1 = new HashMap<>(); 32 | ids1.put("a", "a1"); 33 | ids1.put("b", "b1"); 34 | ids1.put("c", "c1"); 35 | DefinedMetricStatuskey id1 = new DefinedMetricStatuskey("id1", ids1); 36 | 37 | Map ids2 = new HashMap<>(); 38 | ids2.put("a", "a1"); 39 | ids2.put("b", "b1"); 40 | DefinedMetricStatuskey id2 = new DefinedMetricStatuskey("id1", ids2); 41 | 42 | assertNotEquals(id1, id2); 43 | assertNotEquals(id1.hashCode(), id2.hashCode()); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/status/StatusValue.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status; 2 | 3 | import java.io.Serializable; 4 | 5 | import org.apache.spark.streaming.State; 6 | import org.apache.spark.streaming.Time; 7 | 8 | import lombok.Getter; 9 | 10 | public abstract class StatusValue implements Serializable { 11 | 12 | private static final long serialVersionUID = -2362831624128266105L; 13 | 14 | @Getter 15 | private long status_update_time = 0; 16 | 17 | public StatusValue() { 18 | status_update_time = 0; 19 | } 20 | 21 | @SuppressWarnings("unchecked") 22 | public void update(State status, Time time) { 23 | status_update_time = time.milliseconds(); 24 | 25 | status.update((T) this); 26 | } 27 | 28 | @Override 29 | public int hashCode() { 30 | final int prime = 31; 31 | int result = 1; 32 | result = prime * result + (int) (status_update_time ^ (status_update_time >>> 32)); 33 | return result; 34 | } 35 | 36 | @Override 37 | public boolean equals(Object obj) { 38 | if (this == obj) 39 | return true; 40 | if (obj == null) 41 | return false; 42 | if (getClass() != obj.getClass()) 43 | return false; 44 | StatusValue other = (StatusValue) obj; 45 | if (status_update_time != other.status_update_time) 46 | return false; 47 | return true; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/exdemon/monitor/analysis/StringAnalysisTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.analysis; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | 5 | import java.time.Instant; 6 | 7 | import org.junit.Test; 8 | 9 | import ch.cern.exdemon.metrics.Metric; 10 | import ch.cern.exdemon.metrics.value.BooleanValue; 11 | import ch.cern.exdemon.metrics.value.FloatValue; 12 | import ch.cern.exdemon.metrics.value.StringValue; 13 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult; 14 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult.Status; 15 | 16 | public class StringAnalysisTest { 17 | 18 | private StringAnalysis analysis = new StringAnalysis() { 19 | private static final long serialVersionUID = 2630885101669119453L; 20 | 21 | @Override 22 | public AnalysisResult process(Instant timestamp, String value) { 23 | return AnalysisResult.buildWithStatus(Status.OK, ""); 24 | } 25 | }; 26 | 27 | @Test 28 | public void shouldAnalyzeOnlyStringMetric() { 29 | assertEquals(Status.OK, analysis.apply(new Metric(Instant.now(), new StringValue(""), null)).getStatus()); 30 | assertEquals(Status.EXCEPTION, analysis.apply(new Metric(Instant.now(), new FloatValue(0f), null)).getStatus()); 31 | assertEquals(Status.EXCEPTION, analysis.apply(new Metric(Instant.now(), new BooleanValue(true), null)).getStatus()); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/exdemon/monitor/analysis/BooleanAnalysisTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.analysis; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | 5 | import java.time.Instant; 6 | 7 | import org.junit.Test; 8 | 9 | import ch.cern.exdemon.metrics.Metric; 10 | import ch.cern.exdemon.metrics.value.BooleanValue; 11 | import ch.cern.exdemon.metrics.value.FloatValue; 12 | import ch.cern.exdemon.metrics.value.StringValue; 13 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult; 14 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult.Status; 15 | 16 | public class BooleanAnalysisTest { 17 | 18 | private BooleanAnalysis analysis = new BooleanAnalysis() { 19 | private static final long serialVersionUID = 2630885101669119453L; 20 | 21 | @Override 22 | public AnalysisResult process(Instant timestamp, boolean value) { 23 | return AnalysisResult.buildWithStatus(Status.OK, ""); 24 | } 25 | }; 26 | 27 | @Test 28 | public void shouldAnalyzeOnlyStringMetric() { 29 | assertEquals(Status.EXCEPTION, analysis.apply(new Metric(Instant.now(), new StringValue(""), null)).getStatus()); 30 | assertEquals(Status.EXCEPTION, analysis.apply(new Metric(Instant.now(), new FloatValue(0f), null)).getStatus()); 31 | assertEquals(Status.OK, analysis.apply(new Metric(Instant.now(), new BooleanValue(true), null)).getStatus()); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/exdemon/monitor/analysis/NumericAnalysisTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.analysis; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | 5 | import java.time.Instant; 6 | 7 | import org.junit.Test; 8 | 9 | import ch.cern.exdemon.metrics.Metric; 10 | import ch.cern.exdemon.metrics.value.BooleanValue; 11 | import ch.cern.exdemon.metrics.value.FloatValue; 12 | import ch.cern.exdemon.metrics.value.StringValue; 13 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult; 14 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult.Status; 15 | 16 | public class NumericAnalysisTest { 17 | 18 | private NumericAnalysis analysis = new NumericAnalysis() { 19 | private static final long serialVersionUID = 2630885101669119453L; 20 | 21 | @Override 22 | public AnalysisResult process(Instant timestamp, double value) { 23 | return AnalysisResult.buildWithStatus(Status.OK, ""); 24 | } 25 | }; 26 | 27 | @Test 28 | public void shouldAnalyzeOnlyStringMetric() { 29 | assertEquals(Status.EXCEPTION, analysis.apply(new Metric(Instant.now(), new StringValue(""), null)).getStatus()); 30 | assertEquals(Status.OK, analysis.apply(new Metric(Instant.now(), new FloatValue(0f), null)).getStatus()); 31 | assertEquals(Status.EXCEPTION, analysis.apply(new Metric(Instant.now(), new BooleanValue(true), null)).getStatus()); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /doc/users-manual/running.md: -------------------------------------------------------------------------------- 1 | # Running the application 2 | 3 | ## Download and building 4 | 5 | First, you need to clone this repository and build it with [Apache Maven](https://maven.apache.org/). 6 | 7 | ``` 8 | git clone https://github.com/cerndb/metrics-monitor.git 9 | cd metrics-monitor 10 | mvn -DskipTests package 11 | ``` 12 | 13 | You will find the project JAR at target/metrics-monitor-VERSION.jar 14 | 15 | ## Prepare environment with Apache Spark 16 | 17 | This project is based in [Apache Spark](https://spark.apache.org/), so you need the binaries. 18 | You may have it installed or it can be downloaded from https://spark.apache.org/downloads.html. 19 | 20 | The project is developed based on the version 2.3.0 of Spark, so we recommend to use this version. 21 | You can download it from [here](https://archive.apache.org/dist/spark/spark-2.3.0/spark-2.3.0-bin-hadoop2.7.tgz). 22 | 23 | ## Running 24 | 25 | As you may know, applications implemented with Spark can be run standalone, on YARN, Mesos or other platforms. 26 | 27 | To run this applications you can use the following command: 28 | 29 | ``` 30 | $SPARK_HOME/bin/spark-submit \ 31 | --repositories https://repository.cloudera.com/artifactory/cloudera-repos/ \ 32 | --packages org.apache.spark:spark-streaming-kafka-0-10_2.11:2.1.0,org.reflections:reflections:0.9.9 \ 33 | --class ch.cern.exdemon.Driver \ 34 | target/metrics-monitor-VERSION.jar \ 35 | 36 | ``` -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/var/agg/CountBoolsAgregation.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var.agg; 2 | 3 | import java.time.Instant; 4 | import java.util.Collection; 5 | 6 | import ch.cern.exdemon.components.RegisterComponentType; 7 | import ch.cern.exdemon.metrics.DatedValue; 8 | import ch.cern.exdemon.metrics.value.BooleanValue; 9 | import ch.cern.exdemon.metrics.value.FloatValue; 10 | import ch.cern.exdemon.metrics.value.Value; 11 | 12 | @RegisterComponentType("count_bools") 13 | public class CountBoolsAgregation extends Aggregation { 14 | 15 | private static final long serialVersionUID = 8274088208286521725L; 16 | 17 | @Override 18 | public Class inputType() { 19 | return BooleanValue.class; 20 | } 21 | 22 | @Override 23 | public Value aggregate(Collection values, Instant time) { 24 | return new FloatValue(values.stream() 25 | .mapToDouble(v -> v.getValue().getAsAggregated().isPresent() ? 26 | v.getValue().getAsAggregated().get().getAsFloat().get() : 27 | 1) 28 | .sum()); 29 | } 30 | 31 | @Override 32 | public Class returnType() { 33 | return FloatValue.class; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/var/agg/CountStringsAgregation.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var.agg; 2 | 3 | import java.time.Instant; 4 | import java.util.Collection; 5 | 6 | import ch.cern.exdemon.components.RegisterComponentType; 7 | import ch.cern.exdemon.metrics.DatedValue; 8 | import ch.cern.exdemon.metrics.value.FloatValue; 9 | import ch.cern.exdemon.metrics.value.StringValue; 10 | import ch.cern.exdemon.metrics.value.Value; 11 | 12 | @RegisterComponentType("count_strings") 13 | public class CountStringsAgregation extends Aggregation { 14 | 15 | private static final long serialVersionUID = 8274088208286521725L; 16 | 17 | @Override 18 | public Class inputType() { 19 | return StringValue.class; 20 | } 21 | 22 | @Override 23 | public Value aggregate(Collection values, Instant time) { 24 | return new FloatValue(values.stream() 25 | .mapToDouble(v -> v.getValue().getAsAggregated().isPresent() ? 26 | v.getValue().getAsAggregated().get().getAsFloat().get() : 27 | 1) 28 | .sum()); 29 | } 30 | 31 | @Override 32 | public Class returnType() { 33 | return FloatValue.class; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/exdemon/DriverTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | import static org.junit.Assert.fail; 5 | 6 | import org.junit.Test; 7 | 8 | import ch.cern.properties.ConfigurationException; 9 | import ch.cern.properties.Properties; 10 | 11 | public class DriverTest { 12 | 13 | @Test 14 | public void notConfiguredMetricSource() throws Exception{ 15 | Properties props = new Properties(); 16 | props.setProperty("spark.driver.allowMultipleContexts", "true"); 17 | 18 | try{ 19 | Driver driver = new Driver(props); 20 | 21 | driver.createNewStreamingContext(null); 22 | 23 | fail(); 24 | }catch(ConfigurationException e){ 25 | assertEquals("At least one metric source must be configured", e.getMessage()); 26 | } 27 | } 28 | 29 | @Test 30 | public void configurationWithAnalysisResultsSink() throws Exception{ 31 | Properties props = new Properties(); 32 | props.setProperty("spark.driver.allowMultipleContexts", "true"); 33 | props.setProperty("metrics.source.kafka.type", "kafka"); 34 | props.setProperty("metrics.source.kafka.topics", "topic"); 35 | props.setProperty("results.sink.type", "elastic"); 36 | 37 | Driver driver = new Driver(props); 38 | 39 | driver.createNewStreamingContext(null); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/http/JsonPOSTRequest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.http; 2 | 3 | import java.io.UnsupportedEncodingException; 4 | import java.text.ParseException; 5 | 6 | import org.apache.http.client.methods.HttpPost; 7 | import org.apache.http.entity.StringEntity; 8 | 9 | import ch.cern.exdemon.json.JSON; 10 | import lombok.EqualsAndHashCode; 11 | import lombok.Getter; 12 | import lombok.ToString; 13 | 14 | @ToString 15 | @EqualsAndHashCode(callSuper=false) 16 | public class JsonPOSTRequest { 17 | 18 | @Getter 19 | private String url; 20 | 21 | @Getter 22 | private JSON json; 23 | 24 | public JsonPOSTRequest(String url, JSON json) { 25 | this.url = url; 26 | this.json = json; 27 | } 28 | 29 | public void addProperty(String key, String value) throws ParseException { 30 | json.setProperty(key, value); 31 | } 32 | 33 | public HttpPost toPostMethod() throws UnsupportedEncodingException { 34 | HttpPost httpPost = new HttpPost(url); 35 | 36 | httpPost.setEntity(new StringEntity(json.toString())); 37 | httpPost.setHeader("Accept", "application/json"); 38 | httpPost.setHeader("Content-type", "application/json"); 39 | 40 | return httpPost; 41 | } 42 | 43 | public JsonPOSTRequest setUrlIfNull(String url) { 44 | if(this.url == null) 45 | this.url = url; 46 | 47 | return this; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/source/MetricsSource.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.source; 2 | 3 | import org.apache.spark.streaming.api.java.JavaDStream; 4 | import org.apache.spark.streaming.api.java.JavaStreamingContext; 5 | 6 | import ch.cern.exdemon.components.Component; 7 | import ch.cern.exdemon.components.Component.Type; 8 | import ch.cern.exdemon.components.ComponentType; 9 | import ch.cern.exdemon.components.ConfigurationResult; 10 | import ch.cern.properties.Properties; 11 | 12 | @ComponentType(Type.METRIC_SOURCE) 13 | public abstract class MetricsSource extends Component { 14 | 15 | private static final long serialVersionUID = -6197974524956447741L; 16 | 17 | private int partitions; 18 | 19 | @Override 20 | public ConfigurationResult config(Properties properties) { 21 | properties.isTypeDefined(); 22 | 23 | partitions = (int) properties.getLong("partitions", -1); 24 | 25 | return ConfigurationResult.SUCCESSFUL(); 26 | } 27 | 28 | public JavaDStream stream(JavaStreamingContext ssc){ 29 | JavaDStream stream = createJavaDStream(ssc); 30 | 31 | return partitions > 0 ? stream.repartition(partitions) : stream; 32 | } 33 | 34 | /** 35 | * Obtain metrics from external services as JSON strings. 36 | * 37 | * @param ssc Spark context 38 | * @return DStream of valid JSON object strings. 39 | */ 40 | protected abstract JavaDStream createJavaDStream(JavaStreamingContext ssc); 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/filter/NotEqualMetricPredicate.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.filter; 2 | 3 | import java.io.Serializable; 4 | import java.text.ParseException; 5 | import java.util.Map; 6 | import java.util.function.Predicate; 7 | import java.util.regex.Pattern; 8 | import java.util.regex.PatternSyntaxException; 9 | 10 | import lombok.Getter; 11 | 12 | public class NotEqualMetricPredicate implements Predicate>, Serializable { 13 | 14 | private static final long serialVersionUID = -1044577733678850309L; 15 | 16 | @Getter 17 | private String key; 18 | 19 | @Getter 20 | private Pattern value; 21 | 22 | public NotEqualMetricPredicate(String key, String value) throws ParseException { 23 | this.key = key; 24 | 25 | try { 26 | this.value = Pattern.compile(value); 27 | } catch (PatternSyntaxException e) { 28 | throw new ParseException(e.getDescription(), 0); 29 | } 30 | } 31 | 32 | @Override 33 | public boolean test(Map attributes) { 34 | Predicate> notExist = metric -> !attributes.containsKey(key); 35 | Predicate> notMatch = metric -> !value.matcher(attributes.get(key)).matches(); 36 | 37 | return notExist.or(notMatch).test(attributes); 38 | } 39 | 40 | @Override 41 | public String toString() { 42 | return key + " != \"" + value + "\""; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/ComputeMonitorKeysF.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor; 2 | 3 | import java.util.Iterator; 4 | import java.util.Map; 5 | 6 | import org.apache.spark.api.java.function.PairFlatMapFunction; 7 | 8 | import ch.cern.exdemon.components.ComponentsCatalog; 9 | import ch.cern.exdemon.components.Component.Type; 10 | import ch.cern.exdemon.metrics.Metric; 11 | import ch.cern.properties.Properties; 12 | import scala.Tuple2; 13 | 14 | public class ComputeMonitorKeysF implements PairFlatMapFunction{ 15 | 16 | private static final long serialVersionUID = 2181051149359938177L; 17 | 18 | private Properties componentsSourceProperties; 19 | 20 | public ComputeMonitorKeysF(Properties componentsSourceProperties) { 21 | this.componentsSourceProperties = componentsSourceProperties; 22 | } 23 | 24 | @Override 25 | public Iterator> call(Metric metric) throws Exception { 26 | ComponentsCatalog.init(componentsSourceProperties); 27 | 28 | Map monitors = ComponentsCatalog.get(Type.MONITOR); 29 | 30 | return monitors.values().stream() 31 | .filter(monitor -> monitor.getFilter().test(metric)) 32 | .map(monitor -> new MonitorStatusKey(monitor.getId(), monitor.getMetricIDs(metric))) 33 | .map(ids -> new Tuple2(ids, metric)) 34 | .iterator(); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/spark/status/storage/manager/ToStringPatternStatusKeyFilterTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status.storage.manager; 2 | 3 | import static org.junit.Assert.assertFalse; 4 | import static org.junit.Assert.assertTrue; 5 | 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | 9 | import org.junit.Test; 10 | 11 | import ch.cern.exdemon.metrics.defined.DefinedMetricStatuskey; 12 | import ch.cern.exdemon.monitor.MonitorStatusKey; 13 | import ch.cern.exdemon.monitor.trigger.TriggerStatusKey; 14 | import ch.cern.spark.status.TestStatus; 15 | import scala.Tuple2; 16 | 17 | public class ToStringPatternStatusKeyFilterTest { 18 | 19 | @Test 20 | public void filterByID() throws Exception { 21 | ToStringPatternStatusKeyFilter filter = new ToStringPatternStatusKeyFilter(".*tpsrv12.*"); 22 | 23 | assertFalse(filter.call(new Tuple2<>(new DefinedMetricStatuskey("dm1", new HashMap<>()), new TestStatus(1)))); 24 | 25 | assertTrue(filter.call(new Tuple2<>(new DefinedMetricStatuskey("tpsrv12", new HashMap<>()), new TestStatus(1)))); 26 | 27 | Map att = new HashMap<>(); 28 | att.put("", "tpsrv12"); 29 | assertTrue(filter.call(new Tuple2<>(new MonitorStatusKey("m1", att), new TestStatus(1)))); 30 | 31 | att = new HashMap<>(); 32 | att.put("", "tpsrv12"); 33 | assertTrue(filter.call(new Tuple2<>(new TriggerStatusKey("m1", "dm2", att), new TestStatus(1)))); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /conf/log4j.properties: -------------------------------------------------------------------------------- 1 | # Set everything to be logged to the console 2 | log4j.rootCategory=INFO, console, http 3 | 4 | log4j.appender.console=org.apache.log4j.ConsoleAppender 5 | log4j.appender.console.target=System.err 6 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 8 | 9 | log4j.appender.file=org.apache.log4j.RollingFileAppender 10 | log4j.appender.file.File=/root/exdemon/log/spark-job.log 11 | log4j.appender.file.MaxFileSize=10MB 12 | log4j.appender.file.MaxBackupIndex=10 13 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 14 | log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 15 | 16 | log4j.appender.http=ch.cern.log4j.HTTPAppender 17 | log4j.appender.http.url=http://monit-logs.cern.ch:10012/ 18 | log4j.appender.http.add= producer="tape" type="exdemon-spark" environment="" 19 | 20 | # There are monitors looking at these lines 21 | log4j.logger.org.apache.spark.streaming.scheduler.JobScheduler=INFO, console, http 22 | log4j.logger.org.apache.spark.scheduler.TaskSetManager=INFO, console, http 23 | 24 | # Settings to quiet third party logs that are too verbose 25 | log4j.logger.org.eclipse.jetty=WARN 26 | log4j.logger.org.apache.hadoop.yarn.server.webproxy=ERROR 27 | log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO 28 | log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO 29 | log4j.logger.org.apache.spark.streaming.kafka010.CachedKafkaConsumer=INFO -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/var/VariableStatuses.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var; 2 | 3 | import java.time.Instant; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | import ch.cern.spark.status.StatusValue; 8 | import ch.cern.spark.status.storage.ClassNameAlias; 9 | import lombok.EqualsAndHashCode; 10 | import lombok.ToString; 11 | 12 | @ClassNameAlias("defined-metric-status") 13 | @ToString 14 | @EqualsAndHashCode(callSuper=false) 15 | public class VariableStatuses extends StatusValue { 16 | 17 | private static final long serialVersionUID = 3020679839103994736L; 18 | 19 | private Map statuses; 20 | 21 | private Instant processedBatchTime; 22 | 23 | public VariableStatuses() { 24 | statuses = new HashMap<>(); 25 | } 26 | 27 | public boolean newProcessedBatchTime(Instant time) { 28 | if(this.processedBatchTime == null) { 29 | this.processedBatchTime = time; 30 | 31 | return true; 32 | }else { 33 | if(this.processedBatchTime.compareTo(time) == 0) { 34 | return false; 35 | }else{ 36 | this.processedBatchTime = time; 37 | 38 | return true; 39 | } 40 | } 41 | } 42 | 43 | public void put(String varName, VariableStatus status) { 44 | statuses.put(varName, status); 45 | } 46 | 47 | public boolean containsVariable(String StatusValue) { 48 | return statuses.containsKey(StatusValue); 49 | } 50 | 51 | public VariableStatus get(String varName) { 52 | return statuses.get(varName); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /bin/status-manager: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2016, CERN 4 | # This software is distributed under the terms of the GNU General Public 5 | # Licence version 3 (GPL Version 3), copied verbatim in the file "LICENSE". 6 | # In applying this license, CERN does not waive the privileges and immunities 7 | # granted to it by virtue of its status as Intergovernmental Organization 8 | # or submit itself to any jurisdiction. 9 | 10 | # Reference: http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in 11 | SOURCE="${BASH_SOURCE[0]}" 12 | while [ -h "$SOURCE" ]; do 13 | DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" 14 | SOURCE="$(readlink "$SOURCE")" 15 | [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" 16 | done 17 | SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" 18 | 19 | HOME=$SCRIPT_DIR/.. 20 | 21 | if [ -e $HOME/conf/exdemon-env.sh ] ; then 22 | . $HOME/conf/exdemon-env.sh 23 | fi 24 | 25 | SPARK_CMD=spark-submit 26 | if [ ! -z "${SPARK_HOME}" ]; then 27 | SPARK_CMD=$SPARK_HOME/bin/$SPARK_CMD 28 | fi 29 | 30 | $SPARK_CMD \ 31 | --master local \ 32 | --driver-memory 2G \ 33 | --packages org.apache.spark:spark-streaming-kafka-0-10_2.11:2.2.1,org.reflections:reflections:0.9.9 \ 34 | --class ch.cern.spark.status.storage.manager.StatusesManagerCLI \ 35 | --conf spark.authenticate=false \ 36 | $HOME/lib/exdemon-*.jar \ 37 | -c file://$HOME/conf/exdemon.properties \ 38 | --print json \ 39 | --save /tmp/status \ 40 | "$@" 41 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/var/agg/CountTruesAgregation.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var.agg; 2 | 3 | import java.time.Instant; 4 | import java.util.Collection; 5 | 6 | import ch.cern.exdemon.components.RegisterComponentType; 7 | import ch.cern.exdemon.metrics.DatedValue; 8 | import ch.cern.exdemon.metrics.value.BooleanValue; 9 | import ch.cern.exdemon.metrics.value.FloatValue; 10 | import ch.cern.exdemon.metrics.value.Value; 11 | 12 | @RegisterComponentType("count_true") 13 | public class CountTruesAgregation extends Aggregation { 14 | 15 | private static final long serialVersionUID = 8274088208286521725L; 16 | 17 | @Override 18 | public Class inputType() { 19 | return BooleanValue.class; 20 | } 21 | 22 | @Override 23 | public Value aggregate(Collection values, Instant time) { 24 | return new FloatValue(values.stream() 25 | .filter(v -> v.getValue().getAsBoolean().get() || v.getValue().getAsAggregated().isPresent()) 26 | .mapToDouble(v -> v.getValue().getAsAggregated().isPresent() ? 27 | v.getValue().getAsAggregated().get().getAsFloat().get() : 28 | 1) 29 | .sum()); 30 | } 31 | 32 | @Override 33 | public Class returnType() { 34 | return FloatValue.class; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/UpdateMonitorStatusesF.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor; 2 | 3 | import java.util.Optional; 4 | 5 | import org.apache.spark.streaming.State; 6 | 7 | import ch.cern.exdemon.components.ComponentsCatalog; 8 | import ch.cern.exdemon.components.Component.Type; 9 | import ch.cern.exdemon.metrics.Metric; 10 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult; 11 | import ch.cern.properties.Properties; 12 | import ch.cern.spark.status.StatusValue; 13 | import ch.cern.spark.status.UpdateStatusFunction; 14 | 15 | public class UpdateMonitorStatusesF extends UpdateStatusFunction { 16 | 17 | private static final long serialVersionUID = 3156649511706333348L; 18 | 19 | private Properties componentsSourceProperties; 20 | 21 | public UpdateMonitorStatusesF(Properties componentsSourceProperties) { 22 | this.componentsSourceProperties = componentsSourceProperties; 23 | } 24 | 25 | @Override 26 | protected Optional update(MonitorStatusKey ids, Metric metric, State status) 27 | throws Exception { 28 | ComponentsCatalog.init(componentsSourceProperties); 29 | 30 | Optional monitorOpt = ComponentsCatalog.get(Type.MONITOR, ids.getID()); 31 | if(!monitorOpt.isPresent()) { 32 | status.remove(); 33 | 34 | return Optional.empty(); 35 | } 36 | 37 | return monitorOpt.get().process(status, metric); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/var/agg/CountFalsesAgregation.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var.agg; 2 | 3 | import java.time.Instant; 4 | import java.util.Collection; 5 | 6 | import ch.cern.exdemon.components.RegisterComponentType; 7 | import ch.cern.exdemon.metrics.DatedValue; 8 | import ch.cern.exdemon.metrics.value.BooleanValue; 9 | import ch.cern.exdemon.metrics.value.FloatValue; 10 | import ch.cern.exdemon.metrics.value.Value; 11 | 12 | @RegisterComponentType("count_false") 13 | public class CountFalsesAgregation extends Aggregation { 14 | 15 | private static final long serialVersionUID = 8274088208286521725L; 16 | 17 | @Override 18 | public Class inputType() { 19 | return BooleanValue.class; 20 | } 21 | 22 | @Override 23 | public Value aggregate(Collection values, Instant time) { 24 | return new FloatValue(values.stream() 25 | .filter(v -> !v.getValue().getAsBoolean().get() || v.getValue().getAsAggregated().isPresent()) 26 | .mapToDouble(v -> v.getValue().getAsAggregated().isPresent() ? 27 | v.getValue().getAsAggregated().get().getAsFloat().get() : 28 | 1) 29 | .sum()); 30 | } 31 | 32 | @Override 33 | public Class returnType() { 34 | return FloatValue.class; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/value/FloatValue.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.value; 2 | 3 | import java.time.Instant; 4 | import java.util.Optional; 5 | 6 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 7 | import ch.cern.exdemon.metrics.defined.equation.var.VariableStatuses; 8 | import lombok.EqualsAndHashCode; 9 | 10 | @EqualsAndHashCode(callSuper = false) 11 | public class FloatValue extends Value implements ValueComputable { 12 | 13 | private static final long serialVersionUID = 6026199196915653369L; 14 | 15 | private float num; 16 | 17 | public FloatValue(double value) { 18 | this.num = (float) value; 19 | } 20 | 21 | public FloatValue(float value) { 22 | this.num = value; 23 | } 24 | 25 | @Override 26 | public FloatValue compute(VariableStatuses store, Instant time) { 27 | return new FloatValue(num); 28 | } 29 | 30 | @Override 31 | public Optional getAsFloat() { 32 | return Optional.of(this.num); 33 | } 34 | 35 | public static FloatValue from(String value_string) { 36 | return new FloatValue(Float.parseFloat(value_string)); 37 | } 38 | 39 | @Override 40 | public Class returnType() { 41 | return FloatValue.class; 42 | } 43 | 44 | @Override 45 | public String toString() { 46 | return Float.toString(num); 47 | } 48 | 49 | @Override 50 | public String getSource() { 51 | if (source == null) 52 | return toString(); 53 | else 54 | return super.toString(); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/var/agg/MaxAggregation.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var.agg; 2 | 3 | import java.time.Instant; 4 | import java.util.Collection; 5 | 6 | import ch.cern.exdemon.components.RegisterComponentType; 7 | import ch.cern.exdemon.metrics.DatedValue; 8 | import ch.cern.exdemon.metrics.value.ExceptionValue; 9 | import ch.cern.exdemon.metrics.value.FloatValue; 10 | import ch.cern.exdemon.metrics.value.Value; 11 | 12 | @RegisterComponentType("max") 13 | public class MaxAggregation extends Aggregation { 14 | 15 | private static final long serialVersionUID = 8713765353223035040L; 16 | 17 | public MaxAggregation() { 18 | } 19 | 20 | @Override 21 | public Class inputType() { 22 | return FloatValue.class; 23 | } 24 | 25 | @Override 26 | public Value aggregate(Collection values, Instant time) { 27 | if(values.isEmpty()) 28 | return new ExceptionValue("no values"); 29 | 30 | return new FloatValue(values.stream().mapToDouble(v -> v.getValue().getAsAggregated().isPresent() ? 31 | v.getValue().getAsAggregated().get().getAsFloat().get() : 32 | v.getValue().getAsFloat().get()) 33 | .max().getAsDouble()); 34 | } 35 | 36 | @Override 37 | public Class returnType() { 38 | return FloatValue.class; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/var/agg/MinAggregation.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var.agg; 2 | 3 | import java.time.Instant; 4 | import java.util.Collection; 5 | 6 | import ch.cern.exdemon.components.RegisterComponentType; 7 | import ch.cern.exdemon.metrics.DatedValue; 8 | import ch.cern.exdemon.metrics.value.ExceptionValue; 9 | import ch.cern.exdemon.metrics.value.FloatValue; 10 | import ch.cern.exdemon.metrics.value.Value; 11 | 12 | @RegisterComponentType("min") 13 | public class MinAggregation extends Aggregation { 14 | 15 | private static final long serialVersionUID = 8713765353223035040L; 16 | 17 | public MinAggregation() { 18 | } 19 | 20 | @Override 21 | public Class inputType() { 22 | return FloatValue.class; 23 | } 24 | 25 | @Override 26 | public Value aggregate(Collection values, Instant time) { 27 | if(values.isEmpty()) 28 | return new ExceptionValue("no values"); 29 | 30 | return new FloatValue(values.stream().mapToDouble(v -> v.getValue().getAsAggregated().isPresent() ? 31 | v.getValue().getAsAggregated().get().getAsFloat().get() : 32 | v.getValue().getAsFloat().get()) 33 | .min().getAsDouble()); 34 | } 35 | 36 | @Override 37 | public Class returnType() { 38 | return FloatValue.class; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/schema/MetricSchemasF.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.schema; 2 | 3 | import java.util.Iterator; 4 | import java.util.Map; 5 | import java.util.stream.Stream; 6 | 7 | import org.apache.spark.api.java.function.FlatMapFunction; 8 | 9 | import ch.cern.exdemon.components.ComponentsCatalog; 10 | import ch.cern.exdemon.components.Component.Type; 11 | import ch.cern.exdemon.json.JSON; 12 | import ch.cern.exdemon.metrics.Metric; 13 | import ch.cern.properties.Properties; 14 | 15 | public class MetricSchemasF implements FlatMapFunction { 16 | 17 | private static final long serialVersionUID = 116123198242814348L; 18 | 19 | private String sourceID; 20 | 21 | private Properties componentsSourceProperties; 22 | 23 | public MetricSchemasF(Properties componentsSourceProperties, String sourceId) { 24 | this.sourceID = sourceId; 25 | this.componentsSourceProperties = componentsSourceProperties; 26 | } 27 | 28 | @Override 29 | public Iterator call(String jsonString) throws Exception { 30 | ComponentsCatalog.init(componentsSourceProperties); 31 | 32 | JSON jsonObject = new JSON(jsonString); 33 | 34 | Map schemas = ComponentsCatalog.get(Type.SCHEMA); 35 | 36 | Stream metrics = schemas.values().stream() 37 | .filter(schema -> schema.containsSource(sourceID)) 38 | .flatMap(schema -> schema.call(jsonObject).stream()); 39 | 40 | return metrics.map(metric -> { 41 | metric.getAttributes().put("$source", sourceID); 42 | return metric; 43 | }).iterator(); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/spark/Batches.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | import java.util.LinkedList; 6 | import java.util.List; 7 | 8 | public class Batches implements Serializable{ 9 | 10 | private static final long serialVersionUID = 7460188812026964754L; 11 | 12 | private List> batches = new ArrayList<>(); 13 | 14 | public void add(int batch, T element) { 15 | if(batches.size() <= batch) 16 | batches.add(batch, new LinkedList<>()); 17 | 18 | batches.get(batch).add(element); 19 | } 20 | 21 | public int size() { 22 | return batches.size(); 23 | } 24 | 25 | public List get(int batch) { 26 | return batches.get(batch); 27 | } 28 | 29 | public void addBatch(List batch) { 30 | batches.add(batches.size(), batch); 31 | } 32 | 33 | @Override 34 | public boolean equals(Object obj) { 35 | if (this == obj) 36 | return true; 37 | if (obj == null) 38 | return false; 39 | if (getClass() != obj.getClass()) 40 | return false; 41 | @SuppressWarnings("rawtypes") 42 | Batches other = (Batches) obj; 43 | if (batches == null) { 44 | if (other.batches != null) 45 | return false; 46 | } else if (!batches.equals(other.batches)) 47 | return false; 48 | return true; 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return "Batches [batches=" + batches + "]"; 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/var/PropertiesVariable.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var; 2 | 3 | import java.time.Instant; 4 | import java.util.Map; 5 | import java.util.Optional; 6 | 7 | import ch.cern.exdemon.components.ConfigurationResult; 8 | import ch.cern.exdemon.metrics.Metric; 9 | import ch.cern.exdemon.metrics.value.PropertiesValue; 10 | import ch.cern.exdemon.metrics.value.Value; 11 | import ch.cern.properties.Properties; 12 | 13 | public class PropertiesVariable extends Variable { 14 | 15 | private Properties properties; 16 | 17 | public PropertiesVariable(String name, Map variables, Properties variablesProperties) { 18 | super(name, variables, variablesProperties); 19 | } 20 | 21 | @Override 22 | public ConfigurationResult config(Properties properties, Optional> typeOpt) { 23 | ConfigurationResult confResult = super.config(properties, typeOpt); 24 | 25 | properties.remove("delay"); 26 | this.properties = properties; 27 | 28 | return confResult; 29 | } 30 | 31 | @Override 32 | public Value compute(Optional storeOpt, Instant time) { 33 | PropertiesValue value = new PropertiesValue(name, properties); 34 | value.setSource("props(" + name + ")"); 35 | 36 | return value; 37 | } 38 | 39 | @Override 40 | public Class returnType() { 41 | return PropertiesValue.class; 42 | } 43 | 44 | @Override 45 | public boolean test(Metric t) { 46 | return false; 47 | } 48 | 49 | @Override 50 | public String toString() { 51 | return "props(" + name + ")"; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/analysis/types/htm/PersistableKryoSerializer.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.analysis.types.htm; 2 | 3 | import java.io.IOException; 4 | import java.io.OutputStream; 5 | 6 | import org.apache.log4j.Logger; 7 | import org.numenta.nupic.model.Persistable; 8 | import org.numenta.nupic.network.Persistence; 9 | import org.numenta.nupic.network.PersistenceAPI; 10 | 11 | import com.esotericsoftware.kryo.Kryo; 12 | import com.esotericsoftware.kryo.Serializer; 13 | import com.esotericsoftware.kryo.io.Input; 14 | import com.esotericsoftware.kryo.io.Output; 15 | 16 | public class PersistableKryoSerializer extends Serializer { 17 | 18 | private transient final static Logger LOG = Logger.getLogger(PersistableKryoSerializer.class.getName()); 19 | 20 | private static PersistenceAPI persistence = Persistence.get(); 21 | 22 | @Override 23 | public void write(Kryo kryo, Output output, Persistable persistable) { 24 | persistable.preSerialize(); 25 | 26 | byte[] bytes = persistence.serializer().serialize(persistable); 27 | 28 | OutputStream os = output.getOutputStream(); 29 | try { 30 | os.write(bytes); 31 | } catch (IOException e) { 32 | LOG.error("Serialization error", e); 33 | } 34 | } 35 | 36 | @Override 37 | public Persistable read(Kryo kryo, Input input, Class clazz) { 38 | byte[] bytes = input.getBuffer(); 39 | 40 | Persistable persistable = persistence.read(bytes); 41 | 42 | return persistable; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/bool/IfBoolFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.bool; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.Function; 7 | import ch.cern.exdemon.metrics.defined.equation.functions.FunctionCaller; 8 | import ch.cern.exdemon.metrics.value.BooleanValue; 9 | import ch.cern.exdemon.metrics.value.Value; 10 | 11 | public class IfBoolFunc extends Function { 12 | 13 | public static String REPRESENTATION = "if_bool"; 14 | 15 | public static Class[] argumentTypes = types(BooleanValue.class, BooleanValue.class, BooleanValue.class); 16 | 17 | public IfBoolFunc(ValueComputable... arguments) throws ParseException { 18 | super(REPRESENTATION, argumentTypes, arguments); 19 | } 20 | 21 | @Override 22 | public Class returnType() { 23 | return BooleanValue.class; 24 | } 25 | 26 | @Override 27 | protected Value compute(Value... values) { 28 | return values[0].getAsBoolean().get() ? values[1] : values[2]; 29 | } 30 | 31 | public static class Caller implements FunctionCaller{ 32 | 33 | @Override 34 | public String getFunctionRepresentation() { 35 | return REPRESENTATION; 36 | } 37 | 38 | @Override 39 | public Class[] getArgumentTypes() { 40 | return argumentTypes; 41 | } 42 | 43 | @Override 44 | public ValueComputable call(ValueComputable... arguments) throws ParseException { 45 | return new IfBoolFunc(arguments); 46 | } 47 | 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/var/agg/SumAggregation.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var.agg; 2 | 3 | import java.time.Instant; 4 | import java.util.Collection; 5 | 6 | import ch.cern.exdemon.components.RegisterComponentType; 7 | import ch.cern.exdemon.metrics.DatedValue; 8 | import ch.cern.exdemon.metrics.value.ExceptionValue; 9 | import ch.cern.exdemon.metrics.value.FloatValue; 10 | import ch.cern.exdemon.metrics.value.Value; 11 | 12 | @RegisterComponentType("sum") 13 | public class SumAggregation extends Aggregation { 14 | 15 | private static final long serialVersionUID = 8713765353223035040L; 16 | 17 | public SumAggregation() { 18 | } 19 | 20 | @Override 21 | public Class inputType() { 22 | return FloatValue.class; 23 | } 24 | 25 | @Override 26 | public Value aggregate(Collection values, Instant time) { 27 | if(values.isEmpty()) 28 | return new ExceptionValue("no values"); 29 | 30 | return new FloatValue(values.stream() 31 | .map(v -> v.getValue().getAsAggregated().isPresent() ? 32 | v.getValue().getAsAggregated().get().getAsFloat().get() : 33 | v.getValue().getAsFloat().get()) 34 | .mapToDouble(d -> d) 35 | .sum()); 36 | } 37 | 38 | @Override 39 | public Class returnType() { 40 | return FloatValue.class; 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/ComputeDefinedMetricKeysF.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined; 2 | 3 | import java.util.Iterator; 4 | import java.util.Map; 5 | 6 | import org.apache.spark.api.java.function.PairFlatMapFunction; 7 | 8 | import ch.cern.exdemon.components.ComponentsCatalog; 9 | import ch.cern.exdemon.components.Component.Type; 10 | import ch.cern.exdemon.metrics.Metric; 11 | import ch.cern.properties.Properties; 12 | import scala.Tuple2; 13 | 14 | public class ComputeDefinedMetricKeysF implements PairFlatMapFunction { 15 | 16 | private static final long serialVersionUID = -2525624532462429053L; 17 | 18 | private Properties propertiesSourceProps; 19 | 20 | public ComputeDefinedMetricKeysF(Properties propertiesSourceProps) { 21 | this.propertiesSourceProps = propertiesSourceProps; 22 | } 23 | 24 | @Override 25 | public Iterator> call(Metric metric) throws Exception { 26 | ComponentsCatalog.init(propertiesSourceProps); 27 | 28 | Map definedMetrics = ComponentsCatalog.get(Type.METRIC); 29 | 30 | return definedMetrics.values().stream() 31 | .filter(definedMetric -> definedMetric.testIfApplyForAnyVariable(metric)) 32 | .map(definedMetric -> new Tuple2<>(definedMetric.getId(), definedMetric.getGroupByAttributes(metric.getAttributes()))) 33 | .filter(pair -> pair._2.isPresent()) 34 | .map(pair -> new DefinedMetricStatuskey(pair._1, pair._2.get())) 35 | .map(ids -> new Tuple2(ids, metric)) 36 | .iterator(); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/spark/status/storage/manager/IDStatusKeyFilterTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status.storage.manager; 2 | 3 | import static org.junit.Assert.assertFalse; 4 | import static org.junit.Assert.assertTrue; 5 | 6 | import java.util.HashMap; 7 | 8 | import org.junit.Test; 9 | 10 | import ch.cern.exdemon.metrics.defined.DefinedMetricStatuskey; 11 | import ch.cern.exdemon.monitor.MonitorStatusKey; 12 | import ch.cern.exdemon.monitor.trigger.TriggerStatusKey; 13 | import ch.cern.spark.status.TestStatus; 14 | import scala.Tuple2; 15 | 16 | public class IDStatusKeyFilterTest { 17 | 18 | @Test 19 | public void filterByID() throws Exception { 20 | IDStatusKeyFilter filter = new IDStatusKeyFilter("dm2"); 21 | 22 | assertFalse(filter.call(new Tuple2<>(new DefinedMetricStatuskey("dm1", new HashMap<>()), new TestStatus(1)))); 23 | assertTrue(filter.call(new Tuple2<>(new DefinedMetricStatuskey("dm2", new HashMap<>()), new TestStatus(1)))); 24 | assertFalse(filter.call(new Tuple2<>(new MonitorStatusKey("m1", new HashMap<>()), new TestStatus(1)))); 25 | assertTrue(filter.call(new Tuple2<>(new MonitorStatusKey("dm2", new HashMap<>()), new TestStatus(1)))); 26 | assertFalse(filter.call(new Tuple2<>(new TriggerStatusKey("m1", "n1", new HashMap<>()), new TestStatus(1)))); 27 | 28 | filter = new IDStatusKeyFilter("dm2:2"); 29 | 30 | assertFalse(filter.call(new Tuple2<>(new TriggerStatusKey("dm1", "2", new HashMap<>()), new TestStatus(1)))); 31 | assertTrue(filter.call(new Tuple2<>(new TriggerStatusKey("dm2", "2", new HashMap<>()), new TestStatus(1)))); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/SparkKryoRegistrator.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon; 2 | 3 | import org.apache.spark.serializer.KryoRegistrator; 4 | import org.numenta.nupic.algorithms.AnomalyLikelihood; 5 | import org.numenta.nupic.model.Persistable; 6 | import org.numenta.nupic.network.Network; 7 | 8 | import com.esotericsoftware.kryo.Kryo; 9 | 10 | import ch.cern.exdemon.metrics.ValueHistory; 11 | import ch.cern.exdemon.metrics.defined.DefinedMetricStatuskey; 12 | import ch.cern.exdemon.metrics.defined.equation.var.VariableStatuses; 13 | import ch.cern.exdemon.metrics.defined.equation.var.agg.AggregationValues; 14 | import ch.cern.exdemon.monitor.MonitorStatusKey; 15 | import ch.cern.exdemon.monitor.analysis.types.htm.PersistableKryoSerializer; 16 | import ch.cern.exdemon.monitor.trigger.TriggerStatus; 17 | import ch.cern.exdemon.monitor.trigger.TriggerStatusKey; 18 | 19 | public class SparkKryoRegistrator implements KryoRegistrator { 20 | 21 | @Override 22 | public void registerClasses(Kryo kryo) { 23 | kryo.register(DefinedMetricStatuskey.class); 24 | kryo.register(DefinedMetricStatuskey.class); 25 | kryo.register(MonitorStatusKey.class); 26 | kryo.register(TriggerStatusKey.class); 27 | kryo.register(TriggerStatus.class); 28 | kryo.register(ValueHistory.class); 29 | kryo.register(VariableStatuses.class); 30 | kryo.register(AggregationValues.class); 31 | 32 | kryo.register(Persistable.class, new PersistableKryoSerializer()); 33 | kryo.register(Network.class, new PersistableKryoSerializer()); 34 | kryo.register(AnomalyLikelihood.class, new PersistableKryoSerializer()); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/num/IfFloatFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.num; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.Function; 7 | import ch.cern.exdemon.metrics.defined.equation.functions.FunctionCaller; 8 | import ch.cern.exdemon.metrics.value.BooleanValue; 9 | import ch.cern.exdemon.metrics.value.FloatValue; 10 | import ch.cern.exdemon.metrics.value.Value; 11 | 12 | public class IfFloatFunc extends Function { 13 | 14 | public static String REPRESENTATION = "if_float"; 15 | 16 | public static Class[] argumentTypes = types(BooleanValue.class, FloatValue.class, FloatValue.class); 17 | 18 | public IfFloatFunc(ValueComputable... arguments) 19 | throws ParseException { 20 | super(REPRESENTATION, argumentTypes, arguments); 21 | } 22 | 23 | @Override 24 | protected Value compute(Value... values) { 25 | return values[0].getAsBoolean().get() ? values[1] : values[2]; 26 | } 27 | 28 | @Override 29 | public Class returnType() { 30 | return FloatValue.class; 31 | } 32 | 33 | public static class Caller implements FunctionCaller{ 34 | 35 | @Override 36 | public String getFunctionRepresentation() { 37 | return REPRESENTATION; 38 | } 39 | 40 | @Override 41 | public Class[] getArgumentTypes() { 42 | return argumentTypes; 43 | } 44 | 45 | @Override 46 | public ValueComputable call(ValueComputable... arguments) throws ParseException { 47 | return new IfFloatFunc(arguments); 48 | } 49 | 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/json/JSONParser.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.json; 2 | 3 | import java.lang.reflect.Type; 4 | import java.time.Instant; 5 | import java.time.ZoneOffset; 6 | import java.time.ZonedDateTime; 7 | import java.time.format.DateTimeFormatter; 8 | 9 | import com.google.gson.Gson; 10 | import com.google.gson.GsonBuilder; 11 | import com.google.gson.JsonElement; 12 | import com.google.gson.JsonPrimitive; 13 | import com.google.gson.JsonSerializationContext; 14 | import com.google.gson.JsonSerializer; 15 | 16 | public class JSONParser { 17 | 18 | public static String TIMESTAMP_OUTPUT_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ"; 19 | 20 | private final transient static Gson gson = new GsonBuilder() 21 | .registerTypeAdapter(Instant.class, new JsonSerializer() { 22 | DateTimeFormatter formatter = DateTimeFormatter.ofPattern(TIMESTAMP_OUTPUT_FORMAT); 23 | 24 | @Override 25 | public JsonElement serialize(Instant instant, Type type, JsonSerializationContext context) { 26 | return new JsonPrimitive(ZonedDateTime.ofInstant(instant , ZoneOffset.systemDefault()).format(formatter)); 27 | } 28 | }) 29 | .create(); 30 | 31 | public static JSON parse(T javaObject) { 32 | return javaObject == null ? null : new JSON(gson.toJson(javaObject)); 33 | } 34 | 35 | public static boolean isValid(String jsonInString) { 36 | if(jsonInString == null) 37 | return false; 38 | 39 | try { 40 | gson.fromJson(jsonInString, Object.class); 41 | return true; 42 | } catch(com.google.gson.JsonSyntaxException ex) { 43 | return false; 44 | } 45 | } 46 | 47 | } -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/functions/string/IfStringFunc.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.functions.string; 2 | 3 | import java.text.ParseException; 4 | 5 | import ch.cern.exdemon.metrics.defined.equation.ValueComputable; 6 | import ch.cern.exdemon.metrics.defined.equation.functions.Function; 7 | import ch.cern.exdemon.metrics.defined.equation.functions.FunctionCaller; 8 | import ch.cern.exdemon.metrics.value.BooleanValue; 9 | import ch.cern.exdemon.metrics.value.StringValue; 10 | import ch.cern.exdemon.metrics.value.Value; 11 | 12 | public class IfStringFunc extends Function{ 13 | 14 | public static String REPRESENTATION = "if_string"; 15 | 16 | public static Class[] argumentTypes = types(BooleanValue.class, StringValue.class, StringValue.class); 17 | 18 | public IfStringFunc(ValueComputable... arguments) 19 | throws ParseException { 20 | super(REPRESENTATION, argumentTypes, arguments); 21 | } 22 | 23 | @Override 24 | protected Value compute(Value... values) { 25 | return values[0].getAsBoolean().get() ? values[1] : values[2]; 26 | } 27 | 28 | @Override 29 | public Class returnType() { 30 | return StringValue.class; 31 | } 32 | 33 | public static class Caller implements FunctionCaller{ 34 | 35 | @Override 36 | public String getFunctionRepresentation() { 37 | return REPRESENTATION; 38 | } 39 | 40 | @Override 41 | public Class[] getArgumentTypes() { 42 | return argumentTypes; 43 | } 44 | 45 | @Override 46 | public ValueComputable call(ValueComputable... arguments) throws ParseException { 47 | return new IfStringFunc(arguments); 48 | } 49 | 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/var/agg/AvgAggregation.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var.agg; 2 | 3 | import java.time.Instant; 4 | import java.util.Collection; 5 | 6 | import ch.cern.exdemon.components.RegisterComponentType; 7 | import ch.cern.exdemon.metrics.DatedValue; 8 | import ch.cern.exdemon.metrics.value.ExceptionValue; 9 | import ch.cern.exdemon.metrics.value.FloatValue; 10 | import ch.cern.exdemon.metrics.value.Value; 11 | import lombok.EqualsAndHashCode; 12 | 13 | @RegisterComponentType("avg") 14 | @EqualsAndHashCode(callSuper=false) 15 | public class AvgAggregation extends Aggregation { 16 | 17 | private static final long serialVersionUID = 8713765353223035040L; 18 | 19 | public AvgAggregation() { 20 | } 21 | 22 | @Override 23 | public Class inputType() { 24 | return FloatValue.class; 25 | } 26 | 27 | @Override 28 | public Value aggregate(Collection values, Instant time) { 29 | if(values.isEmpty()) 30 | return new ExceptionValue("no values"); 31 | 32 | return new FloatValue(values.stream() 33 | .mapToDouble(v -> v.getValue().getAsAggregated().isPresent() ? 34 | v.getValue().getAsAggregated().get().getAsFloat().get() : 35 | v.getValue().getAsFloat().get()) 36 | .average().getAsDouble()); 37 | } 38 | 39 | @Override 40 | public Class returnType() { 41 | return FloatValue.class; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/exdemon/monitor/analysis/types/htm/ResultsToFileWriter.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.analysis.types.htm; 2 | 3 | import java.io.FileNotFoundException; 4 | import java.io.PrintWriter; 5 | 6 | import com.esotericsoftware.minlog.Log; 7 | 8 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult; 9 | 10 | public class ResultsToFileWriter { 11 | //private String filename; 12 | PrintWriter writer; 13 | 14 | public ResultsToFileWriter(String filename) { 15 | try { 16 | this.writer = new PrintWriter(filename); 17 | } catch (FileNotFoundException e) { 18 | Log.error("can't open the file: "+filename); 19 | e.printStackTrace(); 20 | } 21 | } 22 | 23 | public void write(AnalysisResult results) { 24 | StringBuilder s = new StringBuilder(); 25 | s.append(results.getAnalysis_timestamp()).append(",") 26 | .append(results.getAnalysisParams().get("anomaly.likelihood")).append(",") 27 | .append(results.getAnalysisParams().get("anomaly.score")).append(",") 28 | .append(results.getStatus() == AnalysisResult.Status.WARNING ? 1 : 0).append(",") 29 | .append(results.getStatus() == AnalysisResult.Status.ERROR ? 1 : 0); 30 | writer.println(s.toString()); 31 | writer.flush(); 32 | } 33 | 34 | public void writeHeader() { 35 | writer.println(getHeader()); 36 | } 37 | 38 | public String getHeader() { 39 | StringBuilder header = new StringBuilder(); 40 | header.append("timestamp").append(",") 41 | .append("anomaly.likelihood").append(",") 42 | .append("anomaly.score").append(",") 43 | .append("warning").append(",") 44 | .append("error"); 45 | return header.toString(); 46 | } 47 | 48 | public void close() { 49 | writer.close(); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/utils/StructTypeUtils.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.utils; 2 | 3 | import java.util.List; 4 | import java.util.Map; 5 | import java.util.Map.Entry; 6 | import java.util.stream.Collectors; 7 | 8 | import org.apache.spark.sql.types.DataType; 9 | import org.apache.spark.sql.types.StructType; 10 | 11 | import ch.cern.utils.Pair; 12 | 13 | public class StructTypeUtils { 14 | 15 | public static StructType create(Map keys) { 16 | StructType struct = new StructType(); 17 | 18 | for (Entry key : keys.entrySet()) 19 | if(!key.getKey().contains(".")) 20 | struct = struct.add(key.getKey(), key.getValue()); 21 | 22 | Map>> groups = keys.entrySet().stream() 23 | .filter(key -> key.getKey().contains(".")) 24 | .collect(Collectors.groupingBy(e -> e.getKey().substring(0, e.getKey().indexOf(".")))); 25 | 26 | for (Entry>> group : groups.entrySet()) { 27 | Map groupKeys = group.getValue().stream() 28 | .map(e -> new Pair<>(e.getKey().substring(e.getKey().indexOf(".") + 1), e.getValue())) 29 | .collect(Collectors.toMap(Pair::first, Pair::second)); 30 | 31 | struct = struct.add(group.getKey(), create(groupKeys)); 32 | } 33 | 34 | return struct; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/trigger/action/actuator/RunActuatorsF.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.trigger.action.actuator; 2 | 3 | import java.util.Optional; 4 | 5 | import org.apache.log4j.Logger; 6 | import org.apache.spark.api.java.function.VoidFunction; 7 | 8 | import ch.cern.exdemon.components.ComponentsCatalog; 9 | import ch.cern.exdemon.components.Component.Type; 10 | import ch.cern.exdemon.monitor.trigger.action.Action; 11 | import ch.cern.properties.Properties; 12 | 13 | public class RunActuatorsF implements VoidFunction { 14 | 15 | private static final long serialVersionUID = -7248445923365556277L; 16 | 17 | private transient final static Logger LOG = Logger.getLogger(RunActuatorsF.class.getName()); 18 | 19 | private Properties componentsSourceProperties; 20 | 21 | public RunActuatorsF(Properties componentsSourceProperties) { 22 | this.componentsSourceProperties = componentsSourceProperties; 23 | } 24 | 25 | @Override 26 | public void call(Action action) throws Exception { 27 | ComponentsCatalog.init(componentsSourceProperties); 28 | 29 | action.getActuatorIDs().stream().forEach(actuatorID -> { 30 | try { 31 | Optional actuatorOpt = ComponentsCatalog.get(Type.ACTUATOR, actuatorID); 32 | 33 | if(actuatorOpt.isPresent()) 34 | actuatorOpt.get().run(action); 35 | else 36 | LOG.error("Action " + action + " could not be run because actuator with id " + actuatorID + " does not exist."); 37 | } catch (Exception e) { 38 | LOG.error("Actuator ID=" +actuatorID + ": problem when running action=" + action, e); 39 | } 40 | }); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/exdemon/components/source/types/ZookeeperComponentsSourceIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.components.source.types; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.curator.test.TestingServer; 6 | import org.junit.After; 7 | import org.junit.Before; 8 | import org.junit.Test; 9 | 10 | import ch.cern.properties.Properties; 11 | 12 | public class ZookeeperComponentsSourceIntegrationTest { 13 | 14 | private TestingServer zkTestServer; 15 | private ZookeeperComponentsSource source; 16 | private Properties sourceProperties; 17 | 18 | @Before 19 | public void startZookeeper() throws Exception { 20 | zkTestServer = new TestingServer(2182); 21 | 22 | source = new ZookeeperComponentsSource(); 23 | sourceProperties = new Properties(); 24 | } 25 | 26 | @Test 27 | public void initialize() throws Exception { 28 | sourceProperties.setProperty("connection_string", "localhost:2182/"); 29 | source.config(sourceProperties); 30 | 31 | source.initialize(); 32 | } 33 | 34 | @Test(expected=IOException.class) 35 | public void noRootPath() throws Exception { 36 | sourceProperties.setProperty("connection_string", "localhost:2182/does-not-exist"); 37 | source.config(sourceProperties); 38 | 39 | source.initialize(); 40 | } 41 | 42 | @Test(expected=IOException.class) 43 | public void notAbleToConnect() throws Exception { 44 | sourceProperties.setProperty("connection_string", "othermachine:2182/"); 45 | source.config(sourceProperties); 46 | 47 | source.initialize(); 48 | } 49 | 50 | @After 51 | public void shutDown() throws IOException, InterruptedException { 52 | zkTestServer.close(); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/exdemon/monitor/analysis/AnalysisTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.analysis; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | import static org.junit.Assert.assertSame; 5 | 6 | import java.time.Instant; 7 | 8 | import org.junit.Test; 9 | 10 | import ch.cern.exdemon.metrics.Metric; 11 | import ch.cern.exdemon.metrics.value.BooleanValue; 12 | import ch.cern.exdemon.metrics.value.ExceptionValue; 13 | import ch.cern.exdemon.metrics.value.FloatValue; 14 | import ch.cern.exdemon.metrics.value.StringValue; 15 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult; 16 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult.Status; 17 | 18 | public class AnalysisTest { 19 | 20 | private Analysis analysis = new Analysis() { 21 | private static final long serialVersionUID = 2630885101669119453L; 22 | 23 | @Override 24 | protected AnalysisResult process(Metric metric) { 25 | return AnalysisResult.buildWithStatus(Status.OK, ""); 26 | } 27 | }; 28 | 29 | @Test 30 | public void shouldGenerateExceptionAnalysisResultFromMetricInException() { 31 | Metric metric = new Metric(Instant.now(), new ExceptionValue(""), null); 32 | AnalysisResult result = analysis.apply(metric); 33 | 34 | assertEquals(Status.EXCEPTION, result.getStatus()); 35 | assertSame(metric, result.getAnalyzed_metric()); 36 | } 37 | 38 | @Test 39 | public void shouldProcessMetricsWhichAreNotException() { 40 | assertEquals(Status.OK, analysis.apply(new Metric(Instant.now(), new StringValue(""), null)).getStatus()); 41 | assertEquals(Status.OK, analysis.apply(new Metric(Instant.now(), new FloatValue(0f), null)).getStatus()); 42 | assertEquals(Status.OK, analysis.apply(new Metric(Instant.now(), new BooleanValue(true), null)).getStatus()); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/utils/StringUtilsTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.utils; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | 5 | import java.util.Optional; 6 | 7 | import org.junit.Test; 8 | 9 | public class StringUtilsTest { 10 | 11 | @Test 12 | public void getLastCharacter(){ 13 | assertEquals(Optional.of('4'), StringUtils.getLastCharacter("1234")); 14 | assertEquals(Optional.of('m'), StringUtils.getLastCharacter("1234 m")); 15 | assertEquals(Optional.of(' '), StringUtils.getLastCharacter("1234 ")); 16 | assertEquals(Optional.empty(), StringUtils.getLastCharacter("")); 17 | } 18 | 19 | @Test 20 | public void countLines(){ 21 | assertEquals(1, StringUtils.countLines("1234")); 22 | assertEquals(1, StringUtils.countLines("1234\n")); 23 | assertEquals(2, StringUtils.countLines("1234\n5678")); 24 | assertEquals(2, StringUtils.countLines("1234\n5678\n")); 25 | } 26 | 27 | @Test 28 | public void headLines(){ 29 | String input = "1\n" 30 | + "2\n" 31 | + "3\n" 32 | + "4"; 33 | 34 | String expected = "1\n" 35 | + "2"; 36 | 37 | assertEquals(expected, StringUtils.headLines(input, 2)); 38 | } 39 | 40 | @Test 41 | public void headLinesLessLinesThanMax(){ 42 | String input = "1\n" 43 | + "2\n" 44 | + "3\n" 45 | + "4"; 46 | 47 | assertEquals(input, StringUtils.headLines(input, 20)); 48 | } 49 | 50 | @Test 51 | public void headLinesWithoutEOL(){ 52 | String input = "12345"; 53 | 54 | String expected = "12345"; 55 | 56 | assertEquals(expected, StringUtils.headLines(input, 2)); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/status/storage/manager/ToStringPatternStatusKeyFilter.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status.storage.manager; 2 | 3 | import java.util.regex.Pattern; 4 | 5 | import org.apache.spark.api.java.function.Function; 6 | 7 | import ch.cern.spark.status.StatusKey; 8 | import ch.cern.spark.status.StatusValue; 9 | import lombok.ToString; 10 | import scala.Tuple2; 11 | 12 | @ToString 13 | public class ToStringPatternStatusKeyFilter implements Function, Boolean> { 14 | 15 | private static final long serialVersionUID = -605577220519629679L; 16 | 17 | private Pattern pattern; 18 | 19 | public ToStringPatternStatusKeyFilter(String pattern) { 20 | if(pattern != null) 21 | this.pattern = Pattern.compile(pattern); 22 | } 23 | 24 | @Override 25 | public Boolean call(Tuple2 tuple) throws Exception { 26 | if(pattern == null) 27 | return true; 28 | 29 | if(tuple._1 == null) 30 | return false; 31 | 32 | return pattern.matcher(tuple._1.toString()).matches(); 33 | } 34 | 35 | @Override 36 | public boolean equals(Object obj) { 37 | if (this == obj) 38 | return true; 39 | if (obj == null) 40 | return false; 41 | if (getClass() != obj.getClass()) 42 | return false; 43 | ToStringPatternStatusKeyFilter other = (ToStringPatternStatusKeyFilter) obj; 44 | if (pattern == null) { 45 | if (other.pattern != null) 46 | return false; 47 | } else if (!pattern.pattern().equals(other.pattern.pattern())) 48 | return false; 49 | return true; 50 | } 51 | 52 | @Override 53 | public int hashCode() { 54 | final int prime = 31; 55 | int result = 1; 56 | result = prime * result + ((pattern == null) ? 0 : pattern.pattern().hashCode()); 57 | return result; 58 | } 59 | 60 | 61 | 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/trigger/types/StatusesTrigger.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.trigger.types; 2 | 3 | import java.time.Instant; 4 | import java.util.Optional; 5 | import java.util.Set; 6 | import java.util.stream.Collectors; 7 | import java.util.stream.Stream; 8 | 9 | import ch.cern.exdemon.components.ConfigurationResult; 10 | import ch.cern.exdemon.components.RegisterComponentType; 11 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult.Status; 12 | import ch.cern.exdemon.monitor.trigger.Trigger; 13 | import ch.cern.properties.Properties; 14 | import lombok.ToString; 15 | 16 | @ToString 17 | @RegisterComponentType("statuses") 18 | public class StatusesTrigger extends Trigger { 19 | 20 | private static final long serialVersionUID = -7890231998987060652L; 21 | 22 | private static final String STATUSES_PARAM = "statuses"; 23 | private Set expectedStatuses; 24 | 25 | @Override 26 | public ConfigurationResult config(Properties properties) { 27 | ConfigurationResult configResult = super.config(properties); 28 | 29 | expectedStatuses = Stream.of(properties.getProperty(STATUSES_PARAM).split("\\s")) 30 | .map(String::trim) 31 | .map(String::toUpperCase) 32 | .map(Status::valueOf) 33 | .collect(Collectors.toSet()); 34 | 35 | return configResult.merge(null, properties.warningsIfNotAllPropertiesUsed()); 36 | } 37 | 38 | @Override 39 | public Optional process(Status status, Instant timestamp) { 40 | if(isExpectedStatus(status)){ 41 | return Optional.of("Metric is in status " + status + "."); 42 | }else{ 43 | return Optional.empty(); 44 | } 45 | } 46 | 47 | private boolean isExpectedStatus(Status status) { 48 | return expectedStatuses.contains(status); 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/exdemon/metrics/defined/equation/var/agg/AvgAgregationTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var.agg; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | 5 | import java.time.Instant; 6 | import java.util.LinkedList; 7 | 8 | import org.junit.Test; 9 | 10 | import ch.cern.exdemon.metrics.DatedValue; 11 | import ch.cern.exdemon.metrics.value.AggregatedValue; 12 | import ch.cern.exdemon.metrics.value.FloatValue; 13 | import ch.cern.exdemon.metrics.value.Value; 14 | 15 | public class AvgAgregationTest { 16 | 17 | private AvgAggregation agg = new AvgAggregation(); 18 | 19 | @Test 20 | public void count() { 21 | LinkedList values = new LinkedList(); 22 | values.add(new DatedValue(Instant.EPOCH, new FloatValue(10))); 23 | values.add(new DatedValue(Instant.EPOCH, new FloatValue(20))); 24 | values.add(new DatedValue(Instant.EPOCH, new FloatValue(30))); 25 | 26 | Value result = agg.aggregateValues(values, Instant.EPOCH); 27 | 28 | assertEquals(20, result.getAsAggregated().get().getAsFloat().get(), 0f); 29 | } 30 | 31 | @Test 32 | public void countWithAggValues() { 33 | LinkedList values = new LinkedList(); 34 | values.add(new DatedValue(Instant.EPOCH, new AggregatedValue(new FloatValue(10)))); 35 | values.add(new DatedValue(Instant.EPOCH, new AggregatedValue(new FloatValue(20)))); 36 | values.add(new DatedValue(Instant.EPOCH, new FloatValue(10))); 37 | values.add(new DatedValue(Instant.EPOCH, new FloatValue(20))); 38 | values.add(new DatedValue(Instant.EPOCH, new FloatValue(30))); 39 | 40 | Value result = agg.aggregateValues(values, Instant.EPOCH); 41 | 42 | assertEquals(18, result.getAsAggregated().get().getAsFloat().get(), 0f); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/Metric.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics; 2 | 3 | import java.io.Serializable; 4 | import java.time.Instant; 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | import java.util.Set; 8 | 9 | import ch.cern.exdemon.metrics.value.FloatValue; 10 | import ch.cern.exdemon.metrics.value.Value; 11 | import lombok.EqualsAndHashCode; 12 | import lombok.Getter; 13 | import lombok.NonNull; 14 | import lombok.Setter; 15 | import lombok.ToString; 16 | 17 | @ToString 18 | @EqualsAndHashCode(callSuper = false) 19 | public class Metric implements Serializable { 20 | 21 | private static final long serialVersionUID = -182236104179624396L; 22 | 23 | @Getter 24 | @Setter 25 | @NonNull 26 | private Map attributes; 27 | 28 | @Getter 29 | @Setter 30 | @NonNull 31 | private Instant timestamp; 32 | 33 | @Getter 34 | @NonNull 35 | private Value value; 36 | 37 | public Metric(Instant timestamp, float value, Map ids) { 38 | this(timestamp, new FloatValue(value), ids); 39 | } 40 | 41 | public Metric(@NonNull Instant timestamp, @NonNull Value value, Map attributes) { 42 | if (attributes == null) 43 | this.attributes = new HashMap(); 44 | else 45 | this.attributes = new HashMap(attributes); 46 | 47 | this.timestamp = timestamp; 48 | this.value = value; 49 | } 50 | 51 | public void addAttribute(@NonNull String key, @NonNull String value) { 52 | attributes.put(key, value); 53 | } 54 | 55 | public void removeAttributes(@NonNull Set keySet) { 56 | keySet.forEach(key -> attributes.remove(key)); 57 | } 58 | 59 | @Override 60 | public Metric clone() { 61 | return new Metric(timestamp, value.clone(), new HashMap<>(attributes)); 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/utils/TimeUtilsTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.utils; 2 | 3 | import java.time.Duration; 4 | 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | import ch.cern.properties.ConfigurationException; 9 | 10 | public class TimeUtilsTest { 11 | 12 | @Test 13 | public void stringWithUnitToSeconds() throws ConfigurationException{ 14 | Assert.assertEquals(3, TimeUtils.parsePeriod("3").getSeconds()); 15 | Assert.assertEquals(-23, TimeUtils.parsePeriod("-23").getSeconds()); 16 | Assert.assertEquals(-34, TimeUtils.parsePeriod("-34s").getSeconds()); 17 | Assert.assertEquals(15 * 60, TimeUtils.parsePeriod("15m").getSeconds()); 18 | Assert.assertEquals(2 * 60 * 60, TimeUtils.parsePeriod("2h").getSeconds()); 19 | } 20 | 21 | @Test 22 | public void durationToString(){ 23 | Assert.assertEquals("0 seconds", TimeUtils.toString(Duration.ofSeconds(0))); 24 | Assert.assertEquals("1 second", TimeUtils.toString(Duration.ofSeconds(1))); 25 | Assert.assertEquals("3 seconds", TimeUtils.toString(Duration.ofSeconds(3))); 26 | Assert.assertEquals("1 minute", TimeUtils.toString(Duration.ofSeconds(60))); 27 | Assert.assertEquals("2 minutes", TimeUtils.toString(Duration.ofSeconds(120))); 28 | Assert.assertEquals("2 minutes and 5 seconds", TimeUtils.toString(Duration.ofSeconds(125))); 29 | Assert.assertEquals("1 hour", TimeUtils.toString(Duration.ofHours(1))); 30 | Assert.assertEquals("2 hours", TimeUtils.toString(Duration.ofHours(2))); 31 | Assert.assertEquals("2 hours and 1 minute", TimeUtils.toString(Duration.ofSeconds(2 * 3600 + 60))); 32 | Assert.assertEquals("2 hours and 6 seconds", TimeUtils.toString(Duration.ofSeconds(2 * 3600 + 6))); 33 | Assert.assertEquals("2 hours, 2 minutes and 5 seconds", TimeUtils.toString(Duration.ofSeconds(2 * 3600 + 125))); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /bin/status: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (C) 2018, CERN 4 | # This software is distributed under the terms of the GNU General Public 5 | # Licence version 3 (GPL Version 3), copied verbatim in the file "LICENSE". 6 | # In applying this license, CERN does not waive the privileges and immunities 7 | # granted to it by virtue of its status as Intergovernmental Organization 8 | # or submit itself to any jurisdiction. 9 | 10 | # Reference: http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in 11 | SOURCE="${BASH_SOURCE[0]}" 12 | while [ -h "$SOURCE" ]; do 13 | DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" 14 | SOURCE="$(readlink "$SOURCE")" 15 | [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" 16 | done 17 | SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" 18 | 19 | HOME=$SCRIPT_DIR/.. 20 | 21 | if [ -e $HOME/conf/exdemon-env.sh ] ; then 22 | . $HOME/conf/exdemon-env.sh 23 | fi 24 | 25 | YARN_CMD=yarn 26 | if [ ! -z "${HADOOP_YARN_HOME}" ]; then 27 | YARN_CMD=$HADOOP_YARN_HOME/bin/$YARN_CMD 28 | fi 29 | 30 | ZOOCNX=`sed -n 's/^spark.streaming.listener.connection_string[ ]=//p' $HOME/conf/exdemon.properties` 31 | ZOOPTH=`sed -n 's/^spark.streaming.listener.path[ ]=//p' $HOME/conf/exdemon.properties` 32 | 33 | APPID=`zookeepercli -servers $ZOOCNX -c get $ZOOPTH/id 2>/tmp/exdemon_status_zk.err | grep application_` 34 | RESULT=$? 35 | if [ $RESULT -ne 0 ]; then 36 | echo Error when getting application ID from Zookeeper 37 | cat /tmp/exdemon_status_zk.err 38 | exit 100 39 | fi 40 | 41 | OUTPUT_FILE=`mktemp` 42 | 43 | $YARN_CMD application -status $APPID > $OUTPUT_FILE 2>&1 44 | RESULT=$? 45 | if [ $RESULT -ne 0 ]; then 46 | echo Error when checking app status in YARN for $APPID 47 | cat $OUTPUT_FILE 48 | 49 | exit 101 50 | fi 51 | 52 | cat $OUTPUT_FILE 53 | 54 | cat $OUTPUT_FILE | grep State | grep "RUNNING\|ACCEPTED" >/dev/null 55 | RESULT=$? 56 | 57 | rm -f $OUTPUT_FILE 58 | 59 | exit $RESULT 60 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/spark/status/storage/JavaStatusSerializer.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status.storage; 2 | 3 | import java.io.ByteArrayInputStream; 4 | import java.io.ByteArrayOutputStream; 5 | import java.io.IOException; 6 | import java.io.ObjectInput; 7 | import java.io.ObjectInputStream; 8 | import java.io.ObjectOutput; 9 | import java.io.ObjectOutputStream; 10 | 11 | import ch.cern.spark.status.StatusKey; 12 | import ch.cern.spark.status.StatusValue; 13 | 14 | public class JavaStatusSerializer implements StatusSerializer { 15 | 16 | private static final long serialVersionUID = -406547771533268077L; 17 | 18 | @Override 19 | public StatusKey toKey(byte[] bytes) throws IOException { 20 | return to(bytes); 21 | } 22 | 23 | @Override 24 | public byte[] fromKey(StatusKey key) throws IOException { 25 | return from(key); 26 | } 27 | 28 | @Override 29 | public StatusValue toValue(byte[] bytes) throws IOException { 30 | return to(bytes); 31 | } 32 | 33 | @Override 34 | public byte[] fromValue(StatusValue value) throws IOException { 35 | return from(value); 36 | } 37 | 38 | @SuppressWarnings("unchecked") 39 | public static T to(byte[] bytes) throws IOException { 40 | ObjectInput in = null; 41 | try { 42 | in = new ObjectInputStream(new ByteArrayInputStream(bytes)); 43 | 44 | return (T) in.readObject(); 45 | } catch (ClassNotFoundException e) { 46 | throw new IOException(e); 47 | } finally { 48 | try { 49 | if (in != null) { 50 | in.close(); 51 | } 52 | } catch (IOException ex) {} 53 | } 54 | } 55 | 56 | public static byte[] from(T key) throws IOException { 57 | ByteArrayOutputStream bos = new ByteArrayOutputStream(); 58 | 59 | try { 60 | ObjectOutput out = new ObjectOutputStream(bos); 61 | out.writeObject(key); 62 | out.flush(); 63 | 64 | return bos.toByteArray(); 65 | } finally { 66 | try { 67 | bos.close(); 68 | } catch (IOException ex) {} 69 | } 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/struct/Metric.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.struct; 2 | 3 | import java.io.Serializable; 4 | import java.sql.Timestamp; 5 | import java.util.Map; 6 | 7 | public class Metric implements Serializable { 8 | 9 | private static final long serialVersionUID = -2090504797500130224L; 10 | 11 | private Map att; 12 | private Timestamp timestamp; 13 | private Value value; 14 | 15 | public Metric() { 16 | } 17 | 18 | public Map getAtt() { 19 | return att; 20 | } 21 | 22 | public void setAttributes(Map att) { 23 | this.att = att; 24 | } 25 | 26 | public Timestamp getTimestamp() { 27 | return timestamp; 28 | } 29 | 30 | public void setTimestamp(Timestamp timestamp) { 31 | this.timestamp = timestamp; 32 | } 33 | 34 | public Value getValue() { 35 | return value; 36 | } 37 | 38 | public void setValue(Value value) { 39 | this.value = value; 40 | } 41 | 42 | public static class Value implements Serializable { 43 | 44 | private static final long serialVersionUID = -23682055119611586L; 45 | 46 | private String str = null; 47 | private Boolean bool = null; 48 | private Double num = null; 49 | 50 | public Value() { 51 | } 52 | 53 | public String getStr() { 54 | return str; 55 | } 56 | 57 | public void setStr(String str) { 58 | this.str = str; 59 | } 60 | 61 | public Boolean getBool() { 62 | return bool; 63 | } 64 | 65 | public void setBool(Boolean bool) { 66 | this.bool = bool; 67 | } 68 | 69 | public Double getNum() { 70 | return num; 71 | } 72 | 73 | public void setNum(Double num) { 74 | this.num = num; 75 | } 76 | 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /doc/users-manual/metrics-filter.md: -------------------------------------------------------------------------------- 1 | # Metric filters 2 | 3 | The filter determine the rules a metric must pass in order to accept the metric. 4 | 5 | You can specify that metrics with timestamp older than a period (comparing with current time), should be filtered out. 6 | ``` 7 | filter.timestamp.expire = (default: not set) 8 | ``` 9 | 10 | For "attribute" parameters, you can negate the condition by placing "!" as first character in the value. That would mean: attribute should not be the specified value or should not match the regular expression. 11 | 12 | You can specify a regular expression or an exact value for the attribute: 13 | ``` 14 | filter.attribute. = [!] 15 | ``` 16 | 17 | You can also specify multi-values between quotes (! at the beginning would negate all): 18 | ``` 19 | filter.attribute. = [!]"" "" "" ... 20 | ``` 21 | 22 | You can also filter by some meta-attributes: 23 | ``` 24 | filter.attribute.$source = 25 | filter.attribute.$schema = 26 | filter.attribute.$defined_metric = 27 | filter.attribute.$monitor = 28 | ``` 29 | 30 | More complex filter can be configured using the "expr" parameter. Regular expressions can be used. 31 | 32 | ``` 33 | filter.expr = 34 | ``` 35 | 36 | You can combine "expr" and "attribute" parameters, all attribute parameters are "and" predicates with "expr". 37 | 38 | An example: 39 | 40 | ``` 41 | # No metrics generated more than 24h ago 42 | filter.timestamp.expire = 24h 43 | # CLUSTER must be "cluster1" 44 | # and HOST must be "host1" or "host2" 45 | # and NOT_VALID must not be defined 46 | filter.expr = "CLUSTER = \"cluster1\" & (HOST = 'host1' | HOST='host2') & NOT_VALID != .*" 47 | # Optionally you can specify more conditions 48 | # and $source must be kafka 49 | filter.attribute.$source = kafka 50 | ``` 51 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/var/agg/DiffAggregation.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var.agg; 2 | 3 | import java.time.Instant; 4 | import java.util.Collection; 5 | import java.util.List; 6 | import java.util.stream.Collectors; 7 | 8 | import ch.cern.exdemon.components.RegisterComponentType; 9 | import ch.cern.exdemon.metrics.DatedValue; 10 | import ch.cern.exdemon.metrics.value.ExceptionValue; 11 | import ch.cern.exdemon.metrics.value.FloatValue; 12 | import ch.cern.exdemon.metrics.value.Value; 13 | 14 | @RegisterComponentType("diff") 15 | public class DiffAggregation extends Aggregation { 16 | 17 | private static final long serialVersionUID = 8713765353223035040L; 18 | 19 | public DiffAggregation() { 20 | } 21 | 22 | @Override 23 | public Class inputType() { 24 | return FloatValue.class; 25 | } 26 | 27 | @Override 28 | public Value aggregate(Collection values, Instant time) { 29 | if(values.size() <= 1) 30 | return new ExceptionValue("diff requires at least two values"); 31 | 32 | List sorted = values.stream() 33 | .sorted() 34 | .collect(Collectors.toList()); 35 | 36 | Value lastValue = sorted.get(sorted.size() - 1).getValue(); 37 | Value previousValue = sorted.get(sorted.size() - 2).getValue(); 38 | 39 | if(previousValue.getAsAggregated().isPresent() || lastValue.getAsAggregated().isPresent()) 40 | return new ExceptionValue("latest values have been summarized, diff cannot be computed."); 41 | 42 | double lastValueD = lastValue.getAsFloat().get(); 43 | double previousValueD = previousValue.getAsFloat().get(); 44 | 45 | return new FloatValue(lastValueD - previousValueD); 46 | } 47 | 48 | @Override 49 | public Class returnType() { 50 | return FloatValue.class; 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/trigger/ComputeTriggerKeysF.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.trigger; 2 | 3 | import java.util.Iterator; 4 | import java.util.LinkedList; 5 | import java.util.Map; 6 | import java.util.Set; 7 | 8 | import org.apache.spark.api.java.function.PairFlatMapFunction; 9 | 10 | import ch.cern.exdemon.components.ComponentsCatalog; 11 | import ch.cern.exdemon.components.Component.Type; 12 | import ch.cern.exdemon.monitor.Monitor; 13 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult; 14 | import ch.cern.properties.Properties; 15 | import scala.Tuple2; 16 | 17 | public class ComputeTriggerKeysF implements PairFlatMapFunction { 18 | 19 | private static final long serialVersionUID = 8388632785439398988L; 20 | 21 | private Properties componentsSourceProperties; 22 | 23 | public ComputeTriggerKeysF(Properties componentsSourceProperties) { 24 | this.componentsSourceProperties = componentsSourceProperties; 25 | } 26 | 27 | @Override 28 | public Iterator> call(AnalysisResult analysis) throws Exception { 29 | ComponentsCatalog.init(componentsSourceProperties); 30 | 31 | String monitorID = (String) analysis.getAnalysisParams().get("monitor.name"); 32 | java.util.Optional monitorOpt = ComponentsCatalog.get(Type.MONITOR, monitorID); 33 | if(!monitorOpt.isPresent()) 34 | return new LinkedList>().iterator(); 35 | Monitor monitor = monitorOpt.get(); 36 | 37 | Map metricIDs = monitor.getMetricIDs(analysis.getAnalyzed_metric()); 38 | 39 | Set triggerIDs = monitor.getTriggers().keySet(); 40 | 41 | return triggerIDs.stream() 42 | .map(id -> new TriggerStatusKey(monitorID, id, metricIDs)) 43 | .map(notID -> new Tuple2(notID, analysis)) 44 | .iterator(); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/exdemon/metrics/defined/equation/var/agg/CountAgregationTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var.agg; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | 5 | import java.time.Instant; 6 | import java.util.LinkedList; 7 | 8 | import org.junit.Test; 9 | 10 | import ch.cern.exdemon.metrics.DatedValue; 11 | import ch.cern.exdemon.metrics.value.AggregatedValue; 12 | import ch.cern.exdemon.metrics.value.BooleanValue; 13 | import ch.cern.exdemon.metrics.value.FloatValue; 14 | import ch.cern.exdemon.metrics.value.StringValue; 15 | import ch.cern.exdemon.metrics.value.Value; 16 | 17 | public class CountAgregationTest { 18 | 19 | private CountAgregation agg = new CountAgregation(); 20 | 21 | @Test 22 | public void count() { 23 | LinkedList values = new LinkedList(); 24 | values.add(new DatedValue(Instant.EPOCH, new FloatValue(0.2314))); 25 | values.add(new DatedValue(Instant.EPOCH, new StringValue("egerg"))); 26 | values.add(new DatedValue(Instant.EPOCH, new BooleanValue(true))); 27 | 28 | Value result = agg.aggregateValues(values, Instant.EPOCH); 29 | 30 | assertEquals(values.size(), result.getAsAggregated().get().getAsFloat().get(), 0f); 31 | } 32 | 33 | @Test 34 | public void countWithAggValues() { 35 | LinkedList values = new LinkedList(); 36 | values.add(new DatedValue(Instant.EPOCH, new AggregatedValue(new FloatValue(5)))); 37 | values.add(new DatedValue(Instant.EPOCH, new AggregatedValue(new FloatValue(2)))); 38 | values.add(new DatedValue(Instant.EPOCH, new FloatValue(932845))); 39 | values.add(new DatedValue(Instant.EPOCH, new StringValue("23fdw"))); 40 | values.add(new DatedValue(Instant.EPOCH, new BooleanValue(false))); 41 | 42 | Value result = agg.aggregateValues(values, Instant.EPOCH); 43 | 44 | assertEquals(10, result.getAsAggregated().get().getAsFloat().get(), 0f); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/ComputeBatchDefineMetricsF.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined; 2 | 3 | import java.time.Instant; 4 | import java.util.Iterator; 5 | import java.util.LinkedList; 6 | import java.util.List; 7 | import java.util.Optional; 8 | 9 | import org.apache.spark.api.java.function.FlatMapFunction; 10 | import org.apache.spark.streaming.Time; 11 | 12 | import ch.cern.exdemon.components.ComponentsCatalog; 13 | import ch.cern.exdemon.components.Component.Type; 14 | import ch.cern.exdemon.metrics.Metric; 15 | import ch.cern.exdemon.metrics.defined.equation.var.VariableStatuses; 16 | import ch.cern.properties.Properties; 17 | import scala.Tuple2; 18 | 19 | public class ComputeBatchDefineMetricsF implements FlatMapFunction, Metric> { 20 | 21 | private static final long serialVersionUID = 3779814069810467993L; 22 | 23 | private Instant batchTime; 24 | 25 | private Properties propertiesSourceProps; 26 | 27 | public ComputeBatchDefineMetricsF(Time batchTime, Properties propertiesSourceProps) { 28 | this.batchTime = Instant.ofEpochMilli(batchTime.milliseconds()); 29 | this.propertiesSourceProps = propertiesSourceProps; 30 | } 31 | 32 | @Override 33 | public Iterator call(Tuple2 pair) throws Exception { 34 | ComponentsCatalog.init(propertiesSourceProps); 35 | 36 | List result = new LinkedList<>(); 37 | 38 | DefinedMetricStatuskey ids = pair._1; 39 | VariableStatuses store = pair._2; 40 | 41 | Optional definedMetricOpt = ComponentsCatalog.get(Type.METRIC, ids.getID()); 42 | if(!definedMetricOpt.isPresent()) 43 | return result.iterator(); 44 | DefinedMetric definedMetric = definedMetricOpt.get(); 45 | 46 | Optional metricOpt = definedMetric.generateByBatch(store, batchTime, ids.getMetric_attributes()); 47 | 48 | metricOpt.ifPresent(result::add); 49 | 50 | return result.iterator(); 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/spark/TestInputStream.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark; 2 | 3 | import java.util.List; 4 | 5 | import org.apache.spark.api.java.JavaSparkContext; 6 | import org.apache.spark.rdd.RDD; 7 | import org.apache.spark.streaming.StreamingContext; 8 | import org.apache.spark.streaming.Time; 9 | import org.apache.spark.streaming.api.java.JavaInputDStream; 10 | import org.apache.spark.streaming.dstream.InputDStream; 11 | 12 | import scala.Option; 13 | import scala.reflect.ClassTag; 14 | 15 | public class TestInputStream extends InputDStream { 16 | 17 | private Batches output; 18 | 19 | private long batchDuration; 20 | 21 | public TestInputStream(StreamingContext _ssc, ClassTag evidence$1, Batches input, long batchDuration) { 22 | super(_ssc, evidence$1); 23 | 24 | this.output = input; 25 | this.batchDuration = batchDuration; 26 | } 27 | 28 | @Override 29 | public void start() { 30 | } 31 | 32 | @Override 33 | public void stop() { 34 | } 35 | 36 | @Override 37 | public Option> compute(Time time) { 38 | int inputIndex = (int) (time.milliseconds() / batchDuration) - 1; 39 | 40 | if(inputIndex < output.size()){ 41 | List selectedOutput = output.get(inputIndex); 42 | 43 | return Option.apply(getSparkContext().parallelize(selectedOutput).rdd()); 44 | }else{ 45 | return Option.empty(); 46 | } 47 | } 48 | 49 | public JavaSparkContext getSparkContext() { 50 | return JavaSparkContext.fromSparkContext(context().sparkContext()); 51 | } 52 | 53 | public static class JTestInputStream extends JavaInputDStream{ 54 | 55 | private static final long serialVersionUID = 6668075506727419418L; 56 | 57 | public JTestInputStream(StreamingContext _ssc, ClassTag classTag, Batches inputBatches, long batchDuration) { 58 | super(new TestInputStream<>(_ssc, classTag, inputBatches, batchDuration), classTag); 59 | } 60 | 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/trigger/action/silence/SilencesFilterF.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.trigger.action.silence; 2 | 3 | import java.util.Map; 4 | import java.util.Optional; 5 | 6 | import org.apache.spark.api.java.function.Function; 7 | 8 | import ch.cern.exdemon.components.Component.Type; 9 | import ch.cern.exdemon.components.ComponentsCatalog; 10 | import ch.cern.exdemon.json.JSONParser; 11 | import ch.cern.exdemon.monitor.trigger.action.Action; 12 | import ch.cern.properties.Properties; 13 | 14 | public class SilencesFilterF implements Function { 15 | 16 | private static final long serialVersionUID = 7712606037271217706L; 17 | 18 | private Properties componentsSourceProperties; 19 | 20 | public SilencesFilterF(Properties componentsSourceProps) { 21 | this.componentsSourceProperties = componentsSourceProps; 22 | } 23 | 24 | @Override 25 | public Boolean call(Action action) throws Exception { 26 | ComponentsCatalog.init(componentsSourceProperties); 27 | 28 | Map silences = ComponentsCatalog.get(Type.SILENCE); 29 | 30 | Optional filterResult = silences.values().stream() 31 | .filter(silence -> silence.isActiveAt(action.getCreation_timestamp())) 32 | .map(silence -> shouldSilentAndReport(action, silence)) 33 | .reduce((a, b) -> a && b); 34 | 35 | return filterResult.orElse(true); 36 | } 37 | 38 | private boolean shouldSilentAndReport(Action action, Silence silence) { 39 | boolean actionAllowed = silence.filter(action); 40 | 41 | if(!actionAllowed) 42 | ComponentsCatalog.addToReport(Type.SILENCE, 43 | silence.getId(), 44 | "silenced", 45 | JSONParser.parse(action).toString()); 46 | 47 | return actionAllowed; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/trigger/action/Action.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.trigger.action; 2 | 3 | import java.io.Serializable; 4 | import java.time.Instant; 5 | import java.util.Map; 6 | import java.util.Set; 7 | 8 | import ch.cern.Taggable; 9 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult; 10 | import lombok.Getter; 11 | import lombok.NonNull; 12 | import lombok.Setter; 13 | import lombok.ToString; 14 | 15 | @ToString 16 | public class Action implements Serializable, Taggable { 17 | 18 | private static final long serialVersionUID = 6730655599755849423L; 19 | 20 | @Getter @Setter @NonNull 21 | private Instant creation_timestamp; 22 | 23 | @Getter @Setter @NonNull 24 | private String monitor_id; 25 | 26 | @Getter @Setter @NonNull 27 | private String trigger_id; 28 | 29 | @Getter @Setter @NonNull 30 | private Map metric_attributes; 31 | 32 | @Getter @Setter @NonNull 33 | private String reason; 34 | 35 | @Setter @NonNull 36 | private Map tags; 37 | 38 | @Getter @Setter @NonNull 39 | private Set actuatorIDs; 40 | 41 | @Getter @Setter @NonNull 42 | private AnalysisResult triggeringResult; 43 | 44 | public Action( 45 | @NonNull String monitorID, 46 | @NonNull String trgiggerID, 47 | @NonNull Map metric_attributes, 48 | @NonNull String reason, 49 | @NonNull Set actuatorIDs, 50 | @NonNull Map tags, 51 | @NonNull AnalysisResult triggeringResult) { 52 | this.creation_timestamp = Instant.now(); 53 | this.monitor_id = monitorID; 54 | this.trigger_id = trgiggerID; 55 | this.metric_attributes = metric_attributes; 56 | this.reason = reason; 57 | this.actuatorIDs = actuatorIDs; 58 | this.tags = tags; 59 | this.triggeringResult = triggeringResult; 60 | } 61 | 62 | @Override 63 | public Map getTags() { 64 | return tags; 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/var/agg/SuccessiveCountAgregation.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var.agg; 2 | 3 | import java.time.Instant; 4 | import java.util.Collection; 5 | 6 | import ch.cern.exdemon.components.RegisterComponentType; 7 | import ch.cern.exdemon.metrics.DatedValue; 8 | import ch.cern.exdemon.metrics.Metric; 9 | import ch.cern.exdemon.metrics.ValueHistory; 10 | import ch.cern.exdemon.metrics.defined.equation.var.ValueVariable; 11 | import ch.cern.exdemon.metrics.value.FloatValue; 12 | import ch.cern.exdemon.metrics.value.Value; 13 | 14 | @RegisterComponentType("successive_count") 15 | public class SuccessiveCountAgregation extends Aggregation { 16 | 17 | private static final long serialVersionUID = 8274088208286521725L; 18 | 19 | @Override 20 | public Class inputType() { 21 | return Value.class; 22 | } 23 | 24 | @Override 25 | public Value aggregate(Collection values, Instant time) { 26 | return new FloatValue(values.stream() 27 | .mapToDouble(v -> v.getValue().getAsAggregated().isPresent() ? 28 | v.getValue().getAsAggregated().get().getAsFloat().get() : 29 | 1) 30 | .sum()); 31 | } 32 | 33 | @Override 34 | public Class returnType() { 35 | return FloatValue.class; 36 | } 37 | 38 | @Override 39 | public boolean isFilterEnable() { 40 | return false; 41 | } 42 | 43 | @Override 44 | public void postUpdateStatus(ValueVariable metricVariable, AggregationValues aggValues, Metric metric) { 45 | if(!metricVariable.getFilter().test(metric)) 46 | aggValues.reset(); 47 | } 48 | 49 | @Override 50 | public void postUpdateStatus(ValueVariable metricVariable, ValueHistory history, Metric metric) { 51 | if(!metricVariable.getFilter().test(metric)) 52 | history.reset(); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/DefinedMetrics.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined; 2 | 3 | import java.io.IOException; 4 | import java.util.Optional; 5 | 6 | import org.apache.spark.streaming.api.java.JavaDStream; 7 | import org.apache.spark.streaming.api.java.JavaPairDStream; 8 | 9 | import ch.cern.exdemon.metrics.Metric; 10 | import ch.cern.exdemon.metrics.defined.equation.var.VariableStatuses; 11 | import ch.cern.properties.ConfigurationException; 12 | import ch.cern.properties.Properties; 13 | import ch.cern.spark.status.StateDStream; 14 | import ch.cern.spark.status.Status; 15 | import ch.cern.spark.status.StatusOperation; 16 | 17 | public class DefinedMetrics { 18 | 19 | public static JavaDStream generate( 20 | JavaDStream metrics, 21 | Properties propertiesSourceProps, 22 | Optional>> operationsOpt) 23 | throws ClassNotFoundException, IOException, ConfigurationException{ 24 | 25 | JavaPairDStream resultsKeyed = metrics.flatMapToPair(new ComputeDefinedMetricKeysF(propertiesSourceProps)); 26 | 27 | JavaDStream> operations = resultsKeyed.map(mk -> new StatusOperation<>(mk._1, mk._2)); 28 | 29 | if(operationsOpt.isPresent()) 30 | operations = operations.union(operationsOpt.get()); 31 | 32 | StateDStream statuses = 33 | Status.map( 34 | DefinedMetricStatuskey.class, 35 | VariableStatuses.class, 36 | operations, 37 | new UpdateDefinedMetricStatusesF(propertiesSourceProps)); 38 | 39 | JavaDStream definedMetricsWhenBatch = statuses.statuses().transform((rdd, time) -> rdd.flatMap(new ComputeBatchDefineMetricsF(time, propertiesSourceProps))); 40 | 41 | return statuses.values().union(definedMetricsWhenBatch); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/spark/status/storage/manager/ExpireStatusValueFilterTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.spark.status.storage.manager; 2 | 3 | import static org.junit.Assert.assertFalse; 4 | import static org.junit.Assert.assertTrue; 5 | 6 | import java.time.Duration; 7 | import java.time.Instant; 8 | import java.util.HashMap; 9 | 10 | import org.apache.spark.streaming.StateImpl; 11 | import org.apache.spark.streaming.Time; 12 | import org.junit.Test; 13 | 14 | import ch.cern.exdemon.metrics.defined.DefinedMetricStatuskey; 15 | import ch.cern.spark.status.StatusValue; 16 | import ch.cern.spark.status.TestStatus; 17 | import scala.Tuple2; 18 | 19 | public class ExpireStatusValueFilterTest { 20 | 21 | @Test 22 | public void filterByExpiration() throws Exception { 23 | ExpireStatusValueFilter filter = new ExpireStatusValueFilter(Duration.ofMinutes(4)); 24 | 25 | Instant now = Instant.now(); 26 | 27 | assertTrue(filter.call(new Tuple2<>(new DefinedMetricStatuskey("dm1", new HashMap<>()), new TestStatus(1)))); 28 | 29 | StatusValue status = new TestStatus(1); 30 | status.update(new StateImpl<>(), new Time(now.minus(Duration.ofMinutes(10)).toEpochMilli())); 31 | assertTrue(filter.call(new Tuple2<>(new DefinedMetricStatuskey("dm1", new HashMap<>()), status))); 32 | 33 | status = new TestStatus(1); 34 | status.update(new StateImpl<>(), new Time(now.minus(Duration.ofMinutes(5)).toEpochMilli())); 35 | assertTrue(filter.call(new Tuple2<>(new DefinedMetricStatuskey("dm1", new HashMap<>()), status))); 36 | 37 | status = new TestStatus(1); 38 | status.update(new StateImpl<>(), new Time(now.minus(Duration.ofMinutes(3)).toEpochMilli())); 39 | assertFalse(filter.call(new Tuple2<>(new DefinedMetricStatuskey("dm1", new HashMap<>()), status))); 40 | 41 | status = new TestStatus(1); 42 | status.update(new StateImpl<>(), new Time(now.minus(Duration.ofMinutes(1)).toEpochMilli())); 43 | assertFalse(filter.call(new Tuple2<>(new DefinedMetricStatuskey("dm1", new HashMap<>()), status))); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/ch/cern/exdemon/metrics/defined/equation/var/agg/AggregationValuesTest.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var.agg; 2 | 3 | import static org.junit.Assert.fail; 4 | 5 | import java.time.Duration; 6 | import java.time.Instant; 7 | 8 | import org.junit.Test; 9 | 10 | import ch.cern.exdemon.metrics.defined.equation.ComputationException; 11 | 12 | public class AggregationValuesTest { 13 | 14 | @Test 15 | public void shouldGenerateExceptionWhenMaxSizeIsReached() throws ComputationException { 16 | AggregationValues values = new AggregationValues(3, 0); 17 | 18 | Instant now = Instant.now(); 19 | 20 | values.add(0, 0f, now); 21 | values.getDatedValues(); 22 | 23 | values.add(1, 0f, now); 24 | values.getDatedValues(); 25 | 26 | values.add(2, 0f, now); 27 | values.getDatedValues(); 28 | 29 | values.add(3, 0f, now); 30 | try { 31 | values.getDatedValues(); 32 | fail(); 33 | }catch(Exception e) {} 34 | } 35 | 36 | @Test 37 | public void shouldRecoverAfterMaxSizeIsReached() throws ComputationException { 38 | AggregationValues values = new AggregationValues(3, 0); 39 | 40 | Instant now = Instant.now(); 41 | 42 | values.add(0, 0f, now); 43 | values.getDatedValues(); 44 | 45 | values.add(1, 0f, now.plus(Duration.ofSeconds(2))); 46 | values.getDatedValues(); 47 | 48 | values.add(2, 0f, now.plus(Duration.ofSeconds(4))); 49 | values.getDatedValues(); 50 | 51 | values.add(3, 0f, now.plus(Duration.ofSeconds(6))); 52 | try { 53 | values.getDatedValues(); 54 | fail(); 55 | }catch(Exception e) {} 56 | 57 | values.purge(now.plus(Duration.ofSeconds(1))); 58 | 59 | values.getDatedValues(); 60 | 61 | values.add(4, 0f, now.plus(Duration.ofSeconds(6))); 62 | try { 63 | values.getDatedValues(); 64 | fail(); 65 | }catch(Exception e) {} 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/monitor/analysis/results/sink/types/ElasticAnalysisResultsSink.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.monitor.analysis.results.sink.types; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | import java.util.Map.Entry; 6 | 7 | import org.apache.spark.streaming.api.java.JavaDStream; 8 | import org.elasticsearch.spark.streaming.api.java.JavaEsSparkStreaming; 9 | 10 | import ch.cern.exdemon.components.ConfigurationResult; 11 | import ch.cern.exdemon.components.RegisterComponentType; 12 | import ch.cern.exdemon.json.JSONParser; 13 | import ch.cern.exdemon.monitor.analysis.results.AnalysisResult; 14 | import ch.cern.exdemon.monitor.analysis.results.sink.AnalysisResultsSink; 15 | import ch.cern.properties.Properties; 16 | 17 | @RegisterComponentType("elastic") 18 | public class ElasticAnalysisResultsSink extends AnalysisResultsSink { 19 | 20 | private static final long serialVersionUID = -3422447741754872104L; 21 | 22 | private String indexName; 23 | 24 | private Map elasticConfig; 25 | 26 | @Override 27 | public ConfigurationResult config(Properties properties) { 28 | indexName = properties.getProperty("index"); 29 | 30 | elasticConfig = getElasticConfig(properties); 31 | 32 | return ConfigurationResult.SUCCESSFUL(); 33 | } 34 | 35 | private Map getElasticConfig(Properties properties) { 36 | Properties esProps = properties.getSubset("es"); 37 | 38 | Map outputMap = new HashMap<>(); 39 | 40 | for (Entry entry : esProps.entrySet()) { 41 | String key = (String) entry.getKey(); 42 | String value = (String) entry.getValue(); 43 | 44 | outputMap.put("es." + key, value); 45 | } 46 | 47 | return outputMap; 48 | } 49 | 50 | @Override 51 | public void sink(JavaDStream outputStream) { 52 | JavaDStream stream = outputStream.map(JSONParser::parse).map(Object::toString); 53 | 54 | JavaEsSparkStreaming.saveJsonToEs(stream, indexName, elasticConfig); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/ch/cern/exdemon/metrics/defined/equation/var/agg/LastValueAggregation.java: -------------------------------------------------------------------------------- 1 | package ch.cern.exdemon.metrics.defined.equation.var.agg; 2 | 3 | import java.time.Instant; 4 | import java.util.Collection; 5 | import java.util.stream.Collectors; 6 | 7 | import ch.cern.exdemon.components.RegisterComponentType; 8 | import ch.cern.exdemon.metrics.DatedValue; 9 | import ch.cern.exdemon.metrics.Metric; 10 | import ch.cern.exdemon.metrics.ValueHistory; 11 | import ch.cern.exdemon.metrics.defined.equation.var.ValueVariable; 12 | import ch.cern.exdemon.metrics.value.ExceptionValue; 13 | import ch.cern.exdemon.metrics.value.Value; 14 | 15 | @RegisterComponentType("last") 16 | public class LastValueAggregation extends Aggregation { 17 | 18 | private static final long serialVersionUID = -2652766061746715818L; 19 | 20 | private String typeClassName; 21 | 22 | public LastValueAggregation(Class type) { 23 | this.typeClassName = type.getName(); 24 | } 25 | 26 | @Override 27 | public Class inputType() { 28 | return Value.class; 29 | } 30 | 31 | @Override 32 | public Value aggregate(Collection values, Instant time) { 33 | if(values.isEmpty()) 34 | return new ExceptionValue("no values"); 35 | 36 | Value value = values.stream().sorted().collect(Collectors.toList()).get(values.size() - 1).getValue(); 37 | 38 | return value.getAsAggregated().isPresent() ? value.getAsAggregated().get() : value; 39 | } 40 | 41 | @Override 42 | public Class returnType() { 43 | return getClassType(); 44 | } 45 | 46 | @SuppressWarnings("unchecked") 47 | private Class getClassType() { 48 | try { 49 | return (Class) Class.forName(typeClassName); 50 | } catch (ClassNotFoundException e) { 51 | throw new RuntimeException("Class not found"); 52 | } 53 | } 54 | 55 | @Override 56 | public void postUpdateStatus(ValueVariable metricVariable, ValueHistory history, Metric metric) { 57 | history.reset(); 58 | history.add(metric.getTimestamp(), metric.getValue(), metric); 59 | } 60 | 61 | } 62 | --------------------------------------------------------------------------------