├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── doc ├── ad_model_performance.png ├── egadsadm.png ├── egadstsmd.png ├── evals.png └── ui.png ├── lib └── Synthetic_Anomalies │ ├── config.properties │ └── synthetic_anomaly_generation.py ├── pom.xml ├── screwdriver.yaml └── src ├── main └── java │ └── com │ └── yahoo │ └── egads │ ├── Egads.java │ ├── control │ ├── AnomalyDetector.java │ ├── DetectAnomalyProcessable.java │ ├── ModelAdapter.java │ ├── ProcessableObject.java │ ├── ProcessableObjectFactory.java │ ├── TransformInputProcessable.java │ └── UpdateModelProcessable.java │ ├── data │ ├── Anomaly.java │ ├── AnomalyErrorStorage.java │ ├── FileModelStore.java │ ├── JsonAble.java │ ├── JsonEncoder.java │ ├── MetricMeta.java │ ├── Model.java │ ├── ModelStore.java │ ├── TimeSeries.java │ └── WeightedValue.java │ ├── models │ ├── adm │ │ ├── AdaptiveKernelDensityChangePointDetector.java │ │ ├── AnomalyDetectionAbstractModel.java │ │ ├── AnomalyDetectionModel.java │ │ ├── DBScanModel.java │ │ ├── ExtremeLowDensityModel.java │ │ ├── KSigmaModel.java │ │ ├── NaiveModel.java │ │ └── SimpleThresholdModel.java │ └── tsmm │ │ ├── AutoForecastModel.java │ │ ├── DoubleExponentialSmoothingModel.java │ │ ├── MovingAverageModel.java │ │ ├── MultipleLinearRegressionModel.java │ │ ├── NaiveForecastingModel.java │ │ ├── NullModel.java │ │ ├── OlympicModel.java │ │ ├── OlympicModel2.java │ │ ├── PolynomialRegressionModel.java │ │ ├── RegressionModel.java │ │ ├── SimpleExponentialSmoothingModel.java │ │ ├── SpectralSmoother.java │ │ ├── TimeSeriesAbstractModel.java │ │ ├── TimeSeriesModel.java │ │ ├── TripleExponentialSmoothingModel.java │ │ └── WeightedMovingAverageModel.java │ └── utilities │ ├── AutoSensitivity.java │ ├── DBSCANClusterer.java │ ├── DateTimeCalculator.java │ ├── FileInputProcessor.java │ ├── FileUtils.java │ ├── GUIUtils.java │ ├── IdentifiedDoublePoint.java │ ├── InputProcessor.java │ ├── ListUtils.java │ ├── SpectralMethods.java │ ├── StatsUtils.java │ ├── StdinProcessor.java │ └── Variance.java └── test ├── java └── com │ └── yahoo │ └── egads │ ├── TestAggregate.java │ ├── TestAnomaly.java │ ├── TestAnomalyDetect.java │ ├── TestAutoForecast.java │ ├── TestAutoSensitivity.java │ ├── TestChangePointDetectionAlgorithm.java │ ├── TestGetAnomalyList.java │ ├── TestJsonEncoder.java │ ├── TestMetricMeta.java │ ├── TestMissingData.java │ ├── TestOlympicModel.java │ ├── TestReflectionSpeed.java │ ├── TestStats.java │ ├── TestTimeSeries.java │ ├── TestVariance.java │ ├── data │ └── TestWeightedValue.java │ ├── models │ └── tsmm │ │ └── TestOlympicModel2.java │ └── utilities │ └── TestSpectralMethods.java └── resources ├── config.json ├── cp-exp.csv ├── cp-obs.csv ├── log4j2.xml ├── model_input.csv ├── model_output_10_0.csv ├── model_output_10_1.csv ├── model_output_5_0.csv ├── model_output_5_1.csv ├── sample_config.ini ├── sample_errors.csv ├── sample_errors_2.csv ├── sample_input.csv └── sample_missing_input.csv /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | 3 | *.iml 4 | .idea/ 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | dist: trusty 3 | addons: 4 | hostname: short-hostname 5 | jdk: 6 | - oraclejdk8 7 | install: /bin/true # Skip the mvn install step. 8 | script: 9 | - export MAVEN_OPTS="-Xmx1024m"; 10 | if [[ "$TRAVIS_BRANCH" == "master" && "$TRAVIS_PULL_REQUEST" = "false" ]]; then 11 | mvn clean deploy -q --settings settings.xml; 12 | else 13 | mvn clean test; 14 | fi 15 | notifications: 16 | email: false 17 | -------------------------------------------------------------------------------- /doc/ad_model_performance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/egads/40d3db6e27fea649b2aa23c26d0f14991e6adaae/doc/ad_model_performance.png -------------------------------------------------------------------------------- /doc/egadsadm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/egads/40d3db6e27fea649b2aa23c26d0f14991e6adaae/doc/egadsadm.png -------------------------------------------------------------------------------- /doc/egadstsmd.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/egads/40d3db6e27fea649b2aa23c26d0f14991e6adaae/doc/egadstsmd.png -------------------------------------------------------------------------------- /doc/evals.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/egads/40d3db6e27fea649b2aa23c26d0f14991e6adaae/doc/evals.png -------------------------------------------------------------------------------- /doc/ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yahoo/egads/40d3db6e27fea649b2aa23c26d0f14991e6adaae/doc/ui.png -------------------------------------------------------------------------------- /lib/Synthetic_Anomalies/config.properties: -------------------------------------------------------------------------------- 1 | # --- Use real time-series or New synthetic time-series --- 2 | # use real(input) 0 3 | # genrerate new 1 4 | OUTPUT_FORMAT = 1 5 | 6 | # --- Seasonality type --- 7 | # Only for OUTPUT_FORMAT = 1 8 | # daily 0 9 | # weekly 1 10 | 11 | S_TYPE = 1 12 | 13 | # --- Number of weeks --- 14 | # enter number of weeks you want in output time-series here(maximum 52), 15 | 16 | NUM_WEEKS = 4 17 | 18 | # --- Type of anomaly --- 19 | # Point anomaly(sudden change) 0 20 | # Contextual anomaly(gradual incease or desrease with time) 1 21 | # Anomalous behaviour 2 22 | 23 | A_TYPE = 2 24 | 25 | # --- Frequency of anomaly --- 26 | # enter the probability value that represents frequency of anomalies in data 27 | # For A_TYPE = 1 and 2, Frequncy value applies for second half of time-series because first half should act as a reference to normal time-series 28 | 29 | FREQUENCY = 0.5 30 | 31 | # --- Severity of anomaly --- 32 | # enter the severity amount(0 to 1, 0 being very less sever to 1 being very high sever) of anomaly 33 | # less sever means less variation from normal values: hard to detect 34 | # highly sever means high variation from normal values: easy to detect 35 | # In terms of standard deviation(s.d.): 36 | # 0 ---> 0% deviation in data(pure smoothed time-series) 37 | # 0.5 ---> 100% deviation in data in the case of anomaly 38 | # 1.0 ---> 200% deviation in data in the case of anomaly 39 | # for rest of the values between 0 to 1 it scales linearly as per the above interval values 40 | SEVERITY = 0.5 -------------------------------------------------------------------------------- /screwdriver.yaml: -------------------------------------------------------------------------------- 1 | shared: 2 | image: maven 3 | 4 | jobs: 5 | main: 6 | steps: 7 | - run_arbitrary_script: mvn clean test --quiet 8 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/Egads.java: -------------------------------------------------------------------------------- 1 | // main entry point for egads processing node 2 | 3 | package com.yahoo.egads; 4 | 5 | import java.util.Properties; 6 | import java.io.FileInputStream; 7 | import java.io.InputStream; 8 | import com.yahoo.egads.utilities.*; 9 | import java.io.File; 10 | 11 | /* 12 | * Call stack. 13 | * 14 | * Anomaly Dtection Use Case (Assuming a trained model). 15 | * Egads.Main() 16 | * ProcessableObjectFactory.create 17 | * Create AnomalyDetector 18 | * BuildADModel() 19 | * Create ModelAdapter 20 | * BuildTSModel() 21 | * po.process 22 | * TODO: write to anomaly DB. 23 | * 24 | */ 25 | 26 | public class Egads { 27 | public static void main(String[] args) throws Exception { 28 | 29 | if (args.length == 0) { 30 | System.err.println("Usage: java Egads config.ini (input [STDIN,CSV])"); 31 | System.exit(1); 32 | } 33 | 34 | // TODO: This config will be retrieved from ConfigDB later, 35 | // for now it is assumed it's a static file. 36 | Properties p = new Properties(); 37 | String config = args[0]; 38 | File f = new File(config); 39 | boolean isRegularFile = f.exists(); 40 | 41 | if (isRegularFile) { 42 | InputStream is = new FileInputStream(config); 43 | p.load(is); 44 | } else { 45 | FileUtils.initProperties(config, p); 46 | } 47 | 48 | // Set the input type. 49 | InputProcessor ip = null; 50 | if (p.getProperty("INPUT") == null || p.getProperty("INPUT").equals("CSV")) { 51 | ip = new FileInputProcessor(args[1]); 52 | } else { 53 | ip = new StdinProcessor(); 54 | } 55 | 56 | // Process the input the we received (either STDIN or as a file). 57 | ip.processInput(p); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/control/AnomalyDetector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | /* 8 | * Description: AnomalyDetector applies a set of anomaly detection models (AD algorithms) on a given metric. 9 | * AnomalyDetector provides concrete mechanisms to apply one or more abstract anomaly detection algorithms on 10 | * a time-series at the execution time; in other words, application of a certain anomaly detection algorithm on 11 | * a given time series should be carried out via an AnomalyDetector object. 12 | * The direct application of models on time series is discouraged in EGADS unless for test purposes. 13 | * 14 | * Inputs: 15 | * 1. The 'metric' time series 16 | * - Either an explicit TimeSeries object 17 | * - or the String name of the time series which would require the AnomalyDetector to connect to ModelDB to load 18 | * the appropriate anomaly detection models into the memory (under construction) 19 | * 20 | * 2. The model(s) 21 | * - Either an explicit AnomalyDetectionModel object via addModel() 22 | * - or implicitly loaded from ModelDB when the metric name is provided (under construction) 23 | * 24 | * Features: 25 | * 1. Resetting all the added anomaly detection models via reset() 26 | * 2. Tuning all the added anomaly detection models on the 'metric' via tune() 27 | * 3. Performing anomaly detection on the metric according to all the added anomaly detection models via detect() 28 | * 29 | * Details: 30 | * 1. The time units for interfacing with an AnomalyDetector object is the standard UNIX timestamp; however, AnomalyDetector 31 | * automatically performs logical indexing conversion for the abstract algorithms so that the actual models can 32 | * conveniently work with the logical index instead of UNIX timestamps. The conversion is: 33 | * 34 | * logical_index = (UNIX_timestamp - firstTimeStamp) div period 35 | * UNIX_timestamp = logical_index * period + firstTimeStamp 36 | **/ 37 | 38 | package com.yahoo.egads.control; 39 | 40 | import java.util.ArrayList; 41 | 42 | import com.yahoo.egads.data.Anomaly; 43 | import com.yahoo.egads.data.Anomaly.IntervalSequence; 44 | import com.yahoo.egads.data.TimeSeries; 45 | import com.yahoo.egads.models.adm.AnomalyDetectionModel; 46 | 47 | public class AnomalyDetector { 48 | 49 | protected TimeSeries metric = null; 50 | protected ArrayList models = new ArrayList(); 51 | protected ArrayList isTuned = new ArrayList(); 52 | protected long firstTimeStamp = 0; 53 | protected long period; 54 | 55 | // Construction //////////////////////////////////////////////////////////////////////////////// 56 | 57 | public AnomalyDetector(TimeSeries theMetric, long period, 58 | long firstTimeStamp) throws Exception { 59 | if (theMetric == null) { 60 | throw new Exception("The input metric is null."); 61 | } 62 | 63 | metric = theMetric; 64 | this.period = period; 65 | this.firstTimeStamp = firstTimeStamp; 66 | } 67 | 68 | public AnomalyDetector(TimeSeries theMetric, long period) throws Exception { 69 | if (theMetric == null) { 70 | throw new Exception("The input metric is null."); 71 | } 72 | 73 | metric = theMetric; 74 | this.period = period; 75 | 76 | if (metric.data.size() > 0) { 77 | this.firstTimeStamp = metric.time(0); 78 | } 79 | } 80 | 81 | public AnomalyDetector(String theMetric, long period) throws Exception { 82 | this.period = period; 83 | // TODO: 84 | // 1 - load the models related to theMetric from ModelDB 85 | // 2 - push the loaded models into 'models' 86 | // 3 - create a new TimeSeries for theMetric and set 'metric' 87 | // 4 - set 'firstTimeStamp' 88 | 89 | int modelNum = models.size(); 90 | for (int i = 0; i < modelNum; ++i) { 91 | isTuned.set(i, true); 92 | } 93 | } 94 | 95 | // Configuration Methods //////////////////////////////////////////////////////////////// 96 | 97 | public void setMetric(TimeSeries theMetric, long period) { 98 | metric = theMetric; 99 | this.period = period; 100 | 101 | if (metric.data.size() > 0) { 102 | this.firstTimeStamp = metric.time(0); 103 | } 104 | 105 | reset(); 106 | } 107 | 108 | public void setMetric(TimeSeries theMetric, long period, long firstTimeStamp) { 109 | metric = theMetric; 110 | this.period = period; 111 | this.firstTimeStamp = firstTimeStamp; 112 | reset(); 113 | } 114 | 115 | public void setMetric(String theMetric, long period) { 116 | this.period = period; 117 | firstTimeStamp = 0; 118 | models.clear(); 119 | isTuned.clear(); 120 | 121 | // TODO: 122 | // 1 - load the models related to theMetric from ModelDB 123 | // 2 - push the loaded models into 'models' 124 | // 3 - create a new TimeSeries for theMetric and set 'metric' 125 | // 4 - set 'firstTimeStamp' 126 | 127 | int modelNum = models.size(); 128 | for (int i = 0; i < modelNum; ++i) { 129 | isTuned.set(i, true); 130 | } 131 | } 132 | 133 | public void addModel(AnomalyDetectionModel model) { 134 | model.reset(); 135 | models.add(model); 136 | isTuned.add(false); 137 | } 138 | 139 | // Algorithmic Methods //////////////////////////////////////////////////////////////////// 140 | 141 | public void reset() { 142 | int i = 0; 143 | for (AnomalyDetectionModel model : models) { 144 | model.reset(); 145 | isTuned.set(i, false); 146 | i++; 147 | } 148 | } 149 | 150 | public void tune(TimeSeries.DataSequence expectedValues) throws Exception { 151 | int i = 0; 152 | 153 | metric.data.setLogicalIndices(firstTimeStamp, period); 154 | 155 | for (AnomalyDetectionModel model : models) { 156 | if (!isTuned.get(i)) { 157 | model.tune(metric.data, expectedValues); 158 | isTuned.set(i, true); 159 | } 160 | i++; 161 | } 162 | } 163 | 164 | public ArrayList detect(TimeSeries observedSeries, 165 | TimeSeries.DataSequence expectedSeries) throws Exception { 166 | for (Boolean b : isTuned) { 167 | if (!b) { 168 | throw new Exception( 169 | "All the models need to be tuned before detection."); 170 | } 171 | } 172 | 173 | ArrayList result = new ArrayList(); 174 | observedSeries.data.setLogicalIndices(firstTimeStamp, period); 175 | expectedSeries.setLogicalIndices(firstTimeStamp, period); 176 | 177 | for (AnomalyDetectionModel model : models) { 178 | Anomaly anomaly = new Anomaly(observedSeries.meta.name, 179 | observedSeries.meta); 180 | anomaly.modelName = model.getModelName(); 181 | anomaly.type = model.getType(); 182 | anomaly.intervals = model.detect(observedSeries.data, 183 | expectedSeries); 184 | anomaly.intervals.setLogicalIndices(firstTimeStamp, period); 185 | anomaly.intervals.setTimeStamps(firstTimeStamp, period); 186 | result.add(anomaly); 187 | } 188 | 189 | return result; 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/control/DetectAnomalyProcessable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // A template for doing Anomaly Detection. 8 | 9 | package com.yahoo.egads.control; 10 | 11 | import java.util.ArrayList; 12 | 13 | import com.yahoo.egads.data.Anomaly; 14 | import com.yahoo.egads.data.TimeSeries; 15 | import com.yahoo.egads.utilities.GUIUtils; 16 | import java.util.Properties; 17 | 18 | public class DetectAnomalyProcessable implements ProcessableObject { 19 | private ModelAdapter ma; 20 | private AnomalyDetector ad; 21 | private Properties config; 22 | private ArrayList anomalyList; 23 | 24 | public ArrayList getAnomalyList() { 25 | return anomalyList; 26 | } 27 | 28 | 29 | DetectAnomalyProcessable(ModelAdapter ma, AnomalyDetector ad, Properties config) { 30 | this.ma = ma; 31 | this.ad = ad; 32 | this.config = config; 33 | anomalyList = new ArrayList<>(); 34 | } 35 | 36 | public void process() throws Exception { 37 | 38 | // Resetting the models 39 | ma.reset(); 40 | 41 | // Training the model with the whole metric 42 | ma.train(); 43 | 44 | // Finding the expected values 45 | ArrayList list = ma.forecast( 46 | ma.metric.startTime(), ma.metric.lastTime()); 47 | 48 | // For each model's prediction in the ModelAdapter 49 | for (TimeSeries.DataSequence ds : list) { 50 | // Reseting the anomaly detectors 51 | ad.reset(); 52 | 53 | // Unsupervised tuning of the anomaly detectors 54 | ad.tune(ds); 55 | 56 | // Detecting anomalies for each anomaly detection model in anomaly detector 57 | anomalyList = ad.detect(ad.metric, ds); 58 | 59 | // Writing the anomalies to AnomalyDB 60 | if (config.getProperty("OUTPUT") != null && config.getProperty("OUTPUT").equals("ANOMALY_DB")) { 61 | for (Anomaly anomaly : anomalyList) { 62 | // TODO: Batch Anomaly Process. 63 | } 64 | } else if (config.getProperty("OUTPUT") != null && config.getProperty("OUTPUT").equals("GUI")) { 65 | GUIUtils.plotResults(ma.metric.data, ds, anomalyList, config); 66 | } else if (config.getProperty("OUTPUT") != null && config.getProperty("OUTPUT").equals("PLOT")) { 67 | for (Anomaly anomaly : anomalyList) { 68 | System.out.print(anomaly.toPlotString()); 69 | } 70 | } else { 71 | for (Anomaly anomaly : anomalyList) { 72 | System.out.print(anomaly.toPerlString()); 73 | } 74 | } 75 | } 76 | } 77 | 78 | public ArrayList result() throws Exception { 79 | return getAnomalyList(); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/control/ModelAdapter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | /* 8 | * Description: ModelAdapter applies a set of time-series models (algorithms) on a given metric. ModelAdapter provides 9 | * concrete mechanisms to apply one or more abstract algorithms on a time-series at the execution time; in other words, 10 | * application of a certain algorithm (model) on a given time series should be carried out via a ModelAdapter object. 11 | * The direct application of models on time series is discouraged in EGADS unless for test purposes. 12 | * 13 | * Inputs: 1. The 'metric' time series - Either an explicit TimeSeries object - or the String name of the time series 14 | * which would require the ModelAdapter to connect to ModelDB to load the appropriate models into the memory (under 15 | * construction) 16 | * 17 | * 2. The model(s) - Either an explicit TimeSeriesModel object via addModel() - or implicitly loaded from ModelDB when 18 | * the metric name is provided (under construction) 19 | * 20 | * Features: 1. Resetting all the added models via reset() 2. Training all the added models on the 'metric' via train() 21 | * 3. Updating all the added models for a new time series sequence via update() 4. Forecasting the value of the time 22 | * series according to all the added models for a given time period via forecast() 23 | * 24 | * Details: 1. The time units for interfacing with a ModelAdapter object is the standard UNIX timestamp; however, 25 | * ModelAdapter automatically performs logical indexing conversion for the abstract algorithms so that the actual models 26 | * can conveniently work with the logical index instead of UNIX timestamps. The conversion is: 27 | * 28 | * logical_index = (UNIX_timestamp - firstTimeStamp) div period UNIX_timestamp = logical_index * period + firstTimeStamp 29 | */ 30 | 31 | package com.yahoo.egads.control; 32 | 33 | import java.util.ArrayList; 34 | 35 | import com.yahoo.egads.data.TimeSeries; 36 | import com.yahoo.egads.models.tsmm.TimeSeriesModel; 37 | 38 | public class ModelAdapter { // Encapsulates a metric and the models operating on it 39 | 40 | protected TimeSeries metric = null; 41 | protected ArrayList models = new ArrayList(); 42 | protected ArrayList isTrained = new ArrayList(); 43 | protected long firstTimeStamp = 0; 44 | protected long period; 45 | 46 | // Construction /////////////////////////////////////////////////////////// 47 | 48 | public ModelAdapter(TimeSeries theMetric, long period, long firstTimeStamp) throws Exception { 49 | if (theMetric == null) { 50 | throw new Exception("The input metric is null."); 51 | } 52 | 53 | metric = theMetric; 54 | this.period = period; 55 | this.firstTimeStamp = firstTimeStamp; 56 | } 57 | 58 | public ModelAdapter(TimeSeries theMetric, long period) throws Exception { 59 | if (theMetric == null) { 60 | throw new Exception("The input metric is null."); 61 | } 62 | 63 | metric = theMetric; 64 | this.period = period; 65 | 66 | if (metric.data.size() > 0) { 67 | this.firstTimeStamp = metric.time(0); 68 | } 69 | } 70 | 71 | public ModelAdapter(String theMetric, long period) throws Exception { 72 | this.period = period; 73 | // TODO: 74 | // 1 - load the models related to theMetric from ModelDB 75 | // 2 - push the loaded models into 'models' 76 | // 3 - create a new TimeSeries for theMetric and set 'metric' 77 | // 4 - set 'firstTimeStamp' 78 | 79 | int modelNum = models.size(); 80 | for (int i = 0; i < modelNum; ++i) { 81 | isTrained.set(i, true); 82 | } 83 | } 84 | 85 | // Configuration Methods //////////////////////////////////////////////////////////////// 86 | 87 | public void setMetric(TimeSeries theMetric, long period) { 88 | metric = theMetric; 89 | this.period = period; 90 | 91 | if (metric.data.size() > 0) { 92 | this.firstTimeStamp = metric.time(0); 93 | } 94 | 95 | reset(); 96 | } 97 | 98 | public void setMetric(TimeSeries theMetric, long period, long firstTimeStamp) { 99 | metric = theMetric; 100 | this.period = period; 101 | this.firstTimeStamp = firstTimeStamp; 102 | reset(); 103 | } 104 | 105 | public void setMetric(String theMetric, long period) { 106 | this.period = period; 107 | firstTimeStamp = 0; 108 | models.clear(); 109 | isTrained.clear(); 110 | 111 | // TODO: 112 | // 1 - load the models related to theMetric from ModelDB 113 | // 2 - push the loaded models into 'models' 114 | // 3 - create a new TimeSeries for theMetric and set 'metric' 115 | // 4 - set 'firstTimeStamp' 116 | 117 | int modelNum = models.size(); 118 | for (int i = 0; i < modelNum; ++i) { 119 | isTrained.set(i, true); 120 | } 121 | } 122 | 123 | public void addModel(TimeSeriesModel model) { 124 | model.reset(); 125 | models.add(model); 126 | isTrained.add(false); 127 | } 128 | 129 | public String[] getModelNames() { 130 | String[] names = new String[models.size()]; 131 | for (int i = 0; i < models.size(); ++i) { 132 | names[i] = models.get(i).getModelName(); 133 | } 134 | 135 | return names; 136 | } 137 | 138 | // Algorithmic Methods //////////////////////////////////////////////////////////////////// 139 | 140 | public void reset() { 141 | int i = 0; 142 | for (TimeSeriesModel model : models) { 143 | model.reset(); 144 | isTrained.set(i, false); 145 | i++; 146 | } 147 | } 148 | 149 | public void train() throws Exception { 150 | int i = 0; 151 | 152 | metric.data.setLogicalIndices(firstTimeStamp, period); 153 | 154 | for (TimeSeriesModel model : models) { 155 | if (!isTrained.get(i)) { 156 | model.train(metric.data); 157 | isTrained.set(i, true); 158 | } 159 | i++; 160 | } 161 | } 162 | 163 | public void update(TimeSeries.DataSequence newData) throws Exception { 164 | if (newData == null) { 165 | throw new Exception("The input data sequence is null."); 166 | } 167 | 168 | for (Boolean b : isTrained) { 169 | if (!b) { 170 | throw new Exception("All the models need to be trained before updating."); 171 | } 172 | } 173 | 174 | if (newData.size() > 0) { 175 | newData.setLogicalIndices(firstTimeStamp, period); 176 | 177 | for (TimeSeriesModel model : models) { 178 | model.update(newData); 179 | } 180 | } 181 | } 182 | 183 | public ArrayList forecast(long from, long to) throws Exception { 184 | for (Boolean b : isTrained) { 185 | if (!b) { 186 | throw new Exception("All the models need to be trained before forecasting."); 187 | } 188 | } 189 | 190 | ArrayList result = new ArrayList(); 191 | 192 | for (TimeSeriesModel model : models) { 193 | TimeSeries.DataSequence sequence = null; 194 | if (period != -1) { 195 | sequence = new TimeSeries.DataSequence(from, to, period); 196 | sequence.setLogicalIndices(firstTimeStamp, period); 197 | } else { 198 | sequence = new TimeSeries.DataSequence(metric.data.getTimes(), metric.data.getValues()); 199 | } 200 | 201 | model.predict(sequence); 202 | result.add(sequence); 203 | } 204 | return result; 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/control/ProcessableObject.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // An interface for processable object. 8 | 9 | package com.yahoo.egads.control; 10 | 11 | public interface ProcessableObject { 12 | 13 | // The process() method instantiates the appropriate objects 14 | // depending on the concrete class implementation. 15 | public void process() throws Exception; 16 | 17 | // The result() method returns list of anomalies 18 | public Object result() throws Exception; 19 | 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/control/ProcessableObjectFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // A factory to create tasks based on the data and the config. 8 | 9 | package com.yahoo.egads.control; 10 | 11 | import com.yahoo.egads.data.TimeSeries; 12 | 13 | import java.lang.reflect.Constructor; 14 | import java.util.Properties; 15 | 16 | import com.yahoo.egads.models.adm.*; 17 | import com.yahoo.egads.models.tsmm.*; 18 | 19 | public class ProcessableObjectFactory { 20 | 21 | public static ProcessableObject create(TimeSeries ts, Properties config) { 22 | if (config.getProperty("OP_TYPE") == null) { 23 | throw new IllegalArgumentException("OP_TYPE is NULL"); 24 | } 25 | if (config.getProperty("OP_TYPE").equals("DETECT_ANOMALY")) { 26 | ModelAdapter ma = ProcessableObjectFactory.buildTSModel(ts, config); 27 | AnomalyDetector ad = ProcessableObjectFactory.buildAnomalyModel(ts, config); 28 | return (new DetectAnomalyProcessable(ma, ad, config)); 29 | } else if (config.getProperty("OP_TYPE").equals("UPDATE_MODEL")) { 30 | ModelAdapter ma = ProcessableObjectFactory.buildTSModel(ts, config); 31 | return (new UpdateModelProcessable(ma, ts.data, config)); 32 | } else if (config.getProperty("OP_TYPE").equals("TRANSFORM_INPUT")) { 33 | ModelAdapter ma = ProcessableObjectFactory.buildTSModel(ts, config); 34 | return (new TransformInputProcessable(ma, config)); 35 | } 36 | // Should not be here. 37 | System.err.println("Unknown OP_TYPE, returning UPDATE_MODEL ProcessableObject"); 38 | ModelAdapter ma = ProcessableObjectFactory.buildTSModel(ts, config); 39 | return (new UpdateModelProcessable(ma, ts.data, config)); 40 | } 41 | 42 | private static ModelAdapter buildTSModel(TimeSeries ts, Properties config) { 43 | ModelAdapter ma = null; 44 | try { 45 | Long period = (long) -1; 46 | if (config.getProperty("PERIOD") != null) { 47 | period = new Long(config.getProperty("PERIOD")); 48 | } 49 | if (period == 0) { 50 | if (ts.size() > 1) { 51 | period = ts.data.get(1).time - ts.data.get(0).time; 52 | } else { 53 | period = (long) 1; 54 | } 55 | } 56 | ma = new ModelAdapter(ts, period); 57 | String modelType = config.getProperty("TS_MODEL"); 58 | 59 | Class tsModelClass = Class.forName("com.yahoo.egads.models.tsmm." + modelType); 60 | Constructor constructor = tsModelClass.getConstructor(Properties.class); 61 | TimeSeriesAbstractModel m = (TimeSeriesAbstractModel) constructor.newInstance(config); 62 | ma.addModel(m); 63 | } catch (Exception e) { 64 | e.printStackTrace(); 65 | } 66 | return ma; 67 | } 68 | 69 | private static AnomalyDetector buildAnomalyModel(TimeSeries ts, Properties config) { 70 | AnomalyDetector ad = null; 71 | try { 72 | Long period = (long) -1; 73 | if (config.getProperty("PERIOD") != null) { 74 | period = new Long(config.getProperty("PERIOD")); 75 | } 76 | if (period == 0) { 77 | if (ts.size() > 1) { 78 | period = ts.data.get(1).time - ts.data.get(0).time; 79 | } else { 80 | period = (long) 1; 81 | } 82 | } 83 | ad = new AnomalyDetector(ts, period); 84 | String modelType = config.getProperty("AD_MODEL"); 85 | 86 | Class tsModelClass = Class.forName("com.yahoo.egads.models.adm." + modelType); 87 | Constructor constructor = tsModelClass.getConstructor(Properties.class); 88 | ad.addModel((AnomalyDetectionAbstractModel) constructor.newInstance(config)); 89 | } catch (Exception e) { 90 | e.printStackTrace(); 91 | } 92 | return ad; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/control/TransformInputProcessable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads.control; 8 | 9 | import java.util.ArrayList; 10 | 11 | import com.yahoo.egads.data.TimeSeries; 12 | 13 | import java.util.List; 14 | import java.util.Properties; 15 | 16 | public class TransformInputProcessable implements ProcessableObject { 17 | private ModelAdapter ma; 18 | private Properties config; 19 | private List forecastDatapointList; 20 | 21 | public List getForecastDatapointList() { 22 | return forecastDatapointList; 23 | } 24 | 25 | TransformInputProcessable(ModelAdapter ma, Properties config) { 26 | this.ma = ma; 27 | this.config = config; 28 | } 29 | 30 | public void process() throws Exception { 31 | 32 | // Reseting the models 33 | ma.reset(); 34 | 35 | // Training the model with the whole metric 36 | ma.train(); 37 | 38 | // Finding the expected values 39 | forecastDatapointList = ma.forecast(ma.metric.startTime(), ma.metric.lastTime()); 40 | 41 | String[] modelNames = ma.getModelNames(); 42 | 43 | int i = 0; 44 | // For each model's prediction in the ModelAdapter 45 | for (TimeSeries.DataSequence ds : forecastDatapointList) { 46 | int j = 0; 47 | for (TimeSeries.Entry e : ds) { 48 | System.out.println(e.time + "," + ma.metric.meta.name + "," + ma.metric.meta.fileName + "," 49 | + modelNames[i] + "," + e.value + "," + ma.metric.data.get(j).value + ",0"); 50 | j++; 51 | } 52 | i++; 53 | } 54 | } 55 | 56 | public Object result() throws Exception { 57 | return getForecastDatapointList(); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/control/UpdateModelProcessable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // A template for updateing the model given the data and the model config. 8 | 9 | package com.yahoo.egads.control; 10 | 11 | import com.yahoo.egads.data.TimeSeries; 12 | import java.util.Properties; 13 | 14 | public class UpdateModelProcessable implements ProcessableObject { 15 | 16 | private ModelAdapter ma; 17 | private TimeSeries.DataSequence newData; 18 | private Properties config; 19 | 20 | UpdateModelProcessable(ModelAdapter ma, TimeSeries.DataSequence newData, Properties config) { 21 | this.ma = ma; 22 | this.newData = newData; 23 | this.config = config; 24 | } 25 | 26 | public void process() throws Exception { 27 | this.ma.train(); 28 | this.ma.update(this.newData); 29 | } 30 | 31 | public Object result() throws Exception { 32 | return "Updated"; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/data/AnomalyErrorStorage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads.data; 8 | 9 | import com.yahoo.egads.data.TimeSeries.DataSequence; 10 | import java.util.Map; 11 | import java.util.HashMap; 12 | import java.util.ArrayList; 13 | 14 | public class AnomalyErrorStorage { 15 | 16 | // Denominator used in the MASE error metric. 17 | protected float maseDenom; 18 | // Maps error names to error indicies. 19 | protected Map errorToIndex; 20 | // Maps error index to error names. 21 | protected Map indexToError; 22 | boolean isInit = false; 23 | 24 | // Getter methods. 25 | public Map getErrorToIndex() { 26 | return errorToIndex; 27 | } 28 | public Map getIndexToError() { 29 | return indexToError; 30 | } 31 | 32 | // Force the user to define this constructor that acts as a 33 | // factory method. 34 | public AnomalyErrorStorage() { 35 | // Init error indicies that are filled in computeErrorMetrics method. 36 | errorToIndex = new HashMap(); 37 | errorToIndex.put("mapee", 0); 38 | errorToIndex.put("mae", 1); 39 | errorToIndex.put("smape", 2); 40 | errorToIndex.put("mape", 3); 41 | errorToIndex.put("mase", 4); 42 | indexToError = new HashMap(); 43 | indexToError.put(0, "mapee"); 44 | indexToError.put(1, "mae"); 45 | indexToError.put(2, "smape"); 46 | indexToError.put(3, "mape"); 47 | indexToError.put(4, "mase"); 48 | } 49 | 50 | // Initializes all anomaly errors. 51 | public HashMap> initAnomalyErrors(DataSequence observedSeries, DataSequence expectedSeries) { 52 | int n = observedSeries.size(); 53 | 54 | // init MASE. 55 | for (int i = 1; i < n; i++) { 56 | maseDenom += Math.abs(observedSeries.get(i).value - observedSeries.get(i - 1).value); 57 | } 58 | maseDenom = maseDenom / (n - 1); 59 | HashMap> allErrors = new HashMap>(); 60 | 61 | for (int i = 0; i < n; i++) { 62 | Float[] errors = computeErrorMetrics(expectedSeries.get(i).value, observedSeries.get(i).value); 63 | for (int j = 0; j < errors.length; j++) { 64 | if (!allErrors.containsKey(indexToError.get(j))) { 65 | allErrors.put(indexToError.get(j), new ArrayList()); 66 | } 67 | ArrayList tmp = allErrors.get(indexToError.get(j)); 68 | tmp.add(errors[j]); 69 | allErrors.put(indexToError.get(j), tmp); 70 | } 71 | } 72 | isInit = true; 73 | return allErrors; 74 | } 75 | 76 | // Computes the standard error metrics including MAE, sMAPE, MAPE, MASE. 77 | public Float[] computeErrorMetrics(float expected, float actual) { 78 | float div = expected; 79 | if (expected == (float) 0.0) { 80 | div = (float) 0.0000000001; 81 | } 82 | 83 | // Mean Absolute Error. 84 | float mae = Math.abs(actual - expected); 85 | // Symmetric Mean Absolute Error. 86 | float smape = (200 * Math.abs(actual - expected)) / ((Math.abs(actual) + Math.abs(expected)) == 0 ? (float) 1.0 : (float) (Math.abs(actual) + Math.abs(expected))); 87 | // Mean Absolute Percentage Error. 88 | float mape = Math.abs(actual) == 0 ? (float) 0.0 : ((100 * Math.abs(actual - expected)) / (float) Math.abs(actual)); 89 | // Mean Absolute Scaled Error. 90 | float mase = Math.abs(maseDenom) == 0.0 ? (float) 0.0 : Math.abs(actual - expected) / Math.abs(maseDenom); 91 | // Mean Absolute Percentage Error (scaled by the expected value). 92 | float mapee = (expected == actual) ? (float) 0.0 : Math.abs((100 * ((actual / div) - 1))); 93 | 94 | // Store all errors. 95 | Float[] errors = new Float[5]; 96 | errors[0] = mapee; 97 | errors[1] = mae; 98 | errors[2] = smape; 99 | errors[3] = mape; 100 | errors[4] = mase; 101 | 102 | return errors; 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/data/FileModelStore.java: -------------------------------------------------------------------------------- 1 | package com.yahoo.egads.data; 2 | 3 | import java.io.File; 4 | import java.io.FileInputStream; 5 | import java.io.FileOutputStream; 6 | import java.io.ObjectInputStream; 7 | import java.io.ObjectOutputStream; 8 | 9 | public class FileModelStore implements ModelStore { 10 | String path; 11 | public FileModelStore (String path) { 12 | File dir = new File (path); 13 | dir.mkdirs(); 14 | this.path = path; 15 | } 16 | 17 | @Override 18 | public void storeModel(String tag, Model m) { 19 | String filename = tag.replaceAll("[^\\w_-]", "_"); 20 | String fqn = path + "/" + filename; 21 | try { 22 | ObjectOutputStream o = new ObjectOutputStream(new FileOutputStream (fqn)); 23 | o.writeObject(m); 24 | o.close(); 25 | } catch (Exception e) { 26 | e.printStackTrace(); 27 | } 28 | } 29 | 30 | @Override 31 | public Model retrieveModel(String tag) { 32 | String filename = tag.replaceAll("[^\\w_-]", "_"); 33 | String fqn = path + "/" + filename; 34 | Model m = null; 35 | try { 36 | ObjectInputStream o = new ObjectInputStream(new FileInputStream(fqn)); 37 | m = (Model) o.readObject(); 38 | o.close(); 39 | } catch (Exception e) { 40 | e.printStackTrace(); 41 | } 42 | return m; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/data/JsonAble.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // interface 8 | // Implementing classes can use JsonEncoder to help implement these methods. 9 | 10 | package com.yahoo.egads.data; 11 | 12 | import org.json.JSONObject; 13 | import org.json.JSONStringer; 14 | 15 | public interface JsonAble { 16 | 17 | public void toJson(JSONStringer json_out) throws Exception; 18 | 19 | public void fromJson(JSONObject json_obj) throws Exception; 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/data/JsonEncoder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // function class 8 | // Helps any object encode its member variables as JSON. 9 | // Any JsonAble object can customize its encoding in its own to/fromJson() function 10 | 11 | package com.yahoo.egads.data; 12 | 13 | import java.lang.reflect.Constructor; 14 | import java.lang.reflect.Field; 15 | import java.lang.reflect.Modifier; 16 | import java.lang.reflect.ParameterizedType; 17 | import java.util.ArrayList; 18 | import java.util.Iterator; 19 | import org.json.JSONArray; 20 | import org.json.JSONStringer; 21 | import org.json.JSONObject; 22 | 23 | public class JsonEncoder { 24 | 25 | // methods //////////////////////////////////////////////// 26 | 27 | public static String toJson(Object object) throws Exception { 28 | JSONStringer jsonOut = new JSONStringer(); 29 | toJson(object, jsonOut); 30 | return jsonOut.toString(); 31 | } 32 | 33 | public static void // modifies json_out 34 | toJson(Object object, JSONStringer json_out) throws Exception { 35 | json_out.object(); 36 | // for each inherited class... 37 | for (Class c = object.getClass(); c != Object.class; c = c 38 | .getSuperclass()) { 39 | // for each member variable... 40 | Field[] fields = c.getDeclaredFields(); 41 | for (Field f : fields) { 42 | // if variable is static/private... skip it 43 | if (Modifier.isStatic(f.getModifiers())) { 44 | continue; 45 | } 46 | if (Modifier.isPrivate(f.getModifiers())) { 47 | continue; 48 | } 49 | Object value = f.get(object); 50 | 51 | // if variable is a complex type... recurse on sub-objects 52 | if (value instanceof JsonAble) { 53 | json_out.key(f.getName()); 54 | ((JsonAble) value).toJson(json_out); 55 | // if variable is an array... recurse on sub-objects 56 | } else if (value instanceof ArrayList) { 57 | json_out.key(f.getName()); 58 | json_out.array(); 59 | for (Object e : (ArrayList) value) { 60 | toJson(e, json_out); 61 | } 62 | json_out.endArray(); 63 | // if variable is a simple type... convert to json 64 | } else { 65 | json_out.key(f.getName()).value(value); 66 | } 67 | } 68 | } 69 | json_out.endObject(); 70 | } 71 | 72 | public static void fromJson(Object object, String json_str) 73 | throws Exception { 74 | JSONObject jsonObj = new JSONObject(json_str); 75 | fromJson(object, jsonObj); 76 | } 77 | 78 | public static void fromJson(Object object, JSONObject json_obj) 79 | throws Exception { 80 | // for each json key-value, that has a corresponding variable in object ... 81 | for (Iterator k = json_obj.keys(); k.hasNext();) { 82 | String key = (String) k.next(); 83 | Object value = json_obj.get(key); 84 | 85 | // try to access object variable 86 | Field field = null; 87 | try { 88 | field = object.getClass().getField(key); 89 | } catch (Exception e) { 90 | continue; 91 | } 92 | if (Modifier.isStatic(field.getModifiers())) { 93 | continue; 94 | } 95 | if (Modifier.isPrivate(field.getModifiers())) { 96 | continue; 97 | } 98 | Object member = field.get(object); 99 | 100 | if (json_obj.isNull(key)) { 101 | field.set(object, null); 102 | continue; 103 | // if variable is container... recurse 104 | } else if (member instanceof JsonAble) { 105 | ((JsonAble) member).fromJson((JSONObject) value); 106 | // if variable is an array... recurse on sub-objects 107 | } else if (member instanceof ArrayList) { 108 | // Depends on existance of ArrayList template parameter, and T constructor with no arguments. 109 | // May be better to use custom fromJson() in member class. 110 | ArrayList memberArray = (ArrayList) member; 111 | JSONArray jsonArray = (JSONArray) value; 112 | 113 | // find array element constructor 114 | ParameterizedType arrayType = null; 115 | if (field.getGenericType() instanceof ParameterizedType) { 116 | arrayType = (ParameterizedType) field.getGenericType(); 117 | } 118 | for (Class c = member.getClass(); arrayType == null 119 | && c != null; c = c.getSuperclass()) { 120 | if (c.getGenericSuperclass() instanceof ParameterizedType) { 121 | arrayType = (ParameterizedType) c 122 | .getGenericSuperclass(); 123 | } 124 | } 125 | if (arrayType == null) { 126 | throw new Exception( 127 | "could not find ArrayList element type for field 'key'"); 128 | } 129 | Class elementClass = (Class) (arrayType 130 | .getActualTypeArguments()[0]); 131 | Constructor elementConstructor = elementClass.getConstructor(); 132 | 133 | // for each element in JSON array ... append element to member array, recursively decode element 134 | for (int i = 0; i < jsonArray.length(); ++i) { 135 | Object element = elementConstructor.newInstance(); 136 | fromJson(element, jsonArray.getJSONObject(i)); 137 | memberArray.add(element); 138 | } 139 | // if variable is simple value... set 140 | } else if (field.getType() == float.class) { 141 | field.set(object, (float) json_obj.getDouble(key)); 142 | } else { 143 | field.set(object, value); 144 | } 145 | } 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/data/MetricMeta.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // data structure class 8 | // Contains meta-data about a metric. 9 | 10 | package com.yahoo.egads.data; 11 | 12 | import org.json.JSONObject; 13 | import org.json.JSONStringer; 14 | 15 | public class MetricMeta implements JsonAble { 16 | // member data //////////////////////////////////////////////// 17 | 18 | public String id; 19 | public boolean detectAnomalies = false; 20 | public String name; 21 | public String fileName; 22 | public String source; 23 | public String smoothing; 24 | public long[] seasons; 25 | 26 | // construction //////////////////////////////////////////////// 27 | 28 | public MetricMeta() { 29 | } 30 | 31 | public MetricMeta(String id_arg) { 32 | id = id_arg; 33 | } 34 | 35 | // methods //////////////////////////////////////////////// 36 | 37 | // display //////////////////////////////////////////////// 38 | 39 | public String toString() { 40 | StringBuffer str = new StringBuffer(); 41 | str.append("id=" + id); 42 | str.append(" detectAnomalies=" + detectAnomalies); 43 | str.append(" name=" + name); 44 | str.append(" source=" + source); 45 | return str.toString(); 46 | } 47 | 48 | public void toJson(JSONStringer json_out) throws Exception { 49 | JsonEncoder.toJson(this, json_out); 50 | } 51 | 52 | public void fromJson(JSONObject json_obj) throws Exception { 53 | JsonEncoder.fromJson(this, json_obj); 54 | } 55 | 56 | // test //////////////////////////////////////////////// 57 | 58 | // needed for unit tests 59 | public boolean equals(Object other_obj) { 60 | if (!(other_obj instanceof MetricMeta)) { 61 | return false; 62 | } 63 | MetricMeta other = (MetricMeta) other_obj; 64 | if (!equals(id, other.id)) { 65 | return false; 66 | } 67 | if (detectAnomalies != other.detectAnomalies) { 68 | return false; 69 | } 70 | if (!equals(name, other.name)) { 71 | return false; 72 | } 73 | if (!equals(source, other.source)) { 74 | return false; 75 | } 76 | return true; 77 | } 78 | 79 | public static boolean equals(Object o1, Object o2) { 80 | if (o1 == o2) { 81 | return true; 82 | } 83 | if (o1 == null && o2 != null) { 84 | return false; 85 | } 86 | if (o1 != null && o2 == null) { 87 | return false; 88 | } 89 | return o1.equals(o2); 90 | } 91 | 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/data/Model.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // interface 8 | // The generic model interface which other models implement (the TimeSeries model and the 9 | // AnomalyDetection model). 10 | 11 | package com.yahoo.egads.data; 12 | 13 | import java.io.Serializable; 14 | 15 | public interface Model extends JsonAble, Serializable { 16 | 17 | // resets the model. 18 | public void reset(); 19 | 20 | // Gets the model name and type 21 | public String getModelName(); 22 | public String getModelType(); 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/data/ModelStore.java: -------------------------------------------------------------------------------- 1 | package com.yahoo.egads.data; 2 | 3 | public interface ModelStore { 4 | public void storeModel(String tag, Model m); 5 | public Model retrieveModel (String tag); 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/data/WeightedValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | package com.yahoo.egads.data; 7 | 8 | import java.util.Collections; 9 | import java.util.List; 10 | 11 | import com.google.common.collect.Lists; 12 | 13 | /** 14 | * Class for holding a weighted value that can then be aggregated when stored 15 | * in a list using various functions. 16 | */ 17 | public class WeightedValue implements Comparable { 18 | private final double value; 19 | private final int weight; 20 | 21 | /** 22 | * Default ctor. 23 | * @param value The value of the data point. 24 | * @param weight A weight for the data point. 25 | */ 26 | public WeightedValue(final double value, final int weight) { 27 | this.value = value; 28 | this.weight = weight; 29 | } 30 | 31 | @Override 32 | public int compareTo(final WeightedValue other) { 33 | return Double.compare(value, other.value); 34 | } 35 | 36 | /** 37 | * Drops as many of the highest or lowest values as possible, leaving 38 | * at least one value in the list. 39 | * @param accumulator A non-null accumulator list. 40 | * @param count A count of 1 or more. 41 | * @param highest Drop higher values == true or drop lower values == false. 42 | */ 43 | public static void drop(final List accumulator, 44 | final int count, final boolean highest) { 45 | for (int x = 0; x < count; x++) { 46 | if (accumulator.size() <= 1) { 47 | break; 48 | } 49 | if (highest) { 50 | accumulator.remove(Collections.max(accumulator)); 51 | } else { 52 | accumulator.remove(Collections.min(accumulator)); 53 | } 54 | } 55 | } 56 | 57 | /** @return The data point value. */ 58 | public double getValue() { 59 | return value; 60 | } 61 | 62 | /** @return The weight for the data point. */ 63 | public int getWeight() { 64 | return weight; 65 | } 66 | 67 | /** 68 | * Aggregates the values in the list using the given agg function. 69 | * For all functions, NaNs are skipped so if an entire list is NaN'd or the 70 | * list is empty, the results will be a NaN. 71 | * @param values A non-null list of values to aggregate. 72 | * @param agg A non-null or empty aggregator function to use. 73 | * @return An aggregated value or NaN. 74 | * @throws IllegalArgumentException if the values was null, agg was null 75 | * or empty or we had an unimplemented agg function. 76 | */ 77 | public static double aggregate(final List values, 78 | final String agg) { 79 | if (agg == null || agg.isEmpty()) { 80 | throw new IllegalArgumentException("Aggregator cannot be null or empty"); 81 | } 82 | if (values == null) { 83 | throw new IllegalArgumentException("Values cannot be null"); 84 | } 85 | if (values.isEmpty()) { 86 | if (agg.equals("COUNT")) { 87 | return 0; 88 | } 89 | return Double.NaN; 90 | } 91 | 92 | // temps. 93 | int validCount = 0; 94 | double accumulator = 0; 95 | 96 | if (agg.equals("MAX")) { 97 | accumulator = Double.MIN_VALUE; 98 | for (final WeightedValue v : values) { 99 | if (Double.isFinite(v.value)) { 100 | if (v.value > accumulator) { 101 | accumulator = v.value; 102 | } 103 | ++validCount; 104 | } 105 | } 106 | if (validCount < 1) { 107 | return Double.NaN; 108 | } 109 | return accumulator; 110 | } else if (agg.equals("MIN")) { 111 | accumulator = Double.MAX_VALUE; 112 | for (final WeightedValue v : values) { 113 | if (Double.isFinite(v.value)) { 114 | if (v.value < accumulator) { 115 | accumulator = v.value; 116 | } 117 | ++validCount; 118 | } 119 | } 120 | if (validCount < 1) { 121 | return Double.NaN; 122 | } 123 | return accumulator; 124 | } else if (agg.equals("COUNT")) { 125 | int ctr = 0; 126 | for (final WeightedValue v : values) { 127 | if (Double.isFinite(v.value)) { 128 | ++ctr; 129 | } 130 | } 131 | return ctr; 132 | } else if (agg.equals("MEDIAN")) { 133 | final List sorted = Lists.newArrayList(); 134 | for (final WeightedValue v : values) { 135 | if (Double.isFinite(v.value)) { 136 | sorted.add(v.value); 137 | ++validCount; 138 | } 139 | } 140 | if (validCount < 1) { 141 | return Double.NaN; 142 | } 143 | Collections.sort(sorted); 144 | return sorted.get(sorted.size() / 2); 145 | } else if (agg.equals("SUM")) { 146 | for (final WeightedValue v : values) { 147 | if (Double.isFinite(v.value)) { 148 | accumulator += v.value; 149 | ++validCount; 150 | } 151 | } 152 | if (validCount < 1) { 153 | return Double.NaN; 154 | } 155 | return accumulator; 156 | } else if (agg.equals("AVG")) { 157 | for (final WeightedValue v : values) { 158 | if (Double.isFinite(v.value)) { 159 | accumulator += v.value; 160 | ++validCount; 161 | } 162 | } 163 | if (validCount < 1) { 164 | return Double.NaN; 165 | } 166 | return accumulator / validCount; 167 | } else if (agg.equals("WAVG")) { 168 | for (final WeightedValue v : values) { 169 | if (Double.isFinite(v.value)) { 170 | accumulator += v.weight * v.value; 171 | validCount += v.weight; 172 | } 173 | } 174 | if (validCount < 1) { 175 | return Double.NaN; 176 | } 177 | return accumulator / validCount; 178 | } 179 | 180 | throw new IllegalArgumentException("Unimplemented aggregation " 181 | + "function: " + agg); 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/adm/AnomalyDetectionAbstractModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads.models.adm; 8 | 9 | import java.util.Properties; 10 | 11 | import org.json.JSONObject; 12 | import org.json.JSONStringer; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.util.Map; 17 | import java.util.HashMap; 18 | 19 | import com.yahoo.egads.data.JsonEncoder; 20 | 21 | public abstract class AnomalyDetectionAbstractModel implements AnomalyDetectionModel { 22 | 23 | static final Logger logger = LoggerFactory.getLogger(AnomalyDetectionAbstractModel.class); 24 | protected float sDAutoSensitivity = 3; 25 | protected float amntAutoSensitivity = (float) 0.05; 26 | protected String outputDest = ""; 27 | protected String modelName; 28 | 29 | public String getModelName() { 30 | return modelName; 31 | } 32 | 33 | public String getModelType() { 34 | return "Anomaly"; 35 | } 36 | 37 | @Override 38 | public void toJson(JSONStringer json_out) throws Exception { 39 | JsonEncoder.toJson(this, json_out); 40 | } 41 | 42 | @Override 43 | public void fromJson(JSONObject json_obj) throws Exception { 44 | JsonEncoder.fromJson(this, json_obj); 45 | } 46 | 47 | protected String arrayF2S (Float[] input) { 48 | String ret = new String(); 49 | if (input.length == 0) { 50 | return ""; 51 | } 52 | if (input[0] == null) { 53 | ret = "Inf"; 54 | } else { 55 | ret = input[0].toString(); 56 | } 57 | for (int ix = 1; ix < input.length; ix++) { 58 | if (input[ix] == null) { 59 | ret += ":Inf"; 60 | } else { 61 | ret += ":" + input[ix].toString(); 62 | } 63 | } 64 | return ret; 65 | } 66 | 67 | // Parses the THRESHOLD config into a map. 68 | protected Map parseMap(String s) { 69 | if (s == null) { 70 | return new HashMap(); 71 | } 72 | String[] pairs = s.split(","); 73 | Map myMap = new HashMap(); 74 | for (int i = 0; i < pairs.length; i++) { 75 | String pair = pairs[i]; 76 | String[] keyValue = pair.split("#"); 77 | myMap.put(keyValue[0], Float.valueOf(keyValue[1])); 78 | } 79 | return myMap; 80 | } 81 | 82 | // Force the user to define this constructor that acts as a 83 | // factory method. 84 | public AnomalyDetectionAbstractModel(Properties config) { 85 | // Set the assumed amount of anomaly in your data. 86 | if (config.getProperty("AUTO_SENSITIVITY_ANOMALY_PCNT") != null) { 87 | this.amntAutoSensitivity = new Float(config.getProperty("AUTO_SENSITIVITY_ANOMALY_PCNT")); 88 | } 89 | // Set the standard deviation for auto sensitivity. 90 | if (config.getProperty("AUTO_SENSITIVITY_SD") != null) { 91 | this.sDAutoSensitivity = new Float(config.getProperty("AUTO_SENSITIVITY_SD")); 92 | } 93 | this.outputDest = config.getProperty("OUTPUT"); 94 | } 95 | 96 | @Override 97 | public boolean isDetectionWindowPoint(int maxHrsAgo, long windowStart, long anomalyTime, long startTime) { 98 | long unixTime = System.currentTimeMillis() / 1000L; 99 | // consider 'windowStart' if it is greater than or equal to first timestamp 100 | if (windowStart >= startTime) { 101 | return (anomalyTime - windowStart) > 0; 102 | } else { 103 | // use detection window as max hours specified 104 | return ((unixTime - anomalyTime) / 3600) < maxHrsAgo; 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/adm/AnomalyDetectionModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // interface 8 | 9 | package com.yahoo.egads.models.adm; 10 | 11 | import com.yahoo.egads.data.Anomaly; 12 | import com.yahoo.egads.data.TimeSeries; 13 | import com.yahoo.egads.data.Model; 14 | 15 | public interface AnomalyDetectionModel extends Model { 16 | // methods //////////////////////////////////////////////// 17 | 18 | // returns the type of anomalies detected by the model 19 | public String getType(); 20 | 21 | // tune the anomaly detection parameters based on the training data. 22 | public void tune(TimeSeries.DataSequence observedSeries, 23 | TimeSeries.DataSequence expectedSeries) throws Exception; 24 | 25 | // method to check whether the anomaly value is inside the 26 | // detection window or not 27 | public boolean isDetectionWindowPoint(int maxHrsAgo, 28 | long windowStart, 29 | long anomalyTime, 30 | long startTime); 31 | 32 | // detect anomalies. 33 | public Anomaly.IntervalSequence detect( 34 | TimeSeries.DataSequence observedSeries, 35 | TimeSeries.DataSequence expectedSeries) throws Exception; 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/adm/DBScanModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // A simple thresholding model that returns an anomaly if it is above/below a certain threashold. 8 | 9 | package com.yahoo.egads.models.adm; 10 | 11 | import java.util.Properties; 12 | import java.util.Map; 13 | import java.util.HashMap; 14 | import java.util.List; 15 | import java.util.ArrayList; 16 | 17 | import com.yahoo.egads.data.Anomaly.IntervalSequence; 18 | import com.yahoo.egads.data.Anomaly.Interval; 19 | import com.yahoo.egads.data.AnomalyErrorStorage; 20 | import com.yahoo.egads.data.TimeSeries.DataSequence; 21 | import com.yahoo.egads.utilities.DBSCANClusterer; 22 | 23 | import org.apache.commons.math3.ml.clustering.Cluster; 24 | 25 | import com.yahoo.egads.utilities.IdentifiedDoublePoint; 26 | 27 | import org.apache.commons.math3.ml.distance.EuclideanDistance; 28 | import org.json.JSONObject; 29 | import org.json.JSONStringer; 30 | 31 | public class DBScanModel extends AnomalyDetectionAbstractModel { 32 | 33 | // The constructor takes a set of properties 34 | // needed for the simple model. This includes the sensitivity. 35 | private Map threshold; 36 | private int maxHrsAgo; 37 | private long windowStart; 38 | // modelName. 39 | public String modelName = "DBScanModel"; 40 | public AnomalyErrorStorage aes = new AnomalyErrorStorage(); 41 | private DBSCANClusterer dbscan = null; 42 | private int minPoints = 2; 43 | private double eps = 500; 44 | 45 | public DBScanModel(Properties config) { 46 | super(config); 47 | 48 | if (config.getProperty("MAX_ANOMALY_TIME_AGO") == null) { 49 | throw new IllegalArgumentException("MAX_ANOMALY_TIME_AGO is NULL"); 50 | } 51 | this.maxHrsAgo = new Integer(config.getProperty("MAX_ANOMALY_TIME_AGO")); 52 | 53 | this.windowStart = new Long(config.getProperty("DETECTION_WINDOW_START_TIME")); 54 | 55 | this.threshold = parseMap(config.getProperty("THRESHOLD")); 56 | 57 | if (config.getProperty("THRESHOLD") != null && this.threshold.isEmpty() == true) { 58 | throw new IllegalArgumentException("THRESHOLD PARSE ERROR"); 59 | } 60 | } 61 | 62 | public void toJson(JSONStringer json_out) { 63 | 64 | } 65 | 66 | public void fromJson(JSONObject json_obj) { 67 | 68 | } 69 | 70 | public String getModelName() { 71 | return modelName; 72 | } 73 | 74 | @Override 75 | public String getType() { 76 | return "point_outlier"; 77 | } 78 | 79 | @Override 80 | public void reset() { 81 | // At this point, reset does nothing. 82 | } 83 | 84 | @Override 85 | public void tune(DataSequence observedSeries, 86 | DataSequence expectedSeries) throws Exception { 87 | // Compute the time-series of errors. 88 | HashMap> allErrors = aes.initAnomalyErrors(observedSeries, expectedSeries); 89 | List points = new ArrayList(); 90 | EuclideanDistance ed = new EuclideanDistance(); 91 | int n = observedSeries.size(); 92 | 93 | for (int i = 0; i < n; i++) { 94 | double[] d = new double[(aes.getIndexToError().keySet()).size()]; 95 | 96 | for (int e = 0; e < (aes.getIndexToError().keySet()).size(); e++) { 97 | d[e] = allErrors.get(aes.getIndexToError().get(e)).get(i); 98 | } 99 | points.add(new IdentifiedDoublePoint(d, i)); 100 | } 101 | 102 | double sum = 0.0; 103 | double count = 0.0; 104 | for (int i = 0; i < n; i++) { 105 | for (int j = 0; j < n; j++) { 106 | sum += ed.compute(points.get(i).getPoint(), points.get(j).getPoint()); 107 | count++; 108 | } 109 | } 110 | eps = ((double) this.sDAutoSensitivity) * (sum / count); 111 | minPoints = ((int) Math.ceil(((double) this.amntAutoSensitivity) * ((double) n))); 112 | dbscan = new DBSCANClusterer(eps, minPoints); 113 | } 114 | 115 | @Override 116 | public IntervalSequence detect(DataSequence observedSeries, 117 | DataSequence expectedSeries) throws Exception { 118 | 119 | IntervalSequence output = new IntervalSequence(); 120 | int n = observedSeries.size(); 121 | // Get an array of thresholds. 122 | Float[] thresholdErrors = new Float[aes.getErrorToIndex().size()]; 123 | for (Map.Entry entry : this.threshold.entrySet()) { 124 | thresholdErrors[aes.getErrorToIndex().get(entry.getKey())] = Math.abs(entry.getValue()); 125 | } 126 | 127 | // Compute the time-series of errors. 128 | HashMap> allErrors = aes.initAnomalyErrors(observedSeries, expectedSeries); 129 | List points = new ArrayList(); 130 | 131 | for (int i = 0; i < n; i++) { 132 | double[] d = new double[(aes.getIndexToError().keySet()).size()]; 133 | 134 | for (int e = 0; e < (aes.getIndexToError().keySet()).size(); e++) { 135 | d[e] = allErrors.get(aes.getIndexToError().get(e)).get(i); 136 | } 137 | points.add(new IdentifiedDoublePoint(d, i)); 138 | } 139 | 140 | List> cluster = dbscan.cluster(points); 141 | for(Cluster c: cluster) { 142 | for (IdentifiedDoublePoint p : c.getPoints()) { 143 | int i = p.getId(); 144 | Float[] errors = aes.computeErrorMetrics(expectedSeries.get(p.getId()).value, observedSeries.get(p.getId()).value); 145 | logger.debug("TS:" + observedSeries.get(i).time + ",E:" + arrayF2S(errors) + ",TE:" + arrayF2S(thresholdErrors) + ",OV:" + observedSeries.get(i).value + ",EV:" + expectedSeries.get(i).value); 146 | if (observedSeries.get(p.getId()).value != expectedSeries.get(p.getId()).value && 147 | (isDetectionWindowPoint(maxHrsAgo, windowStart, observedSeries.get(p.getId()).time, observedSeries.get(0).time) || 148 | (maxHrsAgo == 0 && p.getId() == (n - 1)))) { 149 | output.add(new Interval(observedSeries.get(p.getId()).time, 150 | p.getId(), 151 | errors, 152 | thresholdErrors, 153 | observedSeries.get(p.getId()).value, 154 | expectedSeries.get(p.getId()).value)); 155 | } 156 | } 157 | } 158 | 159 | return output; 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/adm/ExtremeLowDensityModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // A simple thresholding model that returns an anomaly if it is above/below a certain threashold. 8 | 9 | package com.yahoo.egads.models.adm; 10 | 11 | import java.util.Properties; 12 | import java.util.Map; 13 | import java.util.HashMap; 14 | import java.util.ArrayList; 15 | import com.yahoo.egads.data.Anomaly.IntervalSequence; 16 | import com.yahoo.egads.data.Anomaly.Interval; 17 | import com.yahoo.egads.data.AnomalyErrorStorage; 18 | import com.yahoo.egads.data.TimeSeries.DataSequence; 19 | import com.yahoo.egads.utilities.AutoSensitivity; 20 | 21 | import org.json.JSONObject; 22 | import org.json.JSONStringer; 23 | 24 | public class ExtremeLowDensityModel extends AnomalyDetectionAbstractModel { 25 | 26 | // The constructor takes a set of properties 27 | // needed for the simple model. This includes the sensitivity. 28 | private Map threshold; 29 | private int maxHrsAgo; 30 | private long windowStart; 31 | // modelName. 32 | public String modelName = "ExtremeLowDensityModel"; 33 | public AnomalyErrorStorage aes = new AnomalyErrorStorage(); 34 | 35 | public ExtremeLowDensityModel(Properties config) { 36 | super(config); 37 | 38 | if (config.getProperty("MAX_ANOMALY_TIME_AGO") == null) { 39 | throw new IllegalArgumentException("MAX_ANOMALY_TIME_AGO is NULL"); 40 | } 41 | this.maxHrsAgo = new Integer(config.getProperty("MAX_ANOMALY_TIME_AGO")); 42 | 43 | this.windowStart = new Long(config.getProperty("DETECTION_WINDOW_START_TIME")); 44 | 45 | this.threshold = parseMap(config.getProperty("THRESHOLD")); 46 | 47 | if (config.getProperty("THRESHOLD") != null && this.threshold.isEmpty() == true) { 48 | throw new IllegalArgumentException("THRESHOLD PARSE ERROR"); 49 | } 50 | } 51 | 52 | public void toJson(JSONStringer json_out) { 53 | 54 | } 55 | 56 | public void fromJson(JSONObject json_obj) { 57 | 58 | } 59 | 60 | public String getModelName() { 61 | return modelName; 62 | } 63 | 64 | @Override 65 | public String getType() { 66 | return "point_outlier"; 67 | } 68 | 69 | @Override 70 | public void reset() { 71 | // At this point, reset does nothing. 72 | } 73 | 74 | @Override 75 | public void tune(DataSequence observedSeries, 76 | DataSequence expectedSeries) throws Exception { 77 | // Compute the time-series of errors. 78 | HashMap> allErrors = aes.initAnomalyErrors(observedSeries, expectedSeries); 79 | 80 | for (int i = 0; i < (aes.getIndexToError().keySet()).size(); i++) { 81 | // Add a new error metric if the error metric has not been 82 | // defined by the user. 83 | if (!threshold.containsKey(aes.getIndexToError().get(i))) { 84 | Float[] fArray = (allErrors.get(aes.getIndexToError().get(i))).toArray(new Float[(allErrors.get(aes.getIndexToError().get(i))).size()]); 85 | threshold.put(aes.getIndexToError().get(i), AutoSensitivity.getLowDensitySensitivity(fArray, sDAutoSensitivity, amntAutoSensitivity)); 86 | } 87 | } 88 | } 89 | 90 | // Returns true this point is identified as a potential anomaly. 91 | public boolean isAnomaly(Float[] errors, Map threshold) { 92 | // Cycle through all available thresholds and return 93 | // true if any of them matches. 94 | for (Map.Entry entry : threshold.entrySet()) { 95 | // disable mapee and mape. 96 | if (aes.getErrorToIndex().containsKey(entry.getKey()) == true && 97 | Math.abs(errors[aes.getErrorToIndex().get(entry.getKey())]) >= Math.abs(entry.getValue())) { 98 | return true; 99 | } 100 | } 101 | return false; 102 | } 103 | 104 | @Override 105 | public IntervalSequence detect(DataSequence observedSeries, 106 | DataSequence expectedSeries) throws Exception { 107 | 108 | // At detection time, the anomaly thresholds shouldn't all be 0. 109 | Float threshSum = (float) 0.0; 110 | for (Map.Entry entry : this.threshold.entrySet()) { 111 | threshSum += Math.abs(entry.getValue()); 112 | } 113 | 114 | // Get an array of thresholds. 115 | Float[] thresholdErrors = new Float[aes.getErrorToIndex().size()]; 116 | for (Map.Entry entry : this.threshold.entrySet()) { 117 | thresholdErrors[aes.getErrorToIndex().get(entry.getKey())] = Math.abs(entry.getValue()); 118 | } 119 | 120 | IntervalSequence output = new IntervalSequence(); 121 | int n = observedSeries.size(); 122 | 123 | for (int i = 0; i < n; i++) { 124 | Float[] errors = aes.computeErrorMetrics(expectedSeries.get(i).value, observedSeries.get(i).value); 125 | logger.debug("TS:" + observedSeries.get(i).time + ",E:" + arrayF2S(errors) + ",TE:" + arrayF2S(thresholdErrors) + ",OV:" + observedSeries.get(i).value + ",EV:" + expectedSeries.get(i).value); 126 | if (observedSeries.get(i).value != expectedSeries.get(i).value && 127 | threshSum > (float) 0.0 && 128 | isAnomaly(errors, threshold) == true && 129 | (isDetectionWindowPoint(maxHrsAgo, windowStart, observedSeries.get(i).time, observedSeries.get(0).time) || 130 | (maxHrsAgo == 0 && i == (n - 1)))) { 131 | output.add(new Interval(observedSeries.get(i).time, 132 | i, 133 | errors, 134 | thresholdErrors, 135 | observedSeries.get(i).value, 136 | expectedSeries.get(i).value)); 137 | } 138 | } 139 | return output; 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/adm/KSigmaModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // A simple thresholding model that returns an anomaly if it is above/below a certain threashold. 8 | 9 | package com.yahoo.egads.models.adm; 10 | 11 | import java.util.Properties; 12 | import java.util.Map; 13 | import java.util.HashMap; 14 | import java.util.ArrayList; 15 | 16 | import com.yahoo.egads.data.Anomaly.IntervalSequence; 17 | import com.yahoo.egads.data.Anomaly.Interval; 18 | import com.yahoo.egads.data.TimeSeries.DataSequence; 19 | import com.yahoo.egads.utilities.AutoSensitivity; 20 | import com.yahoo.egads.data.AnomalyErrorStorage; 21 | 22 | import org.json.JSONObject; 23 | import org.json.JSONStringer; 24 | 25 | public class KSigmaModel extends AnomalyDetectionAbstractModel { 26 | 27 | // The constructor takes a set of properties 28 | // needed for the simple model. This includes the sensitivity. 29 | private Map threshold; 30 | private int maxHrsAgo; 31 | private long windowStart; 32 | // modelName. 33 | public String modelName = "KSigmaModel"; 34 | public AnomalyErrorStorage aes = new AnomalyErrorStorage(); 35 | 36 | public KSigmaModel(Properties config) { 37 | super(config); 38 | 39 | if (config.getProperty("MAX_ANOMALY_TIME_AGO") == null) { 40 | throw new IllegalArgumentException("MAX_ANOMALY_TIME_AGO is NULL"); 41 | } 42 | this.maxHrsAgo = new Integer(config.getProperty("MAX_ANOMALY_TIME_AGO")); 43 | 44 | this.windowStart = new Long(config.getProperty("DETECTION_WINDOW_START_TIME")); 45 | 46 | this.threshold = parseMap(config.getProperty("THRESHOLD")); 47 | 48 | if (config.getProperty("THRESHOLD") != null && this.threshold.isEmpty() == true) { 49 | throw new IllegalArgumentException("THRESHOLD PARSE ERROR"); 50 | } 51 | } 52 | 53 | public void toJson(JSONStringer json_out) { 54 | 55 | } 56 | 57 | public void fromJson(JSONObject json_obj) { 58 | 59 | } 60 | 61 | public String getModelName() { 62 | return modelName; 63 | } 64 | 65 | @Override 66 | public String getType() { 67 | return "point_outlier"; 68 | } 69 | 70 | @Override 71 | public void reset() { 72 | // At this point, reset does nothing. 73 | } 74 | 75 | @Override 76 | public void tune(DataSequence observedSeries, DataSequence expectedSeries) throws Exception { 77 | HashMap> allErrors = aes.initAnomalyErrors(observedSeries, expectedSeries); 78 | 79 | for (int i = 0; i < (aes.getIndexToError().keySet()).size(); i++) { 80 | // Add a new error metric if the error metric has not been 81 | // defined by the user. 82 | if (!threshold.containsKey(aes.getIndexToError().get(i))) { 83 | Float[] fArray = (allErrors.get(aes.getIndexToError().get(i))).toArray(new Float[(allErrors.get(aes.getIndexToError().get(i))).size()]); 84 | threshold.put(aes.getIndexToError().get(i), AutoSensitivity.getKSigmaSensitivity(fArray, sDAutoSensitivity)); 85 | } 86 | } 87 | } 88 | 89 | // Returns true this point is identified as a potential anomaly. 90 | public boolean isAnomaly(Float[] errors, Map threshold) { 91 | // Cycle through all available thresholds and return 92 | // true if any of them matches. 93 | for (Map.Entry entry : threshold.entrySet()) { 94 | // disable mapee and mape. 95 | if (aes.getErrorToIndex().containsKey(entry.getKey()) == true && 96 | Math.abs(errors[aes.getErrorToIndex().get(entry.getKey())]) >= Math.abs(entry.getValue())) { 97 | return true; 98 | } 99 | } 100 | return false; 101 | } 102 | 103 | @Override 104 | public IntervalSequence detect(DataSequence observedSeries, 105 | DataSequence expectedSeries) throws Exception { 106 | 107 | // At detection time, the anomaly thresholds shouldn't all be 0. 108 | Float threshSum = (float) 0.0; 109 | for (Map.Entry entry : this.threshold.entrySet()) { 110 | threshSum += Math.abs(entry.getValue()); 111 | } 112 | 113 | // Get an array of thresholds. 114 | Float[] thresholdErrors = new Float[aes.getErrorToIndex().size()]; 115 | for (Map.Entry entry : this.threshold.entrySet()) { 116 | thresholdErrors[aes.getErrorToIndex().get(entry.getKey())] = Math.abs(entry.getValue()); 117 | } 118 | 119 | IntervalSequence output = new IntervalSequence(); 120 | int n = observedSeries.size(); 121 | 122 | for (int i = 0; i < n; i++) { 123 | Float[] errors = aes.computeErrorMetrics(expectedSeries.get(i).value, observedSeries.get(i).value); 124 | logger.debug("TS:" + observedSeries.get(i).time + ",E:" + arrayF2S(errors) + ",TE:" + arrayF2S(thresholdErrors) + ",OV:" + observedSeries.get(i).value + ",EV:" + expectedSeries.get(i).value); 125 | if (observedSeries.get(i).value != expectedSeries.get(i).value && 126 | threshSum > (float) 0.0 && 127 | isAnomaly(errors, threshold) == true && 128 | (isDetectionWindowPoint(maxHrsAgo, windowStart, observedSeries.get(i).time, observedSeries.get(0).time) || 129 | (maxHrsAgo == 0 && i == (n - 1)))) { 130 | output.add(new Interval(observedSeries.get(i).time, 131 | i, 132 | errors, 133 | thresholdErrors, 134 | observedSeries.get(i).value, 135 | expectedSeries.get(i).value)); 136 | } 137 | } 138 | return output; 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/adm/NaiveModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // A simple model that does not require a forecasting model. 8 | // It looks weather or not the max value for the past N hours exceeds 9 | // X %. 10 | 11 | package com.yahoo.egads.models.adm; 12 | 13 | import java.util.Properties; 14 | 15 | import com.yahoo.egads.data.Anomaly.IntervalSequence; 16 | import com.yahoo.egads.data.Anomaly.Interval; 17 | import com.yahoo.egads.data.TimeSeries.DataSequence; 18 | import java.util.Map; 19 | import java.util.ArrayList; 20 | import java.util.HashMap; 21 | import com.yahoo.egads.data.AnomalyErrorStorage; 22 | import com.yahoo.egads.utilities.AutoSensitivity; 23 | 24 | import org.json.JSONObject; 25 | import org.json.JSONStringer; 26 | 27 | public class NaiveModel extends AnomalyDetectionAbstractModel { 28 | 29 | // The constructor takes a set of properties 30 | // needed for the simple model. This includes the sensitivity. 31 | private Map threshold; 32 | private int maxHrsAgo; 33 | private long windowStart; 34 | private Float window_size; 35 | // modelName. 36 | public static String modelName = "NaiveModel"; 37 | public AnomalyErrorStorage aes = new AnomalyErrorStorage(); 38 | 39 | public NaiveModel(Properties config) { 40 | super(config); 41 | 42 | if (config.getProperty("MAX_ANOMALY_TIME_AGO") == null) { 43 | throw new IllegalArgumentException("MAX_ANOMALY_TIME_AGO is NULL"); 44 | } 45 | this.maxHrsAgo = new Integer(config.getProperty("MAX_ANOMALY_TIME_AGO")); 46 | this.windowStart = new Long(config.getProperty("DETECTION_WINDOW_START_TIME")); 47 | if (config.getProperty("WINDOW_SIZE") == null) { 48 | throw new IllegalArgumentException("WINDOW_SIZE is NULL"); 49 | } 50 | this.window_size = new Float(config.getProperty("WINDOW_SIZE")); 51 | if (config.getProperty("THRESHOLD") == null) { 52 | throw new IllegalArgumentException("THRESHOLD is NULL"); 53 | } 54 | this.threshold = parseMap(config.getProperty("THRESHOLD")); 55 | if (config.getProperty("THRESHOLD") != null && this.threshold.isEmpty() == true) { 56 | throw new IllegalArgumentException("THRESHOLD PARSE ERROR"); 57 | } 58 | } 59 | 60 | public void toJson(JSONStringer json_out) { 61 | 62 | } 63 | 64 | public void fromJson(JSONObject json_obj) { 65 | 66 | } 67 | 68 | public String getModelName() { 69 | return modelName; 70 | } 71 | 72 | @Override 73 | public String getType() { 74 | return "point_outlier"; 75 | } 76 | 77 | @Override 78 | public void reset() { 79 | // At this point, reset does nothing. 80 | } 81 | 82 | @Override 83 | public void tune(DataSequence observedSeries, DataSequence expectedSeries) throws Exception { 84 | // TODO: auto detect thresholds. 85 | } 86 | // Returns true this point is identified as a potential anomaly. 87 | public boolean isAnomaly(Float[] errors, Map threshold) { 88 | // Cycle through all available thresholds and return 89 | // true if any of them matches. 90 | for (Map.Entry entry : threshold.entrySet()) { 91 | // disable mapee and mape. 92 | if (aes.getErrorToIndex().containsKey(entry.getKey()) == true && 93 | Math.abs(errors[aes.getErrorToIndex().get(entry.getKey())]) >= Math.abs(entry.getValue())) { 94 | return true; 95 | } 96 | } 97 | return false; 98 | } 99 | 100 | @Override 101 | public IntervalSequence detect(DataSequence observedSeries, 102 | DataSequence expectedSeries) throws Exception { 103 | 104 | // Get an array of thresholds. 105 | Float[] thresholdErrors = new Float[aes.getErrorToIndex().size()]; 106 | for (Map.Entry entry : this.threshold.entrySet()) { 107 | thresholdErrors[aes.getErrorToIndex().get(entry.getKey())] = Math.abs(entry.getValue()); 108 | } 109 | 110 | IntervalSequence output = new IntervalSequence(); 111 | int n = observedSeries.size(); 112 | Integer cutIndex = null; 113 | 114 | // Handle fractional windows which are interpreted as 115 | // % of the entire TimeSeries size. 116 | if (window_size < 1.0) { 117 | cutIndex = Math.round(window_size * ((float) n)); 118 | } else { 119 | cutIndex = Math.round(window_size); 120 | } 121 | 122 | if (cutIndex + 1 > n) { 123 | return output; 124 | } 125 | 126 | Float[] observed = new Float[] {observedSeries.get(0).value, observedSeries.get(0).value}; 127 | Float[] expected = new Float[] {expectedSeries.get(0).value, expectedSeries.get(0).value}; 128 | 129 | int maxIndex = 0; 130 | int minIndex = 0; 131 | 132 | int anomaly = 0; 133 | 134 | for (int k = 0; k < n; k++) { 135 | 136 | if (observed[0] < observedSeries.get(k).value) { 137 | observed[0] = observedSeries.get(k).value; 138 | maxIndex = k; 139 | anomaly = 1; 140 | } 141 | 142 | if (observed[1] > observedSeries.get(k).value) { 143 | observed[1] = observedSeries.get(k).value; 144 | minIndex = k; 145 | anomaly = 1; 146 | } 147 | 148 | if (k < cutIndex) { 149 | continue; 150 | } 151 | 152 | expected[0] = Math.max(expected[0], observedSeries.get(k - cutIndex).value); 153 | expected[1] = Math.min(expected[1], observedSeries.get(k - cutIndex).value); 154 | 155 | // Check for anomalies for min/max. 156 | int anomalyIndex = 0; 157 | for (int i = 0; i < 2; i++) { 158 | Float[] errors = aes.computeErrorMetrics(expected[i], observed[i]); 159 | boolean actualAnomaly = false; 160 | if (i == 0 && observed[i] > expected[i]) { 161 | actualAnomaly = true; 162 | anomalyIndex = maxIndex; 163 | } 164 | if (i == 1 && observed[i] < expected[i]) { 165 | actualAnomaly = true; 166 | anomalyIndex = minIndex; 167 | } 168 | 169 | if (isAnomaly(errors, threshold) == true && actualAnomaly == true && anomaly == 1 && 170 | (isDetectionWindowPoint(maxHrsAgo, windowStart, observedSeries.get(anomalyIndex).time, observedSeries.get(0).time) || 171 | (maxHrsAgo == 0 && i == (n - 1)))) { 172 | anomaly = 0; 173 | logger.debug("TS:" + observedSeries.get(anomalyIndex).time + ",E:" + arrayF2S(errors) + ",TH:" + arrayF2S(thresholdErrors) + ",OV:" + observedSeries.get(anomalyIndex).value + ",EV:" + expected[i]); 174 | output.add(new Interval(observedSeries.get(anomalyIndex).time, 175 | anomalyIndex, 176 | errors, 177 | thresholdErrors, 178 | observed[i], 179 | expected[i], 180 | isAnomaly(errors, threshold))); 181 | } 182 | } 183 | } 184 | return output; 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/adm/SimpleThresholdModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // A simple thresholding model that returns an anomaly if it is above/below a certain threashold. 8 | 9 | package com.yahoo.egads.models.adm; 10 | 11 | import java.util.Properties; 12 | 13 | import com.yahoo.egads.data.Anomaly.IntervalSequence; 14 | import com.yahoo.egads.data.Anomaly.Interval; 15 | import com.yahoo.egads.data.TimeSeries; 16 | import com.yahoo.egads.data.TimeSeries.DataSequence; 17 | import com.yahoo.egads.utilities.AutoSensitivity; 18 | import java.util.Map; 19 | import java.util.HashMap; 20 | 21 | import org.json.JSONObject; 22 | import org.json.JSONStringer; 23 | 24 | public class SimpleThresholdModel extends AnomalyDetectionAbstractModel { 25 | 26 | // The constructor takes a set of properties 27 | // needed for the simple model. This includes the sensitivity. 28 | private Map threshold; 29 | private int maxHrsAgo; 30 | private long windowStart; 31 | // Model name. 32 | private String modelName = "SimpleThresholdModel"; 33 | private String simpleThrType = "AdaptiveKSigmaSensitivity"; 34 | 35 | public SimpleThresholdModel(Properties config) { 36 | super(config); 37 | 38 | this.threshold = parseMap(config.getProperty("THRESHOLD")); 39 | this.maxHrsAgo = new Integer(config.getProperty("MAX_ANOMALY_TIME_AGO")); 40 | this.windowStart = new Long(config.getProperty("DETECTION_WINDOW_START_TIME")); 41 | if (config.getProperty("THRESHOLD") != null && this.threshold.isEmpty() == true) { 42 | throw new IllegalArgumentException("THRESHOLD PARSE ERROR"); 43 | } 44 | if (config.getProperty("SIMPLE_THRESHOLD_TYPE") != null) { 45 | simpleThrType = config.getProperty("SIMPLE_THRESHOLD_TYPE"); 46 | } 47 | } 48 | 49 | public void toJson(JSONStringer json_out) { 50 | 51 | } 52 | 53 | public void fromJson(JSONObject json_obj) { 54 | 55 | } 56 | 57 | @Override 58 | public String getType() { 59 | return "point_outlier"; 60 | } 61 | 62 | public String getModelName() { 63 | return modelName; 64 | } 65 | 66 | @Override 67 | public void reset() { 68 | } 69 | 70 | @Override 71 | public void tune(DataSequence observedSeries, DataSequence expectedSeries) throws Exception { 72 | Float thr[] = null; 73 | if (simpleThrType.equals("AdaptiveKSigmaSensitivity")) { 74 | thr = AutoSensitivity.getAdaptiveKSigmaSensitivity(observedSeries.getValues(), amntAutoSensitivity); 75 | } else { 76 | thr = AutoSensitivity.getAdaptiveMaxMinSigmaSensitivity(observedSeries.getValues(), amntAutoSensitivity, sDAutoSensitivity); 77 | } 78 | if (!threshold.containsKey("max")) { 79 | threshold.put("max", thr[0]); 80 | } 81 | if (!threshold.containsKey("min")) { 82 | threshold.put("min", thr[1]); 83 | } 84 | } 85 | 86 | @Override 87 | public IntervalSequence detect(DataSequence observedSeries, 88 | DataSequence expectedSeries) throws Exception { 89 | IntervalSequence output = new IntervalSequence(); 90 | Float[] thr = new Float[] {threshold.get("max"), threshold.get("min")}; 91 | int n = observedSeries.size(); 92 | for (int i = 0; i < n; i++) { 93 | TimeSeries.Entry entry = observedSeries.get(i); 94 | 95 | if (((thr[0] != null && entry.value >= thr[0]) || (thr[1] != null && entry.value <= thr[1])) && 96 | (isDetectionWindowPoint(maxHrsAgo, windowStart, entry.time, observedSeries.get(0).time) || (maxHrsAgo == 0 && i == (n - 1)))) { 97 | if (thr[0] != null && entry.value >= thr[0]) { 98 | output.add(new Interval(entry.time, i, null, thr, entry.value, thr[0])); 99 | } else { 100 | output.add(new Interval(entry.time, i, null, thr, entry.value, thr[1])); 101 | } 102 | } 103 | } 104 | 105 | return output; 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/tsmm/AutoForecastModel.java: -------------------------------------------------------------------------------- 1 | // 2 | // OpenForecast - open source, general-purpose forecasting package. 3 | // Copyright (C) 2002-2011 Steven R. Gould 4 | // 5 | // This library is free software; you can redistribute it and/or 6 | // modify it under the terms of the GNU Lesser General Public 7 | // License as published by the Free Software Foundation; either 8 | // version 2.1 of the License, or (at your option) any later version. 9 | // 10 | // This library is distributed in the hope that it will be useful, 11 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | // Lesser General Public License for more details. 14 | // 15 | // You should have received a copy of the GNU Lesser General Public 16 | // License along with this library; if not, write to the Free Software 17 | // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 | // 19 | 20 | // Olympic scoring model considers the average of the last k weeks 21 | // (dropping the b highest and lowest values) as the current prediction. 22 | 23 | package com.yahoo.egads.models.tsmm; 24 | 25 | import com.yahoo.egads.data.*; 26 | import org.json.JSONObject; 27 | import org.json.JSONStringer; 28 | import java.util.Properties; 29 | 30 | // Picks the best model from the available EGADS models. 31 | public class AutoForecastModel extends TimeSeriesAbstractModel { 32 | // Stores the properties file to init other models. 33 | private Properties p; 34 | 35 | // Stores the model. 36 | private TimeSeriesAbstractModel myModel = null; 37 | 38 | public AutoForecastModel(Properties config) { 39 | super(config); 40 | modelName = "AutoForecastModel"; 41 | this.p = config; 42 | } 43 | 44 | public void reset() { 45 | // At this point, reset does nothing. 46 | } 47 | 48 | public void train(TimeSeries.DataSequence data) { 49 | // Init all. 50 | OlympicModel olympModel = new OlympicModel(p); 51 | MovingAverageModel movingAvg = new MovingAverageModel(p); 52 | MultipleLinearRegressionModel mlReg = new MultipleLinearRegressionModel(p); 53 | NaiveForecastingModel naive = new NaiveForecastingModel(p); 54 | PolynomialRegressionModel poly = new PolynomialRegressionModel(p); 55 | RegressionModel regr = new RegressionModel(p); 56 | SimpleExponentialSmoothingModel simpleExp = new SimpleExponentialSmoothingModel(p); 57 | TripleExponentialSmoothingModel tripleExp = new TripleExponentialSmoothingModel(p); 58 | WeightedMovingAverageModel weightAvg = new WeightedMovingAverageModel(p); 59 | DoubleExponentialSmoothingModel doubleExp = new DoubleExponentialSmoothingModel(p); 60 | 61 | // Train all. 62 | olympModel.train(data); 63 | movingAvg.train(data); 64 | mlReg.train(data); 65 | naive.train(data); 66 | poly.train(data); 67 | regr.train(data); 68 | simpleExp.train(data); 69 | tripleExp.train(data); 70 | weightAvg.train(data); 71 | doubleExp.train(data); 72 | 73 | // Pick best. 74 | if (betterThan(olympModel, myModel)) { 75 | myModel = olympModel; 76 | } 77 | if (betterThan(movingAvg, myModel)) { 78 | myModel = movingAvg; 79 | } 80 | if (betterThan(mlReg, myModel)) { 81 | myModel = mlReg; 82 | } 83 | if (betterThan(naive, myModel)) { 84 | myModel = naive; 85 | } 86 | if (betterThan(poly, myModel)) { 87 | myModel = poly; 88 | } 89 | if (betterThan(regr, myModel)) { 90 | myModel = regr; 91 | } 92 | if (betterThan(simpleExp, myModel)) { 93 | myModel = simpleExp; 94 | } 95 | if (betterThan(tripleExp, myModel)) { 96 | myModel = tripleExp; 97 | } 98 | if (betterThan(weightAvg, myModel)) { 99 | myModel = weightAvg; 100 | } 101 | if (betterThan(doubleExp, myModel)) { 102 | myModel = doubleExp; 103 | } 104 | 105 | initForecastErrors(myModel, data); 106 | 107 | logger.debug(getBias() + "\t" + getMAD() + "\t" + getMAPE() + "\t" + getMSE() + "\t" + getSAE() + "\t" + 0 + "\t" + 0); 108 | } 109 | 110 | public void update(TimeSeries.DataSequence data) { 111 | 112 | } 113 | 114 | public String getModelName() { 115 | return modelName; 116 | } 117 | 118 | public void predict(TimeSeries.DataSequence sequence) throws Exception { 119 | myModel.predict(sequence); 120 | } 121 | 122 | public void toJson(JSONStringer json_out) { 123 | 124 | } 125 | 126 | public void fromJson(JSONObject json_obj) { 127 | 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/tsmm/DoubleExponentialSmoothingModel.java: -------------------------------------------------------------------------------- 1 | // 2 | // OpenForecast - open source, general-purpose forecasting package. 3 | // Copyright (C) 2002-2011 Steven R. Gould 4 | // 5 | // This library is free software; you can redistribute it and/or 6 | // modify it under the terms of the GNU Lesser General Public 7 | // License as published by the Free Software Foundation; either 8 | // version 2.1 of the License, or (at your option) any later version. 9 | // 10 | // This library is distributed in the hope that it will be useful, 11 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | // Lesser General Public License for more details. 14 | // 15 | // You should have received a copy of the GNU Lesser General Public 16 | // License along with this library; if not, write to the Free Software 17 | // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 | // 19 | 20 | // Olympic scoring model considers the average of the last k weeks 21 | // (dropping the b highest and lowest values) as the current prediction. 22 | 23 | package com.yahoo.egads.models.tsmm; 24 | 25 | import com.yahoo.egads.data.*; 26 | import com.yahoo.egads.data.TimeSeries.Entry; 27 | import org.json.JSONObject; 28 | import org.json.JSONStringer; 29 | import java.util.Properties; 30 | import net.sourceforge.openforecast.DataSet; 31 | import net.sourceforge.openforecast.ForecastingModel; 32 | import net.sourceforge.openforecast.DataPoint; 33 | import net.sourceforge.openforecast.Observation; 34 | import java.util.*; 35 | 36 | // Double exponential smoothing - also known as Holt exponential smoothing - is a refinement of the popular simple 37 | // exponential smoothing model but adds another component which takes into account any trend in the data. 38 | public class DoubleExponentialSmoothingModel extends TimeSeriesAbstractModel { 39 | // methods //////////////////////////////////////////////// 40 | 41 | // The model that will be used for forecasting. 42 | private ForecastingModel forecaster; 43 | 44 | // Stores the historical values. 45 | private TimeSeries.DataSequence data; 46 | 47 | public DoubleExponentialSmoothingModel(Properties config) { 48 | super(config); 49 | modelName = "DoubleExponentialSmoothingModel"; 50 | } 51 | 52 | public void reset() { 53 | // At this point, reset does nothing. 54 | } 55 | 56 | public void train(TimeSeries.DataSequence data) { 57 | this.data = data; 58 | int n = data.size(); 59 | DataPoint dp = null; 60 | DataSet observedData = new DataSet(); 61 | for (int i = 0; i < n; i++) { 62 | dp = new Observation(data.get(i).value); 63 | dp.setIndependentValue("x", i); 64 | observedData.add(dp); 65 | } 66 | observedData.setTimeVariable("x"); 67 | 68 | // TODO: Make weights configurable. 69 | forecaster = new net.sourceforge.openforecast.models.DoubleExponentialSmoothingModel(0.75, 0.1); 70 | forecaster.init(observedData); 71 | initForecastErrors(forecaster, data); 72 | 73 | logger.debug(getBias() + "\t" + getMAD() + "\t" + getMAPE() + "\t" + getMSE() + "\t" + getSAE() + "\t" + 0 + "\t" + 0); 74 | } 75 | 76 | public void update(TimeSeries.DataSequence data) { 77 | 78 | } 79 | 80 | public String getModelName() { 81 | return modelName; 82 | } 83 | 84 | public void predict(TimeSeries.DataSequence sequence) throws Exception { 85 | int n = data.size(); 86 | DataSet requiredDataPoints = new DataSet(); 87 | DataPoint dp; 88 | 89 | for (int count = 0; count < n; count++) { 90 | dp = new Observation(0.0); 91 | dp.setIndependentValue("x", count); 92 | requiredDataPoints.add(dp); 93 | } 94 | forecaster.forecast(requiredDataPoints); 95 | 96 | // Output the results 97 | Iterator it = requiredDataPoints.iterator(); 98 | int i = 0; 99 | while (it.hasNext()) { 100 | DataPoint pnt = ((DataPoint) it.next()); 101 | logger.info(data.get(i).time + "," + data.get(i).value + "," + pnt.getDependentValue()); 102 | sequence.set(i, (new Entry(data.get(i).time, (float) pnt.getDependentValue()))); 103 | i++; 104 | } 105 | } 106 | 107 | public void toJson(JSONStringer json_out) { 108 | 109 | } 110 | 111 | public void fromJson(JSONObject json_obj) { 112 | 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/tsmm/MovingAverageModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // Olympic scoring model considers the average of the last k weeks 8 | // (dropping the b highest and lowest values) as the current prediction. 9 | 10 | package com.yahoo.egads.models.tsmm; 11 | 12 | import com.yahoo.egads.data.*; 13 | import com.yahoo.egads.data.TimeSeries.Entry; 14 | import org.json.JSONObject; 15 | import org.json.JSONStringer; 16 | import java.util.Properties; 17 | import net.sourceforge.openforecast.DataSet; 18 | import net.sourceforge.openforecast.ForecastingModel; 19 | import net.sourceforge.openforecast.DataPoint; 20 | import net.sourceforge.openforecast.Observation; 21 | import java.util.*; 22 | 23 | // A moving average forecast model is based on an artificially constructed time series in which the value for a 24 | // given time period is replaced by the mean of that value and the values for some number of preceding and succeeding time periods. 25 | public class MovingAverageModel extends TimeSeriesAbstractModel { 26 | // methods //////////////////////////////////////////////// 27 | 28 | // The model that will be used for forecasting. 29 | private ForecastingModel forecaster; 30 | 31 | // Stores the historical values. 32 | private TimeSeries.DataSequence data; 33 | 34 | public MovingAverageModel(Properties config) { 35 | super(config); 36 | modelName = "MovingAverageModel"; 37 | } 38 | 39 | public void reset() { 40 | // At this point, reset does nothing. 41 | } 42 | 43 | public void train(TimeSeries.DataSequence data) { 44 | this.data = data; 45 | int n = data.size(); 46 | DataPoint dp = null; 47 | DataSet observedData = new DataSet(); 48 | for (int i = 0; i < n; i++) { 49 | dp = new Observation(data.get(i).value); 50 | dp.setIndependentValue("x", i); 51 | observedData.add(dp); 52 | } 53 | observedData.setTimeVariable("x"); 54 | 55 | // TODO: Make window configurable. 56 | forecaster = new net.sourceforge.openforecast.models.MovingAverageModel(2); 57 | forecaster.init(observedData); 58 | initForecastErrors(forecaster, data); 59 | 60 | logger.debug(getBias() + "\t" + getMAD() + "\t" + getMAPE() + "\t" + getMSE() + "\t" + getSAE() + "\t" + 0 + "\t" + 0); 61 | } 62 | 63 | public void update(TimeSeries.DataSequence data) { 64 | 65 | } 66 | 67 | public String getModelName() { 68 | return modelName; 69 | } 70 | 71 | public void predict(TimeSeries.DataSequence sequence) throws Exception { 72 | int n = data.size(); 73 | DataSet requiredDataPoints = new DataSet(); 74 | DataPoint dp; 75 | 76 | for (int count = 0; count < n; count++) { 77 | dp = new Observation(0.0); 78 | dp.setIndependentValue("x", count); 79 | requiredDataPoints.add(dp); 80 | } 81 | forecaster.forecast(requiredDataPoints); 82 | 83 | // Output the results 84 | Iterator it = requiredDataPoints.iterator(); 85 | int i = 0; 86 | while (it.hasNext()) { 87 | DataPoint pnt = ((DataPoint) it.next()); 88 | logger.info(data.get(i).time + "," + data.get(i).value + "," + pnt.getDependentValue()); 89 | sequence.set(i, (new Entry(data.get(i).time, (float) pnt.getDependentValue()))); 90 | i++; 91 | } 92 | } 93 | 94 | public void toJson(JSONStringer json_out) { 95 | 96 | } 97 | 98 | public void fromJson(JSONObject json_obj) { 99 | 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/tsmm/MultipleLinearRegressionModel.java: -------------------------------------------------------------------------------- 1 | // 2 | // OpenForecast - open source, general-purpose forecasting package. 3 | // Copyright (C) 2002-2011 Steven R. Gould 4 | // 5 | // This library is free software; you can redistribute it and/or 6 | // modify it under the terms of the GNU Lesser General Public 7 | // License as published by the Free Software Foundation; either 8 | // version 2.1 of the License, or (at your option) any later version. 9 | // 10 | // This library is distributed in the hope that it will be useful, 11 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | // Lesser General Public License for more details. 14 | // 15 | // You should have received a copy of the GNU Lesser General Public 16 | // License along with this library; if not, write to the Free Software 17 | // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 | // 19 | 20 | // Olympic scoring model considers the average of the last k weeks 21 | // (dropping the b highest and lowest values) as the current prediction. 22 | 23 | package com.yahoo.egads.models.tsmm; 24 | 25 | import com.yahoo.egads.data.*; 26 | import com.yahoo.egads.data.TimeSeries.Entry; 27 | import org.json.JSONObject; 28 | import org.json.JSONStringer; 29 | import java.util.Properties; 30 | import net.sourceforge.openforecast.DataSet; 31 | import net.sourceforge.openforecast.ForecastingModel; 32 | import net.sourceforge.openforecast.DataPoint; 33 | import net.sourceforge.openforecast.Observation; 34 | import java.util.*; 35 | 36 | // Implements a multiple variable linear regression model using the variables named in the constructor as 37 | // the independent variables, or the variables passed into one of the init methods. 38 | public class MultipleLinearRegressionModel extends TimeSeriesAbstractModel { 39 | // methods //////////////////////////////////////////////// 40 | 41 | // The model that will be used for forecasting. 42 | private ForecastingModel forecaster; 43 | 44 | // Stores the historical values. 45 | private TimeSeries.DataSequence data; 46 | 47 | public MultipleLinearRegressionModel(Properties config) { 48 | super(config); 49 | modelName = "MultipleLinearRegressionModel"; 50 | } 51 | 52 | public void reset() { 53 | // At this point, reset does nothing. 54 | } 55 | 56 | public void train(TimeSeries.DataSequence data) { 57 | this.data = data; 58 | int n = data.size(); 59 | DataPoint dp = null; 60 | DataSet observedData = new DataSet(); 61 | for (int i = 0; i < n; i++) { 62 | dp = new Observation(data.get(i).value); 63 | dp.setIndependentValue("x", i); 64 | observedData.add(dp); 65 | } 66 | observedData.setTimeVariable("x"); 67 | 68 | // TODO: Make configurable. 69 | forecaster = new net.sourceforge.openforecast.models.MultipleLinearRegressionModel(); 70 | forecaster.init(observedData); 71 | initForecastErrors(forecaster, data); 72 | 73 | logger.debug(getBias() + "\t" + getMAD() + "\t" + getMAPE() + "\t" + getMSE() + "\t" + getSAE() + "\t" + 0 + "\t" + 0); 74 | } 75 | 76 | public void update(TimeSeries.DataSequence data) { 77 | 78 | } 79 | 80 | public String getModelName() { 81 | return modelName; 82 | } 83 | 84 | public void predict(TimeSeries.DataSequence sequence) throws Exception { 85 | int n = data.size(); 86 | DataSet requiredDataPoints = new DataSet(); 87 | DataPoint dp; 88 | 89 | for (int count = 0; count < n; count++) { 90 | dp = new Observation(0.0); 91 | dp.setIndependentValue("x", count); 92 | requiredDataPoints.add(dp); 93 | } 94 | forecaster.forecast(requiredDataPoints); 95 | 96 | // Output the results 97 | Iterator it = requiredDataPoints.iterator(); 98 | int i = 0; 99 | while (it.hasNext()) { 100 | DataPoint pnt = ((DataPoint) it.next()); 101 | logger.info(data.get(i).time + "," + data.get(i).value + "," + pnt.getDependentValue()); 102 | sequence.set(i, (new Entry(data.get(i).time, (float) pnt.getDependentValue()))); 103 | i++; 104 | } 105 | } 106 | 107 | public void toJson(JSONStringer json_out) { 108 | 109 | } 110 | 111 | public void fromJson(JSONObject json_obj) { 112 | 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/tsmm/NaiveForecastingModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // Olympic scoring model considers the average of the last k weeks 8 | // (dropping the b highest and lowest values) as the current prediction. 9 | 10 | package com.yahoo.egads.models.tsmm; 11 | 12 | import com.yahoo.egads.data.*; 13 | import com.yahoo.egads.data.TimeSeries.Entry; 14 | import org.json.JSONObject; 15 | import org.json.JSONStringer; 16 | import java.util.Properties; 17 | import net.sourceforge.openforecast.DataSet; 18 | import net.sourceforge.openforecast.ForecastingModel; 19 | import net.sourceforge.openforecast.DataPoint; 20 | import net.sourceforge.openforecast.Observation; 21 | import java.util.*; 22 | 23 | // A naive forecasting model is a special case of the moving average forecasting model where the number of periods used for smoothing is 1. 24 | public class NaiveForecastingModel extends TimeSeriesAbstractModel { 25 | // methods //////////////////////////////////////////////// 26 | 27 | // The model that will be used for forecasting. 28 | private ForecastingModel forecaster; 29 | 30 | // Stores the historical values. 31 | private TimeSeries.DataSequence data; 32 | 33 | public NaiveForecastingModel(Properties config) { 34 | super(config); 35 | modelName = "NaiveForecastingModel"; 36 | } 37 | 38 | public void reset() { 39 | // At this point, reset does nothing. 40 | } 41 | 42 | public void train(TimeSeries.DataSequence data) { 43 | this.data = data; 44 | int n = data.size(); 45 | DataPoint dp = null; 46 | DataSet observedData = new DataSet(); 47 | for (int i = 0; i < n; i++) { 48 | dp = new Observation(data.get(i).value); 49 | dp.setIndependentValue("x", i); 50 | observedData.add(dp); 51 | } 52 | observedData.setTimeVariable("x"); 53 | 54 | forecaster = new net.sourceforge.openforecast.models.NaiveForecastingModel(); 55 | forecaster.init(observedData); 56 | initForecastErrors(forecaster, data); 57 | 58 | logger.debug(getBias() + "\t" + getMAD() + "\t" + getMAPE() + "\t" + getMSE() + "\t" + getSAE() + "\t" + 0 + "\t" + 0); 59 | } 60 | 61 | public void update(TimeSeries.DataSequence data) { 62 | 63 | } 64 | 65 | public String getModelName() { 66 | return modelName; 67 | } 68 | 69 | public void predict(TimeSeries.DataSequence sequence) throws Exception { 70 | int n = data.size(); 71 | DataSet requiredDataPoints = new DataSet(); 72 | DataPoint dp; 73 | 74 | for (int count = 0; count < n; count++) { 75 | dp = new Observation(0.0); 76 | dp.setIndependentValue("x", count); 77 | requiredDataPoints.add(dp); 78 | } 79 | forecaster.forecast(requiredDataPoints); 80 | 81 | // Output the results 82 | Iterator it = requiredDataPoints.iterator(); 83 | int i = 0; 84 | while (it.hasNext()) { 85 | DataPoint pnt = ((DataPoint) it.next()); 86 | logger.info(data.get(i).time + "," + data.get(i).value + "," + pnt.getDependentValue()); 87 | sequence.set(i, (new Entry(data.get(i).time, (float) pnt.getDependentValue()))); 88 | i++; 89 | } 90 | } 91 | 92 | public void toJson(JSONStringer json_out) { 93 | 94 | } 95 | 96 | public void fromJson(JSONObject json_obj) { 97 | 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/tsmm/NullModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // Null model, returns prediction the same as a prediction. 8 | 9 | package com.yahoo.egads.models.tsmm; 10 | 11 | import com.yahoo.egads.data.*; 12 | import com.yahoo.egads.data.TimeSeries.Entry; 13 | 14 | import java.util.Properties; 15 | import java.util.ArrayList; 16 | import java.util.Collections; 17 | 18 | import com.yahoo.egads.utilities.FileUtils; 19 | 20 | public class NullModel extends TimeSeriesAbstractModel { 21 | // methods //////////////////////////////////////////////// 22 | 23 | private static final long serialVersionUID = 1L; 24 | private TimeSeries.DataSequence data = null; 25 | 26 | public NullModel(Properties config) { 27 | super(config); 28 | } 29 | 30 | public void reset() { 31 | // At this point, reset does nothing. 32 | } 33 | 34 | public void train(TimeSeries.DataSequence data) { 35 | this.data = data; 36 | } 37 | 38 | public void update(TimeSeries.DataSequence data) { 39 | } 40 | 41 | public String getModelName() { 42 | return "NullModel"; 43 | } 44 | 45 | public void predict(TimeSeries.DataSequence sequence) throws Exception { 46 | int n = data.size(); 47 | for (int i = 0; i < n; i++) { 48 | sequence.set(i, (new Entry(data.get(i).time, (long) 0.0))); 49 | logger.info(data.get(i).time + "," + data.get(i).value + "," + data.get(i).value); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/tsmm/PolynomialRegressionModel.java: -------------------------------------------------------------------------------- 1 | // 2 | // OpenForecast - open source, general-purpose forecasting package. 3 | // Copyright (C) 2002-2011 Steven R. Gould 4 | // 5 | // This library is free software; you can redistribute it and/or 6 | // modify it under the terms of the GNU Lesser General Public 7 | // License as published by the Free Software Foundation; either 8 | // version 2.1 of the License, or (at your option) any later version. 9 | // 10 | // This library is distributed in the hope that it will be useful, 11 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | // Lesser General Public License for more details. 14 | // 15 | // You should have received a copy of the GNU Lesser General Public 16 | // License along with this library; if not, write to the Free Software 17 | // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 | // 19 | 20 | // Olympic scoring model considers the average of the last k weeks 21 | // (dropping the b highest and lowest values) as the current prediction. 22 | 23 | package com.yahoo.egads.models.tsmm; 24 | 25 | import com.yahoo.egads.data.*; 26 | import com.yahoo.egads.data.TimeSeries.Entry; 27 | import org.json.JSONObject; 28 | import org.json.JSONStringer; 29 | import java.util.Properties; 30 | import net.sourceforge.openforecast.DataSet; 31 | import net.sourceforge.openforecast.ForecastingModel; 32 | import net.sourceforge.openforecast.DataPoint; 33 | import net.sourceforge.openforecast.Observation; 34 | import java.util.*; 35 | 36 | // Implements a single variable polynomial regression model using the variable named in the constructor as the independent variable. 37 | public class PolynomialRegressionModel extends TimeSeriesAbstractModel { 38 | // methods //////////////////////////////////////////////// 39 | 40 | // The model that will be used for forecasting. 41 | private ForecastingModel forecaster; 42 | 43 | // Stores the historical values. 44 | private TimeSeries.DataSequence data; 45 | 46 | public PolynomialRegressionModel(Properties config) { 47 | super(config); 48 | modelName = "PolynomialRegressionModel"; 49 | } 50 | 51 | public void reset() { 52 | // At this point, reset does nothing. 53 | } 54 | 55 | public void train(TimeSeries.DataSequence data) { 56 | this.data = data; 57 | int n = data.size(); 58 | DataPoint dp = null; 59 | DataSet observedData = new DataSet(); 60 | for (int i = 0; i < n; i++) { 61 | dp = new Observation(data.get(i).value); 62 | dp.setIndependentValue("x", i); 63 | observedData.add(dp); 64 | } 65 | observedData.setTimeVariable("x"); 66 | 67 | // TODO: Make degrees configurable. 68 | forecaster = new net.sourceforge.openforecast.models.PolynomialRegressionModel("x", 3); 69 | forecaster.init(observedData); 70 | initForecastErrors(forecaster, data); 71 | 72 | logger.debug(getBias() + "\t" + getMAD() + "\t" + getMAPE() + "\t" + getMSE() + "\t" + getSAE() + "\t" + 0 + "\t" + 0); 73 | } 74 | 75 | public void update(TimeSeries.DataSequence data) { 76 | 77 | } 78 | 79 | public String getModelName() { 80 | return modelName; 81 | } 82 | 83 | public void predict(TimeSeries.DataSequence sequence) throws Exception { 84 | int n = data.size(); 85 | DataSet requiredDataPoints = new DataSet(); 86 | DataPoint dp; 87 | 88 | for (int count = 0; count < n; count++) { 89 | dp = new Observation(0.0); 90 | dp.setIndependentValue("x", count); 91 | requiredDataPoints.add(dp); 92 | } 93 | forecaster.forecast(requiredDataPoints); 94 | 95 | // Output the results 96 | Iterator it = requiredDataPoints.iterator(); 97 | int i = 0; 98 | while (it.hasNext()) { 99 | DataPoint pnt = ((DataPoint) it.next()); 100 | logger.info(data.get(i).time + "," + data.get(i).value + "," + pnt.getDependentValue()); 101 | sequence.set(i, (new Entry(data.get(i).time, (float) pnt.getDependentValue()))); 102 | i++; 103 | } 104 | } 105 | 106 | public void toJson(JSONStringer json_out) { 107 | 108 | } 109 | 110 | public void fromJson(JSONObject json_obj) { 111 | 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/tsmm/RegressionModel.java: -------------------------------------------------------------------------------- 1 | // 2 | // OpenForecast - open source, general-purpose forecasting package. 3 | // Copyright (C) 2002-2011 Steven R. Gould 4 | // 5 | // This library is free software; you can redistribute it and/or 6 | // modify it under the terms of the GNU Lesser General Public 7 | // License as published by the Free Software Foundation; either 8 | // version 2.1 of the License, or (at your option) any later version. 9 | // 10 | // This library is distributed in the hope that it will be useful, 11 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | // Lesser General Public License for more details. 14 | // 15 | // You should have received a copy of the GNU Lesser General Public 16 | // License along with this library; if not, write to the Free Software 17 | // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 | // 19 | 20 | // Olympic scoring model considers the average of the last k weeks 21 | // (dropping the b highest and lowest values) as the current prediction. 22 | 23 | package com.yahoo.egads.models.tsmm; 24 | 25 | import com.yahoo.egads.data.*; 26 | import com.yahoo.egads.data.TimeSeries.Entry; 27 | import org.json.JSONObject; 28 | import org.json.JSONStringer; 29 | import java.util.Properties; 30 | import net.sourceforge.openforecast.DataSet; 31 | import net.sourceforge.openforecast.ForecastingModel; 32 | import net.sourceforge.openforecast.DataPoint; 33 | import net.sourceforge.openforecast.Observation; 34 | import java.util.*; 35 | 36 | // Implements a single variable linear regression model using the variable named in the constructor as the independent variable. 37 | public class RegressionModel extends TimeSeriesAbstractModel { 38 | // methods //////////////////////////////////////////////// 39 | 40 | // The model that will be used for forecasting. 41 | private ForecastingModel forecaster; 42 | 43 | // Stores the historical values. 44 | private TimeSeries.DataSequence data; 45 | 46 | public RegressionModel(Properties config) { 47 | super(config); 48 | modelName = "RegressionModel"; 49 | } 50 | 51 | public void reset() { 52 | // At this point, reset does nothing. 53 | } 54 | 55 | public void train(TimeSeries.DataSequence data) { 56 | this.data = data; 57 | int n = data.size(); 58 | DataPoint dp = null; 59 | DataSet observedData = new DataSet(); 60 | for (int i = 0; i < n; i++) { 61 | dp = new Observation(data.get(i).value); 62 | dp.setIndependentValue("x", i); 63 | observedData.add(dp); 64 | } 65 | observedData.setTimeVariable("x"); 66 | 67 | forecaster = new net.sourceforge.openforecast.models.RegressionModel("x"); 68 | forecaster.init(observedData); 69 | initForecastErrors(forecaster, data); 70 | 71 | logger.debug(getBias() + "\t" + getMAD() + "\t" + getMAPE() + "\t" + getMSE() + "\t" + getSAE() + "\t" + 0 + "\t" + 0); 72 | } 73 | 74 | public void update(TimeSeries.DataSequence data) { 75 | 76 | } 77 | 78 | public String getModelName() { 79 | return modelName; 80 | } 81 | 82 | public void predict(TimeSeries.DataSequence sequence) throws Exception { 83 | int n = data.size(); 84 | DataSet requiredDataPoints = new DataSet(); 85 | DataPoint dp; 86 | 87 | for (int count = 0; count < n; count++) { 88 | dp = new Observation(0.0); 89 | dp.setIndependentValue("x", count); 90 | requiredDataPoints.add(dp); 91 | } 92 | forecaster.forecast(requiredDataPoints); 93 | 94 | // Output the results 95 | Iterator it = requiredDataPoints.iterator(); 96 | int i = 0; 97 | while (it.hasNext()) { 98 | DataPoint pnt = ((DataPoint) it.next()); 99 | logger.info(data.get(i).time + "," + data.get(i).value + "," + pnt.getDependentValue()); 100 | sequence.set(i, (new Entry(data.get(i).time, (float) pnt.getDependentValue()))); 101 | i++; 102 | } 103 | } 104 | 105 | public void toJson(JSONStringer json_out) { 106 | 107 | } 108 | 109 | public void fromJson(JSONObject json_obj) { 110 | 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/tsmm/SimpleExponentialSmoothingModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // Olympic scoring model considers the average of the last k weeks 8 | // (dropping the b highest and lowest values) as the current prediction. 9 | 10 | package com.yahoo.egads.models.tsmm; 11 | 12 | import com.yahoo.egads.data.*; 13 | import com.yahoo.egads.data.TimeSeries.Entry; 14 | import org.json.JSONObject; 15 | import org.json.JSONStringer; 16 | import java.util.Properties; 17 | import net.sourceforge.openforecast.DataSet; 18 | import net.sourceforge.openforecast.ForecastingModel; 19 | import net.sourceforge.openforecast.DataPoint; 20 | import net.sourceforge.openforecast.Observation; 21 | import java.util.*; 22 | 23 | // A simple exponential smoothing forecast model is a very popular model used to produce a smoothed Time Series. 24 | public class SimpleExponentialSmoothingModel extends TimeSeriesAbstractModel { 25 | // methods //////////////////////////////////////////////// 26 | 27 | // The model that will be used for forecasting. 28 | private ForecastingModel forecaster; 29 | 30 | // Stores the historical values. 31 | private TimeSeries.DataSequence data; 32 | 33 | public SimpleExponentialSmoothingModel(Properties config) { 34 | super(config); 35 | modelName = "SimpleExponentialSmoothingModel"; 36 | } 37 | 38 | public void reset() { 39 | // At this point, reset does nothing. 40 | } 41 | 42 | public void train(TimeSeries.DataSequence data) { 43 | this.data = data; 44 | int n = data.size(); 45 | DataPoint dp = null; 46 | DataSet observedData = new DataSet(); 47 | for (int i = 0; i < n; i++) { 48 | dp = new Observation(data.get(i).value); 49 | dp.setIndependentValue("x", i); 50 | observedData.add(dp); 51 | } 52 | observedData.setTimeVariable("x"); 53 | 54 | // TODO: Make weights configurable. 55 | forecaster = new net.sourceforge.openforecast.models.SimpleExponentialSmoothingModel(0.75); 56 | forecaster.init(observedData); 57 | initForecastErrors(forecaster, data); 58 | 59 | logger.debug(getBias() + "\t" + getMAD() + "\t" + getMAPE() + "\t" + getMSE() + "\t" + getSAE() + "\t" + 0 + "\t" + 0); 60 | } 61 | 62 | public void update(TimeSeries.DataSequence data) { 63 | 64 | } 65 | 66 | public String getModelName() { 67 | return modelName; 68 | } 69 | 70 | public void predict(TimeSeries.DataSequence sequence) throws Exception { 71 | int n = data.size(); 72 | DataSet requiredDataPoints = new DataSet(); 73 | DataPoint dp; 74 | 75 | for (int count = 0; count < n; count++) { 76 | dp = new Observation(0.0); 77 | dp.setIndependentValue("x", count); 78 | requiredDataPoints.add(dp); 79 | } 80 | forecaster.forecast(requiredDataPoints); 81 | 82 | // Output the results 83 | Iterator it = requiredDataPoints.iterator(); 84 | int i = 0; 85 | while (it.hasNext()) { 86 | DataPoint pnt = ((DataPoint) it.next()); 87 | logger.info(data.get(i).time + "," + data.get(i).value + "," + pnt.getDependentValue()); 88 | sequence.set(i, (new Entry(data.get(i).time, (float) pnt.getDependentValue()))); 89 | i++; 90 | } 91 | } 92 | 93 | public void toJson(JSONStringer json_out) { 94 | 95 | } 96 | 97 | public void fromJson(JSONObject json_obj) { 98 | 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/tsmm/SpectralSmoother.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads.models.tsmm; 8 | 9 | import java.util.Properties; 10 | import java.util.Hashtable; 11 | 12 | import com.yahoo.egads.data.TimeSeries.Entry; 13 | import com.yahoo.egads.data.TimeSeries.DataSequence; 14 | import com.yahoo.egads.utilities.SpectralMethods; 15 | import com.yahoo.egads.utilities.SpectralMethods.FilteringMethod; 16 | 17 | /** 18 | * SpectralSmoother implements the smoothing technique based on the Singular Value Decomposition (SVD) of the input time-series' Hankel matrix. 19 | * For further details on the methodology please refer to utilities/SpectralMethods.java documentation. 20 | * 21 | * The input parameters: 22 | * 1. 'WINDOW_SIZE' determines the size of the sliding window for spectral smoothing. Typically should be larger than the 23 | * largest seasonality present in the time-series. 24 | * 2. 'FILTERING_METHOD' determines the filtering method to be used by spectral smoothing. 25 | * Possible values: K_GAP, VARIANCE, EXPLICIT, SMOOTHNESS, EIGEN_RATIO, GAP_RATIO 26 | * Refer to utilities/SpectralMethods.java documentation for more details. 27 | * 3. 'FILTERING_PARAM' determines the tuning parameter for the specified filtering method. 28 | * Refer to utilities/SpectralMethods.java documentation for more details. 29 | * 30 | * @author amizadeh 31 | * 32 | */ 33 | 34 | public class SpectralSmoother extends TimeSeriesAbstractModel { 35 | 36 | protected Hashtable map = new Hashtable(); 37 | protected int windowSize; 38 | protected FilteringMethod method; 39 | protected double methodParameter; 40 | 41 | public SpectralSmoother(Properties config) { 42 | super(config); 43 | if (config.getProperty("FILTERING_METHOD") == null) { 44 | throw new IllegalArgumentException("FILTERING_METHOD is NULL"); 45 | } 46 | 47 | if (config.getProperty("WINDOW_SIZE") == null) { 48 | throw new IllegalArgumentException("WINDOW_SIZE is NULL"); 49 | } 50 | 51 | this.windowSize = new Integer(config.getProperty("WINDOW_SIZE")); 52 | this.method = FilteringMethod.valueOf(config.getProperty("FILTERING_METHOD")); 53 | 54 | if (config.getProperty("FILTERING_PARAM") == null) { 55 | switch (this.method) { 56 | case VARIANCE: 57 | this.methodParameter = 0.99; 58 | break; 59 | 60 | case SMOOTHNESS: 61 | this.methodParameter = 0.97; 62 | break; 63 | 64 | case K_GAP: 65 | this.methodParameter = 8; 66 | break; 67 | 68 | case EXPLICIT: 69 | this.methodParameter = 10; 70 | break; 71 | 72 | case EIGEN_RATIO: 73 | this.methodParameter = 0.1; 74 | break; 75 | 76 | case GAP_RATIO: 77 | this.methodParameter = 0.01; 78 | break; 79 | 80 | default: 81 | throw new IllegalArgumentException("Invalid FILTERING_METHOD value"); 82 | } 83 | } else { 84 | this.methodParameter = new Double(config.getProperty("FILTERING_PARAM")); 85 | } 86 | } 87 | 88 | @Override 89 | public void reset() { 90 | map.clear(); 91 | } 92 | 93 | @Override 94 | public void train(DataSequence data) throws Exception { 95 | this.reset(); 96 | DataSequence smoothedData = SpectralMethods.mFilter(data, windowSize, method, methodParameter); 97 | 98 | for (Entry e : smoothedData) { 99 | map.put(e.logicalIndex, e.value); 100 | } 101 | } 102 | 103 | @Override 104 | public void update(DataSequence data) throws Exception { 105 | 106 | DataSequence smoothedData = SpectralMethods.mFilter(data, windowSize, method, methodParameter); 107 | 108 | for (Entry e : smoothedData) { 109 | map.put(e.logicalIndex, e.value); 110 | } 111 | } 112 | 113 | @Override 114 | public void predict(DataSequence sequence) throws Exception { 115 | 116 | for (Entry e : sequence) { 117 | Float val = map.get(e.logicalIndex); 118 | e.value = (val == null) ? 0 : val; 119 | } 120 | } 121 | 122 | @Override 123 | public String getModelName() { 124 | return "SpectralSmoother"; 125 | } 126 | 127 | } 128 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/tsmm/TimeSeriesModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // interface 8 | 9 | package com.yahoo.egads.models.tsmm; 10 | 11 | import com.yahoo.egads.data.TimeSeries; 12 | import com.yahoo.egads.data.Model; 13 | 14 | public interface TimeSeriesModel extends Model { 15 | // methods //////////////////////////////////////////////// 16 | 17 | void train(TimeSeries.DataSequence data) throws Exception; 18 | 19 | public abstract void update(TimeSeries.DataSequence data) throws Exception; 20 | 21 | // predicts the values of the time series specified by the 'time' fields of the sequence and sets the 'value' fields of the sequence 22 | public abstract void predict(TimeSeries.DataSequence sequence) throws Exception; 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/tsmm/TripleExponentialSmoothingModel.java: -------------------------------------------------------------------------------- 1 | // 2 | // OpenForecast - open source, general-purpose forecasting package. 3 | // Copyright (C) 2002-2011 Steven R. Gould 4 | // 5 | // This library is free software; you can redistribute it and/or 6 | // modify it under the terms of the GNU Lesser General Public 7 | // License as published by the Free Software Foundation; either 8 | // version 2.1 of the License, or (at your option) any later version. 9 | // 10 | // This library is distributed in the hope that it will be useful, 11 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | // Lesser General Public License for more details. 14 | // 15 | // You should have received a copy of the GNU Lesser General Public 16 | // License along with this library; if not, write to the Free Software 17 | // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 | // 19 | 20 | // Olympic scoring model considers the average of the last k weeks 21 | // (dropping the b highest and lowest values) as the current prediction. 22 | 23 | package com.yahoo.egads.models.tsmm; 24 | 25 | import com.yahoo.egads.data.*; 26 | import com.yahoo.egads.data.TimeSeries.Entry; 27 | import org.json.JSONObject; 28 | import org.json.JSONStringer; 29 | import java.util.Properties; 30 | import net.sourceforge.openforecast.DataSet; 31 | import net.sourceforge.openforecast.ForecastingModel; 32 | import net.sourceforge.openforecast.DataPoint; 33 | import net.sourceforge.openforecast.Observation; 34 | import java.util.*; 35 | 36 | // Triple exponential smoothing - also known as the Winters method - is a refinement of the popular double exponential 37 | // smoothing model but adds another component which takes into account any seasonality - or periodicity - in the data. 38 | public class TripleExponentialSmoothingModel extends TimeSeriesAbstractModel { 39 | // methods //////////////////////////////////////////////// 40 | 41 | // The model that will be used for forecasting. 42 | private ForecastingModel forecaster; 43 | 44 | // Stores the historical values. 45 | private TimeSeries.DataSequence data; 46 | 47 | //Store the smoothing factors for level, trend and seasonality 48 | private final double alpha; 49 | private final double beta; 50 | private final double gamma; 51 | 52 | public TripleExponentialSmoothingModel(Properties config) { 53 | super(config); 54 | 55 | String temp = config.getProperty("ALPHA", "0.75"); 56 | if (temp == null || temp.isEmpty()) { 57 | throw new IllegalArgumentException("ALPHA is required, " 58 | + "e.g. 0.2 or 0.5"); 59 | } 60 | alpha = Double.parseDouble(temp); 61 | temp = config.getProperty("BETA", "0.001"); 62 | if (temp == null || temp.isEmpty()) { 63 | throw new IllegalArgumentException("BETA is required, " 64 | + "e.g. 0.2 or 0.5"); 65 | } 66 | beta = Double.parseDouble(temp); 67 | temp = config.getProperty("GAMMA", "0.001"); 68 | if (temp == null || temp.isEmpty()) { 69 | throw new IllegalArgumentException("GAMMA is required, " 70 | + "e.g. 0.2 or 0.5"); 71 | } 72 | gamma = Double.parseDouble(temp); 73 | modelName = "TripleExponentialSmoothingModel"; 74 | } 75 | 76 | public void reset() { 77 | // At this point, reset does nothing. 78 | } 79 | 80 | public void train(TimeSeries.DataSequence data) { 81 | this.data = data; 82 | int n = data.size(); 83 | DataPoint dp = null; 84 | DataSet observedData = new DataSet(); 85 | for (int i = 0; i < n; i++) { 86 | dp = new Observation(data.get(i).value); 87 | dp.setIndependentValue("x", i); 88 | observedData.add(dp); 89 | } 90 | observedData.setTimeVariable("x"); 91 | observedData.setPeriodsPerYear(12); 92 | 93 | forecaster = new net.sourceforge.openforecast.models.TripleExponentialSmoothingModel(alpha, beta, gamma); 94 | forecaster.init(observedData); 95 | initForecastErrors(forecaster, data); 96 | 97 | logger.debug(getBias() + "\t" + getMAD() + "\t" + getMAPE() + "\t" + getMSE() + "\t" + getSAE() + "\t" + 0 + "\t" + 0); 98 | } 99 | 100 | public void update(TimeSeries.DataSequence data) { 101 | 102 | } 103 | 104 | public String getModelName() { 105 | return modelName; 106 | } 107 | 108 | public void predict(TimeSeries.DataSequence sequence) throws Exception { 109 | int n = data.size(); 110 | DataSet requiredDataPoints = new DataSet(); 111 | DataPoint dp; 112 | 113 | for (int count = 0; count < n; count++) { 114 | dp = new Observation(0.0); 115 | dp.setIndependentValue("x", count); 116 | requiredDataPoints.add(dp); 117 | } 118 | forecaster.forecast(requiredDataPoints); 119 | 120 | // Output the results 121 | Iterator it = requiredDataPoints.iterator(); 122 | int i = 0; 123 | while (it.hasNext()) { 124 | DataPoint pnt = ((DataPoint) it.next()); 125 | logger.info(data.get(i).time + "," + data.get(i).value + "," + pnt.getDependentValue()); 126 | sequence.set(i, (new Entry(data.get(i).time, (float) pnt.getDependentValue()))); 127 | i++; 128 | } 129 | } 130 | 131 | public void toJson(JSONStringer json_out) { 132 | 133 | } 134 | 135 | public void fromJson(JSONObject json_obj) { 136 | 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/models/tsmm/WeightedMovingAverageModel.java: -------------------------------------------------------------------------------- 1 | // 2 | // OpenForecast - open source, general-purpose forecasting package. 3 | // Copyright (C) 2002-2011 Steven R. Gould 4 | // 5 | // This library is free software; you can redistribute it and/or 6 | // modify it under the terms of the GNU Lesser General Public 7 | // License as published by the Free Software Foundation; either 8 | // version 2.1 of the License, or (at your option) any later version. 9 | // 10 | // This library is distributed in the hope that it will be useful, 11 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | // Lesser General Public License for more details. 14 | // 15 | // You should have received a copy of the GNU Lesser General Public 16 | // License along with this library; if not, write to the Free Software 17 | // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 | // 19 | 20 | // Olympic scoring model considers the average of the last k weeks 21 | // (dropping the b highest and lowest values) as the current prediction. 22 | 23 | package com.yahoo.egads.models.tsmm; 24 | 25 | import com.yahoo.egads.data.*; 26 | import com.yahoo.egads.data.TimeSeries.Entry; 27 | import org.json.JSONObject; 28 | import org.json.JSONStringer; 29 | import java.util.Properties; 30 | import net.sourceforge.openforecast.DataSet; 31 | import net.sourceforge.openforecast.ForecastingModel; 32 | import net.sourceforge.openforecast.DataPoint; 33 | import net.sourceforge.openforecast.Observation; 34 | import java.util.*; 35 | 36 | // A weighted moving average forecast model is based on an artificially constructed time series in which the value 37 | // for a given time period is replaced by the weighted mean of that value and the values for some number of preceding time periods. 38 | public class WeightedMovingAverageModel extends TimeSeriesAbstractModel { 39 | // methods //////////////////////////////////////////////// 40 | 41 | // The model that will be used for forecasting. 42 | private ForecastingModel forecaster; 43 | 44 | // Stores the historical values. 45 | private TimeSeries.DataSequence data; 46 | 47 | public WeightedMovingAverageModel(Properties config) { 48 | super(config); 49 | modelName = "WeightedMovingAverageModel"; 50 | } 51 | 52 | public void reset() { 53 | // At this point, reset does nothing. 54 | } 55 | 56 | public void train(TimeSeries.DataSequence data) { 57 | this.data = data; 58 | int n = data.size(); 59 | DataPoint dp = null; 60 | DataSet observedData = new DataSet(); 61 | for (int i = 0; i < n; i++) { 62 | dp = new Observation(data.get(i).value); 63 | dp.setIndependentValue("x", i); 64 | observedData.add(dp); 65 | } 66 | observedData.setTimeVariable("x"); 67 | 68 | // TODO: Make weights configurable. 69 | forecaster = new net.sourceforge.openforecast.models.WeightedMovingAverageModel(new double[]{0.75, 0.25}); 70 | forecaster.init(observedData); 71 | 72 | initForecastErrors(forecaster, data); 73 | 74 | logger.debug(getBias() + "\t" + getMAD() + "\t" + getMAPE() + "\t" + getMSE() + "\t" + getSAE() + "\t" + 0 + "\t" + 0); 75 | } 76 | 77 | public void update(TimeSeries.DataSequence data) { 78 | 79 | } 80 | 81 | public String getModelName() { 82 | return modelName; 83 | } 84 | 85 | public void predict(TimeSeries.DataSequence sequence) throws Exception { 86 | int n = data.size(); 87 | DataSet requiredDataPoints = new DataSet(); 88 | DataPoint dp; 89 | 90 | for (int count = 0; count < n; count++) { 91 | dp = new Observation(0.0); 92 | dp.setIndependentValue("x", count); 93 | requiredDataPoints.add(dp); 94 | } 95 | forecaster.forecast(requiredDataPoints); 96 | 97 | // Output the results 98 | Iterator it = requiredDataPoints.iterator(); 99 | int i = 0; 100 | while (it.hasNext()) { 101 | DataPoint pnt = ((DataPoint) it.next()); 102 | logger.info(data.get(i).time + "," + data.get(i).value + "," + pnt.getDependentValue()); 103 | sequence.set(i, (new Entry(data.get(i).time, (float) pnt.getDependentValue()))); 104 | i++; 105 | } 106 | } 107 | 108 | public void toJson(JSONStringer json_out) { 109 | 110 | } 111 | 112 | public void fromJson(JSONObject json_obj) { 113 | 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/utilities/AutoSensitivity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // Provides the auto-sensitivity solution for EGADS 8 | // using the bootstrapping framework. 9 | 10 | package com.yahoo.egads.utilities; 11 | import java.util.ArrayList; 12 | import org.apache.commons.lang.ArrayUtils; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.util.Arrays; 17 | import java.util.Collections; 18 | 19 | public class AutoSensitivity { 20 | private static final Logger logger = LoggerFactory.getLogger(AutoSensitivity.class); 21 | 22 | // Computes sensitivity based on the density distribution. 23 | // Assumes that anomalies constitute at most 5% of the data. 24 | public static Float getLowDensitySensitivity(Float[] data, float sDAutoSensitivy, float amntAutoSensitivity) { 25 | Float toReturn = Float.POSITIVE_INFINITY; 26 | Arrays.sort(data, Collections.reverseOrder()); 27 | while (data.length > 0) { 28 | 29 | ArrayList fData = new ArrayList(); 30 | fData.add(data[0]); 31 | data = ((Float[]) ArrayUtils.remove(data, 0)); 32 | 33 | Float centroid = (float) fData.get(0); 34 | Float maxDelta = (float) sDAutoSensitivy * StatsUtils.getSD(data, StatsUtils.getMean(data)); 35 | 36 | logger.debug("AutoSensitivity: Adding: " + fData.get(0) + " SD: " + maxDelta); 37 | 38 | // Add points while it's in the same cluster or not part of the other cluster. 39 | String localDebug = null; 40 | while (data.length > 0 && 41 | (centroid - data[0]) <= ((float) (maxDelta))) { 42 | float maxDeltaInit = maxDelta; 43 | fData.add(data[0]); 44 | data = ((Float[]) ArrayUtils.remove(data, 0)); 45 | Float[] tmp = new Float[fData.size()]; 46 | tmp = fData.toArray(tmp); 47 | centroid = StatsUtils.getMean(tmp); 48 | 49 | if (data.length > 0) { 50 | Float sdOtherCluster = (float) StatsUtils.getSD(data, StatsUtils.getMean(data)); 51 | maxDelta = sDAutoSensitivy * sdOtherCluster; 52 | logger.debug("AutoSensitivity: Adding: " + data[0] + " SD: " + maxDeltaInit + " SD': " + maxDelta); 53 | } 54 | } 55 | if (data.length > 0) { 56 | logger.debug("AutoSensitivity: Next Point I would have added is " + data[0]); 57 | } 58 | 59 | if (((double) fData.size() / (double) data.length) > amntAutoSensitivity) { 60 | // Cannot do anomaly detection. 61 | logger.debug("AutoSensitivity: Returning " + toReturn + " data size: " + data.length + " fData.size: " + fData.size()); 62 | return toReturn; 63 | } 64 | 65 | toReturn = fData.get(fData.size() - 1); 66 | logger.debug("AutoSensitivity: Updating toReturn: " + toReturn + " SD: " + maxDelta); 67 | return toReturn; 68 | } 69 | return toReturn; 70 | } 71 | 72 | // Uses the simple KSigma rule to get the anoamly sensitivity. 73 | // Assumes that we have a normal distribution. 74 | public static Float getKSigmaSensitivity(Float[] data, float sDAutoSensitivity) { 75 | Float mean = StatsUtils.getMean(data); 76 | Float sd = StatsUtils.getSD(data, mean); 77 | return (mean + (sd * sDAutoSensitivity)); 78 | } 79 | 80 | // Uses the mean as the base to find the static threshold. 81 | public static Float[] getAdaptiveKSigmaSensitivity(Float[] data, float amntAutoSens) { 82 | Float mean = StatsUtils.getMean(data); 83 | Float sd = StatsUtils.getSD(data, mean); 84 | if (sd == (float) 0.0) { 85 | sd = (float) 1.0; 86 | } 87 | Float[] ret = null; 88 | float k = (float) 1; 89 | float incr = (float) 1; 90 | 91 | Float max = null; 92 | Float min = null; 93 | float thresh = mean + Math.abs(sd * k); 94 | int howMany = howManyGreater(data, thresh); 95 | 96 | while (((float) howMany / (float) data.length) > amntAutoSens) { 97 | k += incr; 98 | thresh = mean + Math.abs(sd * k); 99 | howMany = howManyGreater(data, thresh); 100 | } 101 | if (((float) howMany / (float) data.length) <= amntAutoSens) { 102 | max = thresh; 103 | } 104 | k = 1; 105 | thresh = mean - Math.abs(sd * k); 106 | howMany = howManyLess(data, thresh); 107 | 108 | while (((float) howMany / (float) data.length) > amntAutoSens) { 109 | k += incr; 110 | thresh = mean - Math.abs(sd * k); 111 | howMany = howManyLess(data, thresh); 112 | } 113 | if (((float) howMany / (float) data.length) <= amntAutoSens) { 114 | min = thresh; 115 | } 116 | ret = new Float[]{max, min}; 117 | return ret; 118 | } 119 | 120 | // Uses the max/min as the base to find the static threshold. 121 | public static Float[] getAdaptiveMaxMinSigmaSensitivity(Float[] data, float amntAutoSens, float k) { 122 | Arrays.sort(data); 123 | Float mean = StatsUtils.getMean(data); 124 | Float sd = StatsUtils.getSD(data, mean); 125 | if (sd == (float) 0.0) { 126 | sd = (float) 1.0; 127 | } 128 | Float[] ret = null; 129 | 130 | Float max = null; 131 | Float min = null; 132 | int i = 0; 133 | float thresh = data[i] + Math.abs(sd * k); 134 | int howMany = howManyLess(data, thresh); 135 | while (((float) howMany / (float) data.length) <= amntAutoSens) { 136 | min = thresh; 137 | i++; 138 | thresh = data[i] + Math.abs(sd * k); 139 | howMany = howManyLess(data, thresh); 140 | } 141 | i = data.length - 1; 142 | thresh = data[i] - Math.abs(sd * k); 143 | howMany = howManyGreater(data, thresh); 144 | while (((float) howMany / (float) data.length) <= amntAutoSens) { 145 | max = thresh; 146 | i--; 147 | thresh = data[i] - Math.abs(sd * k); 148 | howMany = howManyGreater(data, thresh); 149 | } 150 | 151 | ret = new Float[]{max, min}; 152 | return ret; 153 | } 154 | 155 | private static int howManyGreater(Float[] data, Float value) { 156 | int numgreater = 0; 157 | for (Float f : data) { 158 | if (value <= f) { 159 | numgreater++; 160 | } 161 | } 162 | return numgreater; 163 | } 164 | 165 | private static int howManyLess(Float[] data, Float value) { 166 | int numless = 0; 167 | for (Float f : data) { 168 | if (value >= f) { 169 | numless++; 170 | } 171 | } 172 | return numless; 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/utilities/DateTimeCalculator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // Utility class for date/time formatting and conversion 8 | 9 | package com.yahoo.egads.utilities; 10 | 11 | import java.text.ParseException; 12 | import java.text.SimpleDateFormat; 13 | import java.util.*; 14 | 15 | public class DateTimeCalculator { 16 | // Fields ////////////////////////////////////////////////////////////////////////////////// 17 | private static Calendar cal = Calendar.getInstance(); 18 | private static SimpleDateFormat[] formatters = new SimpleDateFormat[] { 19 | new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"), 20 | new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SS"), 21 | new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S"), 22 | new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS"), 23 | new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SS"), 24 | new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:S"), 25 | new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"), 26 | new SimpleDateFormat("yyyy-MM-dd") }; 27 | private static SimpleDateFormat outputFormatter = new SimpleDateFormat( 28 | "yyyy-MM-dd HH:mm:ss"); 29 | 30 | // Static Methods ////////////////////////////////////////////////////////////////////////// 31 | 32 | // Time in string --> Date object 33 | public static Date getDate(String time) { 34 | Date date = null; 35 | for (int i = 0; i < formatters.length; ++i) { 36 | try { 37 | date = formatters[i].parse(time); 38 | break; 39 | } catch (ParseException e) { 40 | 41 | } 42 | } 43 | 44 | return date; 45 | } 46 | 47 | // You can just write "new Date ( timeInMilliSeconds )" 48 | // Milliseconds --> Date object 49 | public static Date getDate(long timeInMilliSeconds) { 50 | cal.setTimeInMillis(timeInMilliSeconds); 51 | return cal.getTime(); 52 | } 53 | 54 | // Time in string --> Milliseconds 55 | public static long getMilliSeconds(String time) { 56 | Date date = getDate(time); 57 | cal.setTime(date); 58 | return cal.getTimeInMillis(); 59 | } 60 | 61 | // Output the input time series into the standard output format 62 | public static String format(String time) { 63 | return outputFormatter.format(getDate(time)); 64 | } 65 | 66 | // Milliseconds --> Time in string 67 | public static String format(long timeInMilliSeconds) { 68 | cal.setTimeInMillis(timeInMilliSeconds); 69 | return outputFormatter.format(cal.getTime()); 70 | } 71 | 72 | // Add 'amount' to the specified time's field 73 | public static String add(String time, int field, int amount) { 74 | Date date = getDate(time); 75 | 76 | cal.setTime(date); 77 | cal.add(field, amount); 78 | date = cal.getTime(); 79 | 80 | return outputFormatter.format(date); 81 | } 82 | 83 | // Add 'amount' to the specified time's field 84 | public static long add(long timeInMilliSeconds, int field, int amount) { 85 | cal.setTimeInMillis(timeInMilliSeconds); 86 | cal.add(field, amount); 87 | return cal.getTimeInMillis(); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/utilities/FileInputProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads.utilities; 8 | 9 | // Class that implements EGADS file input processing. 10 | 11 | import com.yahoo.egads.control.ProcessableObject; 12 | import com.yahoo.egads.control.ProcessableObjectFactory; 13 | import java.util.Properties; 14 | import com.yahoo.egads.data.TimeSeries; 15 | import java.util.ArrayList; 16 | 17 | public class FileInputProcessor implements InputProcessor { 18 | 19 | private String file = null; 20 | 21 | public FileInputProcessor(String file) { 22 | this.file = file; 23 | } 24 | 25 | public void processInput(Properties p) throws Exception { 26 | // Parse the input timeseries. 27 | ArrayList metrics = com.yahoo.egads.utilities.FileUtils 28 | .createTimeSeries(this.file, p); 29 | for (TimeSeries ts : metrics) { 30 | ProcessableObject po = ProcessableObjectFactory.create(ts, p); 31 | po.process(); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/utilities/FileUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // A utility for creating an array of timeseries objects from the 8 | // csv file. 9 | 10 | package com.yahoo.egads.utilities; 11 | 12 | import com.yahoo.egads.data.TimeSeries; 13 | import java.util.StringTokenizer; 14 | import java.util.ArrayList; 15 | import java.io.BufferedReader; 16 | import java.io.FileReader; 17 | import java.io.IOException; 18 | import java.util.Properties; 19 | 20 | public class FileUtils { 21 | 22 | // Creates a time-series from a file. 23 | public static ArrayList createTimeSeries(String csv_file, Properties config) { 24 | // Input file which needs to be parsed 25 | String fileToParse = csv_file; 26 | BufferedReader fileReader = null; 27 | ArrayList output = new ArrayList(); 28 | 29 | // Delimiter used in CSV file 30 | final String delimiter = ","; 31 | Long interval = null; 32 | Long prev = null; 33 | Integer aggr = 1; 34 | boolean fillMissing = false; 35 | if (config.getProperty("FILL_MISSING") != null && config.getProperty("FILL_MISSING").equals("1")) { 36 | fillMissing = true; 37 | } 38 | if (config.getProperty("AGGREGATION") != null) { 39 | aggr = new Integer(config.getProperty("AGGREGATION")); 40 | } 41 | try { 42 | String line = ""; 43 | // Create the file reader. 44 | fileReader = new BufferedReader(new FileReader(fileToParse)); 45 | 46 | // Read the file line by line 47 | boolean firstLine = true; 48 | while ((line = fileReader.readLine()) != null) { 49 | // Get all tokens available in line. 50 | String[] tokens = line.split(delimiter); 51 | Long curTimestamp = null; 52 | 53 | // Check for the case where there is more than one line preceding the data 54 | if (firstLine == true) { 55 | if (!isNumeric(tokens[0]) && tokens[0].equals("timestamp") == false) { 56 | continue; 57 | } 58 | } 59 | if (firstLine == false && tokens.length > 1) { 60 | curTimestamp = (new Double(tokens[0])).longValue(); 61 | } 62 | for (int i = 1; i < tokens.length; i++) { 63 | // Assume that the first line contains the column names. 64 | if (firstLine) { 65 | TimeSeries ts = new TimeSeries(); 66 | ts.meta.fileName = csv_file; 67 | output.add(ts); 68 | if (isNumeric(tokens[i]) == false) { // Just in case there's a numeric column heading 69 | ts.meta.name = tokens[i]; 70 | } else { 71 | ts.meta.name = "metric_" + i; 72 | output.get(i - 1).append((new Double(tokens[0])).longValue(), 73 | new Float(tokens[i])); 74 | } 75 | } else { 76 | // A naive missing data handler. 77 | if (interval != null && prev != null && interval > 0 && fillMissing == true) { 78 | if ((curTimestamp - prev) != interval) { 79 | int missingValues = (int) ((curTimestamp - prev) / interval); 80 | 81 | Long curTimestampToFill = prev + interval; 82 | for (int j = (missingValues - 1); j > 0; j--) { 83 | Float valToFill = new Float(tokens[i]); 84 | if (output.get(i - 1).size() >= missingValues) { 85 | valToFill = output.get(i - 1).data.get(output.get(i - 1).size() - missingValues).value; 86 | } 87 | output.get(i - 1).append(curTimestampToFill, valToFill); 88 | curTimestampToFill += interval; 89 | } 90 | } 91 | } 92 | // Infer interval. 93 | if (interval == null && prev != null) { 94 | interval = curTimestamp - new Long(prev); 95 | } 96 | 97 | output.get(i - 1).append(curTimestamp, 98 | new Float(tokens[i])); 99 | } 100 | } 101 | if (firstLine == false) { 102 | prev = curTimestamp; 103 | } 104 | firstLine = false; 105 | } 106 | } catch (Exception e) { 107 | e.printStackTrace(); 108 | } finally { 109 | try { 110 | fileReader.close(); 111 | } catch (IOException e) { 112 | e.printStackTrace(); 113 | } 114 | } 115 | // Handle aggregation. 116 | if (aggr > 1) { 117 | for (TimeSeries t : output) { 118 | t.data = t.aggregate(aggr); 119 | t.meta.name += "_aggr_" + aggr; 120 | } 121 | } 122 | return output; 123 | } 124 | 125 | // Checks if the string is numeric. 126 | public static boolean isNumeric(String str) { 127 | try { 128 | Double.parseDouble(str); 129 | } catch (NumberFormatException nfe) { 130 | return false; 131 | } 132 | return true; 133 | } 134 | 135 | // Parses the string array property into an integer property. 136 | public static int[] splitInts(String str) throws IllegalArgumentException { 137 | StringTokenizer tokenizer = new StringTokenizer(str, ","); 138 | int n = tokenizer.countTokens(); 139 | int[] list = new int[n]; 140 | for (int i = 0; i < n; i++) { 141 | String token = tokenizer.nextToken(); 142 | list[i] = Integer.parseInt(token); 143 | } 144 | return list; 145 | } 146 | 147 | // Initializes properties from a string (key:value, separated by ";"). 148 | public static void initProperties(String config, Properties p) { 149 | String delims1 = ";"; 150 | String delims2 = ":"; 151 | 152 | StringTokenizer st1 = new StringTokenizer(config, delims1); 153 | while (st1.hasMoreElements()) { 154 | String[] st2 = (st1.nextToken()).split(delims2); 155 | p.setProperty(st2[0], st2[1]); 156 | } 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/utilities/IdentifiedDoublePoint.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package com.yahoo.egads.utilities; 19 | 20 | import org.apache.commons.math3.ml.clustering.Clusterable; 21 | import org.apache.commons.math3.ml.clustering.DoublePoint; 22 | 23 | /** 24 | * A simple implementation of {@link Clusterable} for points with double coordinates, with an added id. 25 | * @version $Id$ 26 | * @since 3.2 27 | */ 28 | public class IdentifiedDoublePoint extends DoublePoint { 29 | 30 | private final int id; 31 | 32 | /** 33 | * Build an instance wrapping an double array. 34 | *

35 | * The wrapped array is referenced, it is not copied. 36 | * 37 | * @param point the n-dimensional point in double space 38 | */ 39 | public IdentifiedDoublePoint(final double[] point, int id) { 40 | super(point); 41 | this.id = id; 42 | } 43 | 44 | /** 45 | * Build an instance wrapping an integer array. 46 | *

47 | * The wrapped array is copied to an internal double array. 48 | * 49 | * @param point the n-dimensional point in integer space 50 | */ 51 | public IdentifiedDoublePoint(final int[] point, int id) { 52 | super(point); 53 | this.id = id; 54 | } 55 | 56 | public int getId() { 57 | return id; 58 | } 59 | 60 | } -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/utilities/InputProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads.utilities; 8 | 9 | // An interface which other classes extend 10 | // based on what type of input is passed in. 11 | 12 | import java.util.Properties; 13 | 14 | public interface InputProcessor { 15 | 16 | void processInput(Properties p) throws Exception; 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/utilities/ListUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // ListUtils provides some basic math operation on numerical lists 8 | 9 | package com.yahoo.egads.utilities; 10 | 11 | import java.util.Collections; 12 | import java.util.Iterator; 13 | import java.util.LinkedList; 14 | import java.util.List; 15 | 16 | public class ListUtils { 17 | 18 | public static float sumQ(List Q) { 19 | float res = 0; 20 | for (float f : Q) { 21 | res += f; 22 | } 23 | 24 | return res; 25 | } 26 | 27 | public static float sum2Q(List Q) { 28 | float res = 0; 29 | for (float f : Q) { 30 | res += (f * f); 31 | } 32 | 33 | return res; 34 | } 35 | 36 | public static void repQ(LinkedList Q, float value, int repitition) { 37 | Q.clear(); 38 | for (int i = 0; i < repitition; ++i) { 39 | Q.addLast(value); 40 | } 41 | } 42 | 43 | public static float kernelSum(float x, List mean, List sds) { 44 | float res = 0; 45 | float sd = 1; 46 | Iterator it = sds.iterator(); 47 | for (float m : mean) { 48 | if (it.hasNext()) { 49 | sd = it.next(); 50 | } 51 | 52 | res += Math.exp(-Math.pow(x - m, 2) / (2 * Math.pow(sd, 2))) / (sd * Math.sqrt(2 * Math.PI)); 53 | } 54 | 55 | return res; 56 | } 57 | 58 | public static float kernelSubSum(float x, List mean, List sds, int from, int to) { 59 | float res = 0; 60 | float sd = 1; 61 | Iterator it = sds.iterator(); 62 | int i = 0; 63 | 64 | for (float m : mean) { 65 | if (it.hasNext()) { 66 | sd = it.next(); 67 | } 68 | 69 | if (i >= from && i <= to) { 70 | res += Math.exp(-Math.pow(x - m, 2) / (2 * Math.pow(sd, 2))) / (sd * Math.sqrt(2 * Math.PI)); 71 | } 72 | 73 | i++; 74 | if (i > to) { 75 | break; 76 | } 77 | } 78 | 79 | return res; 80 | } 81 | 82 | public static float sumLog(List Q) { 83 | float res = 0; 84 | 85 | for (float f : Q) { 86 | res += Math.log(f); 87 | } 88 | 89 | return res; 90 | } 91 | 92 | public static LinkedList kernelQ(List Q, List mean, List sds) { 93 | LinkedList res = new LinkedList(); 94 | float sd = 1; 95 | float m = 0; 96 | Iterator it1 = mean.iterator(); 97 | Iterator it2 = sds.iterator(); 98 | 99 | for (float x : Q) { 100 | if (it1.hasNext()) { 101 | m = it1.next(); 102 | } 103 | 104 | if (it2.hasNext()) { 105 | sd = it2.next(); 106 | } 107 | 108 | res.addLast((float) (Math.exp(-Math.pow(x - m, 2) / (2 * Math.pow(sd, 2))) / (sd * Math.sqrt(2 * Math.PI)))); 109 | } 110 | 111 | return res; 112 | } 113 | 114 | public static void addQ(LinkedList Q, List P) { 115 | Iterator it = P.iterator(); 116 | float p = 0; 117 | LinkedList temp = new LinkedList(); 118 | 119 | for (float q : Q) { 120 | if (it.hasNext()) { 121 | p = it.next(); 122 | } 123 | 124 | temp.addLast(p + q); 125 | } 126 | 127 | Q.clear(); 128 | Q.addAll(temp); 129 | } 130 | 131 | public static void subtractQ(LinkedList Q, List P) { 132 | Iterator it = P.iterator(); 133 | float p = 0; 134 | LinkedList temp = new LinkedList(); 135 | 136 | for (float q : Q) { 137 | if (it.hasNext()) { 138 | p = it.next(); 139 | } 140 | 141 | temp.addLast(q - p); 142 | } 143 | 144 | Q.clear(); 145 | Q.addAll(temp); 146 | } 147 | 148 | public static void multiplyQ(LinkedList Q, List P) { 149 | Iterator it = P.iterator(); 150 | float p = 0; 151 | LinkedList temp = new LinkedList(); 152 | 153 | for (float q : Q) { 154 | if (it.hasNext()) { 155 | p = it.next(); 156 | } 157 | 158 | temp.addLast(q * p); 159 | } 160 | 161 | Q.clear(); 162 | Q.addAll(temp); 163 | } 164 | 165 | public static LinkedList maxQ(List Q, float m) { 166 | LinkedList temp = new LinkedList(); 167 | 168 | for (float q : Q) { 169 | temp.addLast(Math.max(q, m)); 170 | } 171 | 172 | return temp; 173 | } 174 | 175 | public static float quantile(List Q, float probability) { 176 | Collections.sort(Q); 177 | int n = Q.size(); 178 | 179 | float index = (n - 1) * probability; 180 | int lo = (int) Math.floor(index); 181 | int hi = (int) Math.ceil(index); 182 | float h = index - lo; 183 | float qs = (1 - h) * Q.get(lo) + h * Q.get(hi); 184 | 185 | return qs; 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/utilities/StdinProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads.utilities; 8 | 9 | // Class that implements EGADS STDIN input processor. 10 | 11 | import com.yahoo.egads.control.ProcessableObject; 12 | import com.yahoo.egads.control.ProcessableObjectFactory; 13 | import java.util.Properties; 14 | import java.io.*; 15 | import java.util.ArrayList; 16 | import com.yahoo.egads.data.TimeSeries; 17 | 18 | public class StdinProcessor implements InputProcessor { 19 | 20 | public void processInput(Properties p) throws Exception { 21 | BufferedReader in = new BufferedReader(new InputStreamReader(System.in)); 22 | String s; 23 | Integer aggr = 1; 24 | if (p.getProperty("AGGREGATION") != null) { 25 | aggr = new Integer(p.getProperty("AGGREGATION")); 26 | } 27 | while ((s = in.readLine()) != null && s.length() != 0) { 28 | // Parse the time-series. 29 | ArrayList metrics = createTimeSeries(s, aggr); 30 | for (TimeSeries ts : metrics) { 31 | ProcessableObject po = ProcessableObjectFactory.create(ts, p); 32 | po.process(); 33 | } 34 | } 35 | } 36 | 37 | 38 | // Format of the time-series: meta1\tmeta\2{(2014120205,0),(2014122207,1)}\t{(2014120205,0),(2014122207,0)}... 39 | // Creates a time-series from a file. 40 | private static ArrayList createTimeSeries(String s, Integer aggr) throws Exception { 41 | ArrayList output = new ArrayList(); 42 | String[] tokens = s.split("\t"); 43 | String meta = "meta"; 44 | 45 | int tokenNum = 1; 46 | for (String t : tokens) { 47 | if (t.contains("{(")) { 48 | output.add(convertStringToTS(t, meta + "-" + Integer.toString(tokenNum))); 49 | tokenNum++; 50 | } else { 51 | meta += "-" + t; 52 | } 53 | } 54 | 55 | // Handle aggregation. 56 | if (aggr > 1) { 57 | for (TimeSeries t : output) { 58 | t.data = t.aggregate(aggr); 59 | t.meta.name += "_aggr_" + aggr; 60 | } 61 | } 62 | return output; 63 | } 64 | 65 | private static TimeSeries convertStringToTS(String s, String tokenNum) throws Exception { 66 | TimeSeries ts = new TimeSeries(); 67 | ts.meta.fileName = tokenNum; 68 | ts.meta.name = tokenNum; 69 | String[] tuples = s.split("\\),"); 70 | for (String tuple : tuples) { 71 | tuple = tuple.replaceAll("[{}\\(\\)]", ""); 72 | String[] vals = tuple.split(","); 73 | Float val = new Float(vals[1]); 74 | ts.append(new Long(vals[0]), val); 75 | } 76 | return ts; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/egads/utilities/Variance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // accumulator class 8 | // 9 | // If loss of precision is a concern, maybe switch to Knuth algorithm: 10 | // http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Incremental_algorithm 11 | 12 | package com.yahoo.egads.utilities; 13 | 14 | public class Variance { 15 | // member data //////////////////////////////////////////////// 16 | 17 | public int count = 0; 18 | public float sum = 0; 19 | public float sumSquares = 0; 20 | 21 | // construction //////////////////////////////////////////////// 22 | 23 | public Variance() { 24 | } 25 | 26 | // methods //////////////////////////////////////////////// 27 | 28 | public void reset() { 29 | count = 0; 30 | sum = 0; 31 | sumSquares = 0; 32 | } 33 | 34 | public void increment(float value) { 35 | ++count; 36 | sum += value; 37 | sumSquares += (value * value); 38 | } 39 | 40 | public float currentAverage() { 41 | return sum / (float) count; 42 | } 43 | 44 | public float currentVariance() { 45 | float mean = currentAverage(); 46 | float sqMean = mean * mean; 47 | float meanOfSquares = sumSquares / count; 48 | return meanOfSquares - sqMean; 49 | } 50 | 51 | // display //////////////////////////////////////////////// 52 | 53 | public String toString() { 54 | return "" + currentVariance() + "=(" + sumSquares + "/" + count + ")-(" 55 | + sum + "/" + count + ")^2"; 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/egads/TestAggregate.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads; 8 | 9 | import com.yahoo.egads.data.*; 10 | import org.testng.Assert; 11 | import org.testng.annotations.Test; 12 | 13 | // Test the aggregation feature of the TimeSeries class. 14 | public class TestAggregate { 15 | 16 | @Test 17 | public void testAggregate() throws Exception { 18 | TimeSeries series = new TimeSeries(); 19 | for (int i = 0; i < 100; i++) { 20 | series.append((long)i, (float)1); 21 | } 22 | TimeSeries.DataSequence d = series.aggregate(10); 23 | Assert.assertEquals(d.size(), 10); 24 | for (int i = 0; i < d.size(); i++) { 25 | Assert.assertEquals(d.get(i).value, (float) 1.0); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/egads/TestAnomaly.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads; 8 | 9 | import com.yahoo.egads.data.*; 10 | import org.testng.Assert; 11 | import org.testng.annotations.Test; 12 | 13 | public class TestAnomaly { 14 | 15 | @Test 16 | public void test() { 17 | // just test construction and printing 18 | Anomaly anomaly = new Anomaly("ANOMALY_ID", null); 19 | anomaly.addInterval(1, 11, 1.11f); 20 | System.out.print("\n anomaly = " + anomaly); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/egads/TestAnomalyDetect.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads; 8 | 9 | import com.yahoo.egads.models.tsmm.OlympicModel; 10 | import com.yahoo.egads.models.adm.*; 11 | import com.yahoo.egads.data.Anomaly.IntervalSequence; 12 | import com.yahoo.egads.data.*; 13 | import java.util.ArrayList; 14 | import java.util.Properties; 15 | import java.io.FileInputStream; 16 | import java.io.InputStream; 17 | import org.testng.Assert; 18 | import org.testng.annotations.Test; 19 | 20 | // Tests the basic anoamly detection piece of EGADS. 21 | public class TestAnomalyDetect { 22 | 23 | @Test 24 | public void testOlympicModel() throws Exception { 25 | // Test cases: ref window: 10, 5 26 | // Drops: 0, 1 27 | String[] refWindows = new String[]{"10", "5"}; 28 | String[] drops = new String[]{"0", "1"}; 29 | // Load the true expected values from a file. 30 | String configFile = "src/test/resources/sample_config.ini"; 31 | InputStream is = new FileInputStream(configFile); 32 | Properties p = new Properties(); 33 | p.load(is); 34 | ArrayList actual_metric = com.yahoo.egads.utilities.FileUtils 35 | .createTimeSeries("src/test/resources/model_input.csv", p); 36 | p.setProperty("MAX_ANOMALY_TIME_AGO", "999999999"); 37 | for (int w = 0; w < refWindows.length; w++) { 38 | for (int d = 0; d < drops.length; d++) { 39 | p.setProperty("NUM_WEEKS", refWindows[w]); 40 | p.setProperty("NUM_TO_DROP", drops[d]); 41 | p.setProperty("THRESHOLD", "mapee#100,mase#10"); 42 | // Parse the input timeseries. 43 | ArrayList metrics = com.yahoo.egads.utilities.FileUtils 44 | .createTimeSeries("src/test/resources/model_output_" + refWindows[w] + "_" + drops[d] + ".csv", p); 45 | OlympicModel model = new OlympicModel(p); 46 | model.train(actual_metric.get(0).data); 47 | TimeSeries.DataSequence sequence = new TimeSeries.DataSequence(metrics.get(0).startTime(), 48 | metrics.get(0).lastTime(), 49 | 3600); 50 | sequence.setLogicalIndices(metrics.get(0).startTime(), 3600); 51 | model.predict(sequence); 52 | // Initialize the anomaly detector. 53 | ExtremeLowDensityModel bcm = new ExtremeLowDensityModel(p); 54 | 55 | // Initialize the DBScan anomaly detector. 56 | DBScanModel dbs = new DBScanModel(p); 57 | IntervalSequence anomalies = bcm.detect(actual_metric.get(0).data, sequence); 58 | dbs.tune(actual_metric.get(0).data, sequence); 59 | IntervalSequence anomaliesdb = dbs.detect(actual_metric.get(0).data, sequence); 60 | 61 | // Initialize the SimpleThreshold anomaly detector. 62 | SimpleThresholdModel stm = new SimpleThresholdModel(p); 63 | 64 | stm.tune(actual_metric.get(0).data, sequence); 65 | IntervalSequence anomaliesstm = stm.detect(actual_metric.get(0).data, sequence); 66 | Assert.assertTrue(anomalies.size() > 10); 67 | Assert.assertTrue(anomaliesdb.size() > 2); 68 | Assert.assertTrue(anomaliesstm.size() > 2); 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/egads/TestAutoForecast.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads; 8 | 9 | import com.yahoo.egads.data.Model; 10 | import com.yahoo.egads.models.tsmm.*; 11 | import com.yahoo.egads.utilities.*; 12 | import com.yahoo.egads.data.*; 13 | import java.util.ArrayList; 14 | import java.util.Properties; 15 | import java.io.FileInputStream; 16 | import java.io.InputStream; 17 | import com.yahoo.egads.control.ProcessableObject; 18 | import com.yahoo.egads.control.ProcessableObjectFactory; 19 | import org.testng.Assert; 20 | import org.testng.annotations.Test; 21 | import net.sourceforge.openforecast.Forecaster; 22 | import net.sourceforge.openforecast.ForecastingModel; 23 | import net.sourceforge.openforecast.DataSet; 24 | import net.sourceforge.openforecast.DataPoint; 25 | import net.sourceforge.openforecast.Observation; 26 | import java.util.*; 27 | 28 | // Tests the correct generation of the expected values for olympic 29 | // scoring. 30 | public class TestAutoForecast { 31 | 32 | @Test 33 | public void testAutoForecast() throws Exception { 34 | 35 | String configFile = "src/test/resources/sample_config.ini"; 36 | InputStream is = new FileInputStream(configFile); 37 | Properties p = new Properties(); 38 | p.load(is); 39 | ArrayList metrics = com.yahoo.egads.utilities.FileUtils 40 | .createTimeSeries("src/test/resources/sample_input.csv", p); 41 | AutoForecastModel model = new AutoForecastModel(p); 42 | model.train(metrics.get(0).data); 43 | TimeSeries.DataSequence sequence = new TimeSeries.DataSequence(metrics.get(0).startTime(), 44 | metrics.get(0).lastTime(), 45 | 3600); 46 | 47 | sequence.setLogicalIndices(metrics.get(0).startTime(), 3600); 48 | model.predict(sequence); 49 | Assert.assertEquals(verifyResults(sequence, metrics.get(0).data), true); 50 | } 51 | 52 | // Verifies that the two time-series are identical. 53 | private boolean verifyResults (TimeSeries.DataSequence computed, TimeSeries.DataSequence actual) { 54 | int n = computed.size(); 55 | int n2 = actual.size(); 56 | if (n != n2) { 57 | return false; 58 | } 59 | float precision = (float) 0.000001; 60 | float errorSum = (float) 0.0; 61 | for (int i = 0; i < n; i++) { 62 | errorSum += Math.abs(computed.get(i).value - actual.get(i).value); 63 | } 64 | errorSum /= n; 65 | 66 | if (errorSum <= 5152990) { 67 | return true; 68 | } 69 | return false; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/egads/TestAutoSensitivity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads; 8 | 9 | import com.yahoo.egads.utilities.AutoSensitivity; 10 | import java.util.ArrayList; 11 | import org.testng.Assert; 12 | import org.testng.annotations.Test; 13 | import java.util.Random; 14 | import java.io.BufferedReader; 15 | import java.io.FileReader; 16 | 17 | // Tests the basic anoamly detection piece of EGADS. 18 | public class TestAutoSensitivity { 19 | 20 | @Test 21 | public void testAutoSensitivity() throws Exception { 22 | // Generate a uniform set of numbers and make sure that 23 | // the result is the expected mean. 24 | ArrayList randomNumbers = new ArrayList(); 25 | //note a single Random object is reused here 26 | Random randomGenerator = new Random(); 27 | int numToGenerate = 100; 28 | for (int idx = 1; idx <= numToGenerate; ++idx){ 29 | float randomFloat = randomGenerator.nextFloat(); 30 | randomNumbers.add(randomFloat); 31 | } 32 | Float[] fArray = randomNumbers.toArray(new Float[randomNumbers.size()]); 33 | float threashold = AutoSensitivity.getKSigmaSensitivity(fArray, 1); 34 | Assert.assertTrue(threashold <= 2); 35 | ArrayList sampleErrors = new ArrayList(); 36 | BufferedReader reader = new BufferedReader(new FileReader("src/test/resources/sample_errors.csv")); 37 | 38 | for (String line = reader.readLine(); line != null; line = reader.readLine()) { 39 | Float f = Float.parseFloat(line); 40 | sampleErrors.add(f); 41 | } 42 | 43 | fArray = sampleErrors.toArray(new Float[sampleErrors.size()]); 44 | threashold = AutoSensitivity.getLowDensitySensitivity(fArray, 1, 1); 45 | Assert.assertTrue(threashold > 5000000); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/egads/TestChangePointDetectionAlgorithm.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads; 8 | 9 | import java.util.ArrayList; 10 | import org.testng.Assert; 11 | import java.util.Properties; 12 | import org.testng.annotations.Test; 13 | import com.yahoo.egads.data.TimeSeries; 14 | import com.yahoo.egads.models.adm.AdaptiveKernelDensityChangePointDetector; 15 | import java.io.InputStream; 16 | import java.io.FileInputStream; 17 | 18 | public class TestChangePointDetectionAlgorithm { 19 | @Test 20 | public void testChangePointDetectionAlgorithm() throws Exception { 21 | String configFile = "src/test/resources/sample_config.ini"; 22 | InputStream is = new FileInputStream(configFile); 23 | Properties p = new Properties(); 24 | p.load(is); 25 | TimeSeries observedTS = 26 | com.yahoo.egads.utilities.FileUtils 27 | .createTimeSeries("src/test/resources/cp-obs.csv", p).get(0); 28 | TimeSeries expectedTS = 29 | com.yahoo.egads.utilities.FileUtils 30 | .createTimeSeries("src/test/resources/cp-exp.csv", p).get(0); 31 | 32 | int n = observedTS.size(); 33 | Integer preWindowSize = 2 * 24 * 4; 34 | Integer postWindowSize = 2 * 24 * 4; 35 | Float confidence = 0.8F; 36 | float[] residuals = new float[n]; 37 | 38 | // Computing the residuals 39 | for (int i = 0; i < n; ++i) { 40 | residuals[i] = observedTS.data.get(i).value - expectedTS.data.get(i).value; 41 | } 42 | 43 | p.setProperty("PRE_WINDOW_SIZE", preWindowSize.toString()); 44 | p.setProperty("POST_WINDOW_SIZE", postWindowSize.toString()); 45 | p.setProperty("CONFIDENCE", confidence.toString()); 46 | p.setProperty("MAX_ANOMALY_TIME_AGO", "48"); 47 | 48 | AdaptiveKernelDensityChangePointDetector cpd = new AdaptiveKernelDensityChangePointDetector(p); 49 | ArrayList changePoints = cpd.detectChangePoints(residuals, preWindowSize, postWindowSize, confidence); 50 | Assert.assertTrue(changePoints.size() == 1); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/egads/TestGetAnomalyList.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads; 8 | 9 | import com.yahoo.egads.control.ProcessableObject; 10 | import com.yahoo.egads.control.ProcessableObjectFactory; 11 | import com.yahoo.egads.data.Anomaly; 12 | import com.yahoo.egads.data.TimeSeries; 13 | 14 | import org.testng.Assert; 15 | import org.testng.annotations.Test; 16 | 17 | import java.io.FileInputStream; 18 | import java.io.InputStream; 19 | import java.util.ArrayList; 20 | import java.util.Arrays; 21 | import java.util.List; 22 | import java.util.Properties; 23 | 24 | /* 25 | * Test the result() from ProcessableObject. 26 | */ 27 | public class TestGetAnomalyList { 28 | 29 | @Test 30 | public void testUpdateModelProcessable() throws Exception { 31 | TimeSeries series = new TimeSeries(); 32 | series.append(1L, 11.11f); 33 | series.append(2L, 22.22f); 34 | series.append(3L, 33.33f); 35 | series.append(5L, 55.55f); 36 | series.append(7L, 77.77f); 37 | series.append(9L, 99.99f); 38 | InputStream is = new FileInputStream("src/test/resources/sample_config.ini"); 39 | Properties p = new Properties(); 40 | p.load(is); 41 | p.setProperty("TS_MODEL","OlympicModel"); 42 | p.setProperty("AD_MODEL","KSigmaModel"); 43 | p.setProperty("OP_TYPE","UPDATE_MODEL"); 44 | 45 | ProcessableObject po = ProcessableObjectFactory.create(series, p); 46 | po.process(); 47 | 48 | Assert.assertEquals(po.result().toString(), "Updated"); 49 | } 50 | 51 | @Test 52 | public void testDetectAnomalyProcessable() throws Exception { 53 | 54 | InputStream is = new FileInputStream("src/test/resources/sample_config.ini"); 55 | Properties p = new Properties(); 56 | p.load(is); 57 | p.setProperty("TS_MODEL","OlympicModel"); 58 | p.setProperty("AD_MODEL","KSigmaModel"); 59 | p.setProperty("MAX_ANOMALY_TIME_AGO","0"); 60 | p.setProperty("OP_TYPE","DETECT_ANOMALY"); 61 | 62 | ArrayList metrics = com.yahoo.egads.utilities.FileUtils 63 | .createTimeSeries("src/test/resources/sample_input.csv", p); 64 | 65 | // generate expected result 66 | Long anomalousTime = 1417194000L; 67 | Anomaly anomaly = new Anomaly("value",null); 68 | anomaly.addInterval(anomalousTime, anomalousTime,0.0f); 69 | 70 | // actual result 71 | ProcessableObject po = ProcessableObjectFactory.create(metrics.get(0), p); 72 | po.process(); 73 | 74 | Assert.assertEquals(po.result().toString(), Arrays.asList(anomaly).toString()); 75 | } 76 | 77 | @Test 78 | public void testTransformInputProcessable() throws Exception { 79 | 80 | InputStream is = new FileInputStream("src/test/resources/sample_config.ini"); 81 | Properties p = new Properties(); 82 | p.load(is); 83 | p.setProperty("TS_MODEL","OlympicModel"); 84 | p.setProperty("AD_MODEL","KSigmaModel"); 85 | p.setProperty("OP_TYPE","TRANSFORM_INPUT"); 86 | 87 | TimeSeries series = new TimeSeries(); 88 | series.append(1L, 11.11f); 89 | series.append(2L, 22.22f); 90 | series.append(3L, 33.33f); 91 | 92 | // generate expected result 93 | Long[] times = new Long[]{1L, 2L, 3L}; 94 | Float[] values = new Float[]{11.11f, 22.22f, 33.33f}; 95 | 96 | // actual result 97 | ProcessableObject po = ProcessableObjectFactory.create(series, p); 98 | po.process(); 99 | List actual = (List)po.result(); 100 | Assert.assertEquals(actual.get(0).getValues(), values); 101 | Assert.assertEquals(actual.get(0).getTimes(), times); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/egads/TestJsonEncoder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads; 8 | 9 | import com.yahoo.egads.data.*; 10 | import org.testng.Assert; 11 | import org.testng.annotations.Test; 12 | 13 | public class TestJsonEncoder { 14 | 15 | @Test 16 | public void testAnomalyInterval() throws Exception { 17 | // anomaly interval 18 | Anomaly.Interval interval = new Anomaly.Interval(1L, 2L, 3.33f); 19 | System.out.print("\n interval = " + interval); 20 | String interval_json = JsonEncoder.toJson(interval); 21 | System.out.print("\n interval json = " + interval_json); 22 | Anomaly.Interval interval2 = new Anomaly.Interval(); 23 | JsonEncoder.fromJson(interval2, interval_json); 24 | System.out 25 | .print("\n interval2 json = " + JsonEncoder.toJson(interval2)); 26 | Assert.assertEquals(interval, interval2); 27 | } 28 | 29 | @Test 30 | public void testAnomaly() throws Exception { 31 | // anomaly 32 | Anomaly anomaly = new Anomaly("ANOMALY_ID", new MetricMeta("metric1")); 33 | anomaly.addInterval(2, 2.2f); 34 | anomaly.addInterval(3, 3.3f); 35 | System.out.print("\n anomaly = " + anomaly); 36 | String anomaly_json = JsonEncoder.toJson(anomaly); 37 | System.out.print("\n anomaly json = " + anomaly_json); 38 | Anomaly anomaly2 = new Anomaly(); 39 | JsonEncoder.fromJson(anomaly2, anomaly_json); 40 | System.out.print("\n anomaly2 json = " + JsonEncoder.toJson(anomaly2)); 41 | Assert.assertEquals(anomaly, anomaly2); 42 | } 43 | 44 | @Test 45 | public void testTimeSeries() throws Exception { 46 | // time series 47 | TimeSeries series = new TimeSeries(new float[] { 1, 2, 4 }); 48 | series.meta = new MetricMeta("time series meta"); 49 | series.meta.smoothing = "for_all"; 50 | System.out.print("\n\n series = " + series); 51 | String series_json = JsonEncoder.toJson(series); 52 | System.out.print("\n series json = " + series_json); 53 | TimeSeries series2 = new TimeSeries(); 54 | JsonEncoder.fromJson(series2, series_json); 55 | System.out.print("\n series2 json = " + JsonEncoder.toJson(series2)); 56 | Assert.assertEquals(series, series2); 57 | } 58 | 59 | @Test 60 | public void testMetricMeta() throws Exception { 61 | // metric meta 62 | MetricMeta metric = new MetricMeta("m1"); 63 | metric.detectAnomalies = true; 64 | System.out.print("\n\n metric = " + metric); 65 | String metric_json = JsonEncoder.toJson(metric); 66 | System.out.print("\n metric json = " + metric_json); 67 | MetricMeta metric2 = new MetricMeta(null); 68 | JsonEncoder.fromJson(metric2, metric_json); 69 | System.out.print("\n\n metric2 = " + metric2); 70 | System.out.print("\n metric2 json = " + JsonEncoder.toJson(metric2)); 71 | Assert.assertEquals(metric, metric2); 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/egads/TestMetricMeta.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads; 8 | 9 | import com.yahoo.egads.data.*; 10 | import org.testng.Assert; 11 | import org.testng.annotations.Test; 12 | 13 | public class TestMetricMeta { 14 | 15 | @Test 16 | public void test() { 17 | 18 | // just check for exceptions in construction and printing 19 | MetricMeta metric = new MetricMeta("m1"); 20 | metric.detectAnomalies = true; 21 | metric.source = "somewhere"; 22 | System.out.print("\n metric = " + metric); 23 | 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/egads/TestMissingData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | // a standalone working main method for the main 8 | // EGADS application. 9 | 10 | package com.yahoo.egads; 11 | 12 | import com.yahoo.egads.data.TimeSeries; 13 | import java.util.ArrayList; 14 | import org.testng.annotations.Test; 15 | import org.testng.Assert; 16 | import java.io.InputStream; 17 | import java.io.FileInputStream; 18 | import java.util.Properties; 19 | 20 | public class TestMissingData { 21 | 22 | @Test 23 | public void testMissingData() throws Exception { 24 | String csv_file = "src/test/resources/sample_missing_input.csv"; 25 | String configFile = "src/test/resources/sample_config.ini"; 26 | InputStream is = new FileInputStream(configFile); 27 | Properties p = new Properties(); 28 | p.load(is); 29 | // Auto detect. 30 | p.setProperty("PERIOD", "0"); 31 | p.setProperty("FILL_MISSING", "1"); 32 | 33 | ArrayList metrics = com.yahoo.egads.utilities.FileUtils 34 | .createTimeSeries(csv_file, p); 35 | Assert.assertEquals(metrics.get(0).data.size(), 1433); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/egads/TestOlympicModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads; 8 | 9 | import com.yahoo.egads.data.Model; 10 | import com.yahoo.egads.models.tsmm.TimeSeriesAbstractModel; 11 | import com.yahoo.egads.models.tsmm.OlympicModel; 12 | import com.yahoo.egads.models.tsmm.MovingAverageModel; 13 | import com.yahoo.egads.utilities.*; 14 | import com.yahoo.egads.data.*; 15 | import java.util.ArrayList; 16 | import java.util.Properties; 17 | import java.io.FileInputStream; 18 | import java.io.InputStream; 19 | import com.yahoo.egads.control.ProcessableObject; 20 | import com.yahoo.egads.control.ProcessableObjectFactory; 21 | import org.testng.Assert; 22 | import org.testng.annotations.Test; 23 | 24 | // Tests the correct generation of the expected values for olympic 25 | // scoring. 26 | public class TestOlympicModel { 27 | 28 | @Test 29 | public void testOlympicModel() throws Exception { 30 | // Test cases: ref window: 10, 5 31 | // Drops: 0, 1 32 | String[] refWindows = new String[]{"10", "5"}; 33 | String[] drops = new String[]{"0", "1"}; 34 | // Load the true expected values from a file. 35 | String configFile = "src/test/resources/sample_config.ini"; 36 | InputStream is = new FileInputStream(configFile); 37 | Properties p = new Properties(); 38 | p.load(is); 39 | ArrayList actual_metric = com.yahoo.egads.utilities.FileUtils 40 | .createTimeSeries("src/test/resources/model_input.csv", p); 41 | 42 | for (int w = 0; w < refWindows.length; w++) { 43 | for (int d = 0; d < drops.length; d++) { 44 | p.setProperty("NUM_WEEKS", refWindows[w]); 45 | p.setProperty("NUM_TO_DROP", drops[d]); 46 | // Parse the input timeseries. 47 | ArrayList metrics = com.yahoo.egads.utilities.FileUtils 48 | .createTimeSeries("src/test/resources/model_output_" + refWindows[w] + "_" + drops[d] + ".csv", p); 49 | OlympicModel model = new OlympicModel(p); 50 | model.train(actual_metric.get(0).data); 51 | TimeSeries.DataSequence sequence = new TimeSeries.DataSequence(metrics.get(0).startTime(), 52 | metrics.get(0).lastTime(), 53 | 3600); 54 | 55 | 56 | sequence.setLogicalIndices(metrics.get(0).startTime(), 3600); 57 | model.predict(sequence); 58 | Assert.assertEquals(verifyResults(sequence, metrics.get(0).data), true); 59 | } 60 | } 61 | } 62 | 63 | // Verifies that the two time-series are identical. 64 | private boolean verifyResults (TimeSeries.DataSequence computed, TimeSeries.DataSequence actual) { 65 | int n = computed.size(); 66 | int n2 = actual.size(); 67 | if (n != n2) { 68 | return false; 69 | } 70 | float precision = (float) 0.000001; 71 | for (int i = 0; i < n; i++) { 72 | if (Math.abs(computed.get(i).value - actual.get(i).value) > precision) { 73 | return false; 74 | } 75 | } 76 | return true; 77 | } 78 | 79 | @Test 80 | public void testForecastErrors() throws Exception { 81 | String configFile = "src/test/resources/sample_config.ini"; 82 | InputStream is = new FileInputStream(configFile); 83 | Properties p = new Properties(); 84 | p.load(is); 85 | ArrayList actual_metric = com.yahoo.egads.utilities.FileUtils 86 | .createTimeSeries("src/test/resources/model_input.csv", p); 87 | OlympicModel olympicModel = new OlympicModel(p); 88 | olympicModel.train(actual_metric.get(0).data); 89 | 90 | Assert.assertEquals(olympicModel.getBias(), -26.315675155416635, 1e-10); 91 | Assert.assertEquals(olympicModel.getMAD(), 28.81582062080335, 1e-10); 92 | Assert.assertEquals(Double.isNaN(olympicModel.getMAPE()), true); 93 | Assert.assertEquals(olympicModel.getMSE(), 32616.547275296416, 1e-7); 94 | Assert.assertEquals(olympicModel.getSAE(), 41033.72856402397, 1e-7); 95 | } 96 | 97 | @Test 98 | public void testBetterThan() throws Exception { 99 | String configFile = "src/test/resources/sample_config.ini"; 100 | InputStream is = new FileInputStream(configFile); 101 | Properties p = new Properties(); 102 | p.load(is); 103 | ArrayList actual_metric = com.yahoo.egads.utilities.FileUtils 104 | .createTimeSeries("src/test/resources/model_input.csv", p); 105 | OlympicModel olympicModel = new OlympicModel(p); 106 | olympicModel.train(actual_metric.get(0).data); 107 | 108 | MovingAverageModel movingAverageModel = new MovingAverageModel(p); 109 | movingAverageModel.train(actual_metric.get(0).data); 110 | 111 | // movingAverageModel is better than olympicModel 112 | Assert.assertEquals(TimeSeriesAbstractModel.betterThan(movingAverageModel, olympicModel), true); 113 | Assert.assertEquals(TimeSeriesAbstractModel.betterThan(movingAverageModel, movingAverageModel), false); 114 | Assert.assertEquals(TimeSeriesAbstractModel.betterThan(olympicModel, movingAverageModel), false); 115 | Assert.assertEquals(TimeSeriesAbstractModel.betterThan(olympicModel, olympicModel), false); 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/egads/TestReflectionSpeed.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads; 8 | 9 | import com.yahoo.egads.data.TimeSeries; 10 | 11 | import java.util.ArrayList; 12 | import java.util.Properties; 13 | import java.io.FileInputStream; 14 | import java.io.InputStream; 15 | 16 | import com.yahoo.egads.control.ProcessableObject; 17 | import com.yahoo.egads.control.ProcessableObjectFactory; 18 | 19 | import org.testng.annotations.Test; 20 | 21 | public class TestReflectionSpeed { 22 | 23 | @Test 24 | public void testReflectionSpeed() throws Exception { 25 | // Read command line args. 26 | String csv_file = "src/test/resources/sample_input.csv"; 27 | // TODO: This config will be retreieved from ConfigDB later, 28 | // for now it is assumed it's a static file. 29 | String configFile = "src/test/resources/sample_config.ini"; 30 | InputStream is = new FileInputStream(configFile); 31 | Properties p = new Properties(); 32 | p.load(is); 33 | 34 | // Parse the input timeseries. 35 | ArrayList metrics = com.yahoo.egads.utilities.FileUtils.createTimeSeries(csv_file, p); 36 | long start = System.currentTimeMillis(); 37 | for (TimeSeries ts : metrics) { 38 | ProcessableObject po = ProcessableObjectFactory.create(ts, p); 39 | // Here we don't process the time-series because we're only interested 40 | // in the reflection speed. 41 | // po.process(); 42 | } 43 | System.out.print("\n reflection speed: " + (System.currentTimeMillis() - start) + "ms"); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/egads/TestStats.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads; 8 | 9 | import com.yahoo.egads.utilities.StatsUtils; 10 | import java.util.ArrayList; 11 | import org.testng.Assert; 12 | import org.testng.annotations.Test; 13 | import java.util.Random; 14 | 15 | // Tests the basic anoamly detection piece of EGADS. 16 | public class TestStats { 17 | 18 | private Random fRandom = new Random(); 19 | 20 | @Test 21 | public void testStats() throws Exception { 22 | // Generate a normal distribution. 23 | ArrayList randomNumbers = new ArrayList(); 24 | // Skewed distribution. 25 | ArrayList skewedNumbers = new ArrayList(); 26 | 27 | double mean = 100.0f; 28 | double variance = 5.0f; 29 | int numToGenerate = 10000; 30 | 31 | for (int idx = 1; idx <= numToGenerate; ++idx){ 32 | randomNumbers.add(getGaussian(mean, variance)); 33 | } 34 | 35 | for (int idx = 1; idx <= numToGenerate; ++idx){ 36 | if (idx < numToGenerate/2) { 37 | skewedNumbers.add(1.0); 38 | } else { 39 | skewedNumbers.add(100.0); 40 | } 41 | } 42 | double[] result = StatsUtils.swilk(randomNumbers); 43 | // p val > alpha will NOT reject the null hypothesis that 44 | // the values came from a normal distribution. 45 | Assert.assertTrue(result[1] >= 0.05); 46 | result = StatsUtils.swilk(skewedNumbers); 47 | // p val < alpha will reject the null hypothesis that 48 | // the values came from a normal distribution. 49 | Assert.assertTrue(result[1] <= 0.05); 50 | } 51 | 52 | private double getGaussian(double aMean, double aVariance){ 53 | return aMean + fRandom.nextGaussian() * aVariance; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/egads/TestTimeSeries.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads; 8 | 9 | import com.yahoo.egads.data.*; 10 | import org.testng.Assert; 11 | import org.testng.annotations.Test; 12 | 13 | public class TestTimeSeries { 14 | 15 | @Test(expectedExceptions = Exception.class) 16 | public void testException() throws Exception { 17 | TimeSeries series = new TimeSeries(); 18 | series.append(1L, 11.11f); 19 | series.append(2L, 22.22f); 20 | series.append(3L, 33.33f); 21 | series.append(5L, 55.55f); 22 | series.append(7L, 77.77f); 23 | series.append(9L, 99.99f); 24 | series.append(0L, 00.00f); // causes exception, as it should 25 | } 26 | 27 | @Test 28 | public void test() throws Exception { 29 | TimeSeries series = new TimeSeries(); 30 | series.append(1L, 11.11f); 31 | series.append(2L, 22.22f); 32 | series.append(3L, 33.33f); 33 | series.append(5L, 55.55f); 34 | series.append(7L, 77.77f); 35 | series.append(9L, 99.99f); 36 | System.out.print("\n series = " + series); 37 | Assert.assertEquals(series.minimumPeriod(), 1); 38 | Assert.assertEquals(series.mostFrequentPeriod(), 2); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/egads/TestVariance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads; 8 | 9 | import com.yahoo.egads.utilities.*; 10 | import org.testng.Assert; 11 | import org.testng.annotations.Test; 12 | 13 | public class TestVariance { 14 | 15 | @Test 16 | public void test() { 17 | Variance variance = new Variance(); 18 | variance.increment(1); 19 | variance.increment(2); 20 | variance.increment(3); 21 | variance.increment(3); 22 | variance.increment(3); 23 | variance.increment(4); 24 | variance.increment(5); 25 | System.out.print("\n variance = " + variance); 26 | Assert.assertEquals(variance.currentVariance(), 1.4285717f, 0.001f); 27 | Assert.assertEquals(variance.currentAverage(), 3.0f, 0.001f); 28 | Assert.assertEquals(variance.count, 7); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/egads/data/TestWeightedValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | package com.yahoo.egads.data; 7 | 8 | import static org.testng.AssertJUnit.assertEquals; 9 | import static org.testng.AssertJUnit.assertTrue; 10 | 11 | import java.util.List; 12 | 13 | import org.testng.annotations.Test; 14 | 15 | import com.google.common.collect.Lists; 16 | 17 | public class TestWeightedValue { 18 | 19 | @Test 20 | public void aggregate() throws Exception { 21 | final List values = Lists.newArrayList(); 22 | values.add(new WeightedValue(1.5D, 5)); 23 | values.add(new WeightedValue(42.5D, 1)); 24 | values.add(new WeightedValue(-1.5D, 0)); 25 | values.add(new WeightedValue(3.5D, 1)); 26 | 27 | assertEquals(-1.5D, WeightedValue.aggregate(values, "MIN"), 0.0001); 28 | assertEquals(42.5D, WeightedValue.aggregate(values, "MAX"), 0.0001); 29 | assertEquals(46.0D, WeightedValue.aggregate(values, "SUM"), 0.0001); 30 | assertEquals(4D, WeightedValue.aggregate(values, "COUNT"), 0.0001); 31 | assertEquals(3.5D, WeightedValue.aggregate(values, "MEDIAN"), 0.0001); 32 | assertEquals(11.5D, WeightedValue.aggregate(values, "AVG"), 0.0001); 33 | assertEquals(7.6428D, WeightedValue.aggregate(values, "WAVG"), 0.0001); 34 | 35 | // add some NaNs and it shouldn't change 36 | values.clear(); 37 | values.add(new WeightedValue(1.5D, 5)); 38 | values.add(new WeightedValue(42.5D, 1)); 39 | values.add(new WeightedValue(Double.NaN, 1)); 40 | values.add(new WeightedValue(-1.5D, 0)); 41 | values.add(new WeightedValue(Double.NaN, 4)); 42 | values.add(new WeightedValue(3.5D, 1)); 43 | values.add(new WeightedValue(Double.NaN, 5)); 44 | 45 | assertEquals(-1.5D, WeightedValue.aggregate(values, "MIN"), 0.0001); 46 | assertEquals(42.5D, WeightedValue.aggregate(values, "MAX"), 0.0001); 47 | assertEquals(46.0D, WeightedValue.aggregate(values, "SUM"), 0.0001); 48 | assertEquals(4D, WeightedValue.aggregate(values, "COUNT"), 0.0001); 49 | assertEquals(3.5D, WeightedValue.aggregate(values, "MEDIAN"), 0.0001); 50 | assertEquals(11.5D, WeightedValue.aggregate(values, "AVG"), 0.0001); 51 | assertEquals(7.6428D, WeightedValue.aggregate(values, "WAVG"), 0.0001); 52 | 53 | // all nans should return nan (except count!) 54 | values.clear(); 55 | values.add(new WeightedValue(Double.NaN, 1)); 56 | values.add(new WeightedValue(Double.NaN, 2)); 57 | values.add(new WeightedValue(Double.NaN, 5)); 58 | 59 | assertTrue(Double.isNaN(WeightedValue.aggregate(values, "MIN"))); 60 | assertTrue(Double.isNaN(WeightedValue.aggregate(values, "MAX"))); 61 | assertTrue(Double.isNaN(WeightedValue.aggregate(values, "SUM"))); 62 | assertEquals(0, WeightedValue.aggregate(values, "COUNT"), 0.0001); 63 | assertTrue(Double.isNaN(WeightedValue.aggregate(values, "MEDIAN"))); 64 | assertTrue(Double.isNaN(WeightedValue.aggregate(values, "AVG"))); 65 | assertTrue(Double.isNaN(WeightedValue.aggregate(values, "WAVG"))); 66 | 67 | // empty always returns Nan. 68 | values.clear(); 69 | 70 | assertTrue(Double.isNaN(WeightedValue.aggregate(values, "MIN"))); 71 | assertTrue(Double.isNaN(WeightedValue.aggregate(values, "MAX"))); 72 | assertTrue(Double.isNaN(WeightedValue.aggregate(values, "SUM"))); 73 | assertEquals(0, WeightedValue.aggregate(values, "COUNT"), 0.0001); 74 | assertTrue(Double.isNaN(WeightedValue.aggregate(values, "MEDIAN"))); 75 | assertTrue(Double.isNaN(WeightedValue.aggregate(values, "AVG"))); 76 | assertTrue(Double.isNaN(WeightedValue.aggregate(values, "WAVG"))); 77 | } 78 | 79 | @Test (expectedExceptions = IllegalArgumentException.class) 80 | public void aggregateNullList() throws Exception { 81 | WeightedValue.aggregate(null, "MIN"); 82 | } 83 | 84 | @Test (expectedExceptions = IllegalArgumentException.class) 85 | public void aggregateNullAgg() throws Exception { 86 | final List values = Lists.newArrayList(); 87 | values.add(new WeightedValue(1.5D, 5)); 88 | WeightedValue.aggregate(values, null); 89 | } 90 | 91 | @Test (expectedExceptions = IllegalArgumentException.class) 92 | public void aggregateEmptyAgg() throws Exception { 93 | final List values = Lists.newArrayList(); 94 | values.add(new WeightedValue(1.5D, 5)); 95 | WeightedValue.aggregate(values, ""); 96 | } 97 | 98 | @Test (expectedExceptions = IllegalArgumentException.class) 99 | public void aggregateUnknownAgg() throws Exception { 100 | final List values = Lists.newArrayList(); 101 | values.add(new WeightedValue(1.5D, 5)); 102 | WeightedValue.aggregate(values, "NOTIMPLEMENTED"); 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/egads/utilities/TestSpectralMethods.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Yahoo Inc. 3 | * Copyrights licensed under the GPL License. 4 | * See the accompanying LICENSE file for terms. 5 | */ 6 | 7 | package com.yahoo.egads.utilities; 8 | 9 | import java.util.Arrays; 10 | 11 | import org.apache.commons.math3.linear.MatrixUtils; 12 | import org.apache.commons.math3.linear.RealMatrix; 13 | import org.testng.annotations.Test; 14 | 15 | public class TestSpectralMethods { 16 | @Test 17 | public void f() { 18 | double[][] mat = { {1, 2}, {3, 2}, {2, 5}, {7, 8}, {3, 4}, {8, 9}, {3, 3}}; 19 | RealMatrix data = MatrixUtils.createRealMatrix(mat); 20 | 21 | RealMatrix res = SpectralMethods.createHankelMatrix(data, 3); 22 | for (int i = 0; i < res.getRowDimension(); ++i) { 23 | System.out.println(Arrays.toString(res.getRow(i))); 24 | } 25 | 26 | RealMatrix data2 = SpectralMethods.averageHankelMatrix(res, 3); 27 | for (int i = 0; i < data2.getRowDimension(); ++i) { 28 | System.out.println(Arrays.toString(data2.getRow(i))); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/test/resources/config.json: -------------------------------------------------------------------------------- 1 | [{"version":1,"modificationTime":1410471400,"modifiedBy":"schassey","thresholds":[{"metricId":{"application":"yapache","metric":"requests","namespace":"Flickr","tags":{}},"modelId":"myModelID","modelType":"myModelType","sensitivity":{"units":"myUnits","high":100,"low":10}},{"metricId":{"application":"WS","metric":"3_WS_3","namespace":"FOOBAR","tags":{"Tags_0":"Linux_3"}},"modelId":"myModelID","modelType":"myModelType","sensitivity":{"units":"myUnits","high":100,"low":10}}]}] -------------------------------------------------------------------------------- /src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /src/test/resources/sample_config.ini: -------------------------------------------------------------------------------- 1 | ################### 2 | ### Main Config ### 3 | ################### 4 | 5 | # Specifies the detection window start time in timeseries. 6 | # If you want to use the following MAX_ANOMALY_TIME_AGO 7 | # make this parameter '0' 8 | DETECTION_WINDOW_START_TIME 0 9 | 10 | # Only show anomalies no older than this. 11 | # If this is set to 0, then only output an anomaly 12 | # if it occurs on the last time-stamp. 13 | MAX_ANOMALY_TIME_AGO 999999999 14 | 15 | # Denotes how much should the time-series be aggregated by. 16 | # If set to 1 or less, this setting is ignored. 17 | AGGREGATION 1 18 | 19 | # OP_TYPE specifies the operation type. 20 | # Options: DETECT_ANOMALY, 21 | # UPDATE_MODEL, 22 | # TRANSFORM_INPUT 23 | OP_TYPE DETECT_ANOMALY 24 | 25 | # TS_MODEL specifies the time-series 26 | # model type. 27 | # Options: AutoForecastModel 28 | # DoubleExponentialSmoothingModel 29 | # MovingAverageModel 30 | # MultipleLinearRegressionModel 31 | # NaiveForecastingModel 32 | # OlympicModel 33 | # PolynomialRegressionModel 34 | # RegressionModel 35 | # SimpleExponentialSmoothingModel 36 | # TripleExponentialSmoothingModel 37 | # WeightedMovingAverageModel 38 | # SpectralSmoother 39 | # NullModel 40 | TS_MODEL OlympicModel 41 | 42 | # AD_MODEL specifies the anomaly-detection 43 | # model type. 44 | # Options: ExtremeLowDensityModel 45 | # AdaptiveKernelDensityChangePointDetector 46 | # KSigmaModel 47 | # NaiveModel 48 | # DBScanModel 49 | # SimpleThresholdModel 50 | AD_MODEL ExtremeLowDensityModel 51 | 52 | # Type of the simple threshold model. 53 | # Options: AdaptiveMaxMinSigmaSensitivity 54 | # AdaptiveKSigmaSensitivity 55 | # SIMPLE_THRESHOLD_TYPE 56 | 57 | # Specifies the input src. 58 | # Options: STDIN 59 | # CSV 60 | INPUT CSV 61 | 62 | # Specifies the output src. 63 | # Options: STD_OUT, 64 | # ANOMALY_DB 65 | # GUI 66 | # PLOT 67 | OUTPUT STD_OUT 68 | 69 | # THRESHOLD specifies the threshold (e.g., sensitivity) for anomaly detection model. 70 | # Comment out to auto-detect all thresholds. 71 | # Options: mapee,mae,smape,mape,mase, 72 | # or single numeric for simple threshold model. 73 | # THRESHOLD mape#10,mase#15 74 | 75 | ##################################### 76 | ### Olympic Forecast Model Config ### 77 | ##################################### 78 | 79 | # The possible time-shifts for Olympic Scoring. 80 | TIME_SHIFTS 0,1 81 | 82 | # The possible base windows for Olympic Scoring. 83 | BASE_WINDOWS 24,168 84 | 85 | # Period specifies the periodicity of the 86 | # time-series (e.g., the difference between successive time-stamps). 87 | # Options: (numeric) 88 | # 0 - auto detect. 89 | # -1 - disable. 90 | PERIOD -1 91 | 92 | # Fill missing values. 93 | # Options: 0,1 94 | FILL_MISSING 0 95 | 96 | # NUM_WEEKS specifies the number of weeks 97 | # to use in OlympicScoring. 98 | NUM_WEEKS 8 99 | 100 | # NUM_TO_DROP specifies the number of 101 | # highest and lowest points to drop. 102 | NUM_TO_DROP 0 103 | 104 | # If dynamic parameters is set to 1, then 105 | # EGADS will dynamically vary parameters (NUM_WEEKS) 106 | # to produce the best fit. 107 | DYNAMIC_PARAMETERS 0 108 | 109 | ############################ 110 | ### NaiveModel Config ### 111 | ############################ 112 | 113 | # Window size where the spike is to be found. 114 | WINDOW_SIZE 0.1 115 | 116 | ################################################### 117 | ### ExtremeLowDensityModel & DBScanModel Config ### 118 | ################################################### 119 | 120 | # Denotes the expected % of anomalies 121 | # in your data. 122 | AUTO_SENSITIVITY_ANOMALY_PCNT 0.01 123 | 124 | # Refers to the cluster standard deviation. 125 | AUTO_SENSITIVITY_SD 3.0 126 | 127 | ####################################################### 128 | ### AdaptiveKernelDensityChangePointDetector Config ### 129 | ####################################################### 130 | 131 | # Change point detection parameters 132 | PRE_WINDOW_SIZE 48 133 | POST_WINDOW_SIZE 48 134 | CONFIDENCE 0.8 135 | 136 | ############################### 137 | ### SpectralSmoother Config ### 138 | ############################### 139 | 140 | # WINDOW_SIZE should be greater than the size of longest important seasonality. 141 | # By default it is set to 192 = 8 * 24 which is worth of 8 days (> 1 week) for hourly time-series. 142 | WINDOW_SIZE 192 143 | 144 | # FILTERING_METHOD specifies the filtering method for Spectral Smoothing 145 | # Options: GAP_RATIO (Recommended: FILTERING_PARAM = 0.01) 146 | # EIGEN_RATIO (Recommended: FILTERING_PARAM = 0.1) 147 | # EXPLICIT (Recommended: FILTERING_PARAM = 10) 148 | # K_GAP (Recommended: FILTERING_PARAM = 8) 149 | # VARIANCE (Recommended: FILTERING_PARAM = 0.99) 150 | # SMOOTHNESS (Recommended: FILTERING_PARAM = 0.97) 151 | FILTERING_METHOD GAP_RATIO 152 | 153 | FILTERING_PARAM 0.01 154 | 155 | ############################################## 156 | ### TripleExponentialSmoothingModel Config ### 157 | ############################################## 158 | 159 | # Smoothing factor for the level component 160 | ALPHA 0.75 161 | 162 | # Smoothing factor for the trend component 163 | BETA 0.001 164 | 165 | # Smoothing factor for the seasonal component 166 | GAMMA 0.001 167 | --------------------------------------------------------------------------------