├── MANIFEST.in ├── tests ├── testCase │ └── testCase_10000_20 │ │ ├── model.pkl │ │ └── config.yaml ├── __init__.py ├── cases │ ├── function_module_test.test_last_request_25.json │ ├── function_module_test.test_entire_request_12.json │ ├── json_dsat_test.cases.dsat_case25_0.json │ ├── json_last_cases.test_d_0.json │ ├── json_last_cases.test_e_0.json │ ├── json_spectrum_period.last_0.json │ ├── function_module_test.test_last_request_24.json │ ├── function_module_test.test_last_request_20.json │ ├── function_module_test.test_last_request_21.json │ ├── function_module_test.test_entire_need_fill_request_0.json │ ├── function_module_test.test_entire_need_fill_request_3.json │ ├── function_module_test.test_entire_need_fill_request_4.json │ ├── function_module_test.test_entire_need_fill_request_5.json │ ├── function_module_test.test_entire_need_fill_request_6.json │ ├── function_module_test.test_entire_need_fill_request_2.json │ ├── function_module_test.test_entire_need_fill_request_1.json │ ├── function_module_test.test_entire_need_fill_request_7.json │ ├── json_profiling_cases.case_0.json │ ├── json_dsat_test.cases.dsat_case4_0.json │ ├── json_last_cases.test_b_0.json │ ├── json_last_cases.test_a_0.json │ ├── json_dsat_test.cases.dsat_case6_0.json │ ├── json_dsat_test.cases.dsat_case3_0.json │ ├── json_dsat_test.cases.dsat_case5_0.json │ ├── function_module_test.test_entire_request_5.json │ ├── function_module_test.test_entire_request_9.json │ ├── json_dsat_test.cases.dsat_case21_0.json │ ├── json_dsat_test.cases.dsat_case16_0.json │ ├── json_dsat_test.cases.dsat_case17_0.json │ ├── json_dsat_test.cases.dsat_case18_0.json │ ├── json_dsat_test.cases.dsat_case20_0.json │ ├── json_dsat_test.cases.dsat_case22_0.json │ ├── json_dsat_test.cases.dsat_case15_0.json │ ├── json_dsat_test.cases.dsat_case19_0.json │ ├── function_module_test.test_entire_request_6.json │ ├── json_dsat_test.cases.dsat_case7_0.json │ ├── json_profiling_cases.case_3.json │ ├── function_module_test.test_extreme_data_0.json │ ├── function_module_test.test_extreme_data_2.json │ ├── json_dsat_test.cases.dsat_case26_0.json │ ├── json_dsat_test.cases.dsat_case27_0.json │ ├── function_module_test.test_extreme_data_1.json │ ├── function_module_test.test_extreme_data_3.json │ ├── function_module_test.test_last_request_15.json │ ├── json_dsat_test.cases.dsat_case14_0.json │ ├── json_dsat_test.cases.dsat_case28_0.json │ ├── json_dsat_test.cases.dsat_case29_0.json │ ├── json_dsat_test.cases.dsat_case30_0.json │ └── json_dsat_test.cases.dsat_case31_0.json ├── e2e_test_uvad.py └── e2e_test_mvad.py ├── .script ├── install_poetry.sh └── Dockerfile ├── src └── anomaly_detector │ ├── common │ ├── __init__.py │ ├── error_code.py │ ├── constants.py │ ├── exception.py │ ├── time_util.py │ └── data_processor.py │ ├── univariate │ ├── model │ │ ├── __init__.py │ │ ├── series_compete_processor.py │ │ ├── detect.py │ │ ├── dynamic_threshold.py │ │ ├── spectral_residual_model.py │ │ ├── hbos_detection.py │ │ └── seasonal_series.py │ ├── resource │ │ ├── __init__.py │ │ └── error_message.py │ ├── detectors │ │ ├── __init__.py │ │ ├── detector.py │ │ ├── z_score.py │ │ ├── dynamic_filter.py │ │ └── esd_filter.py │ ├── period │ │ ├── __init__.py │ │ ├── period_detect.py │ │ ├── simple.py │ │ └── spectrum.py │ ├── util │ │ ├── __init__.py │ │ ├── enum.py │ │ ├── stl_helpers.py │ │ ├── fields.py │ │ ├── r_stl.py │ │ └── date_utils.py │ ├── __init__.py │ └── filling_up │ │ └── __init__.py │ ├── multivariate │ ├── __init__.py │ ├── contract.py │ ├── util.py │ └── dataset.py │ ├── __init__.py │ └── base.py ├── CODE_OF_CONDUCT.md ├── setup.py ├── LICENSE ├── SUPPORT.md ├── .github └── workflows │ ├── ci.yml │ └── cd.yml ├── pyproject.toml ├── SECURITY.md ├── README.md └── anomaly-detector-sample-code.ipynb /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include ./src/anomaly_detector/univariate * 2 | -------------------------------------------------------------------------------- /tests/testCase/testCase_10000_20/model.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/anomaly-detector/HEAD/tests/testCase/testCase_10000_20/model.pkl -------------------------------------------------------------------------------- /.script/install_poetry.sh: -------------------------------------------------------------------------------- 1 | export POETRY_HOME=/opt/poetry 2 | python3 -m pip install --user pipx 3 | python3 -m pipx ensurepath 4 | pipx install poetry==1.3.2 5 | $POETRY_HOME/bin/poetry --version -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | -------------------------------------------------------------------------------- /src/anomaly_detector/common/__init__.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/model/__init__.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- -------------------------------------------------------------------------------- /src/anomaly_detector/multivariate/__init__.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/resource/__init__.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/detectors/__init__.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from .esd_filter import ESD 6 | from .z_score import ZScoreDetector 7 | from .dynamic_filter import DynamicThreshold 8 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/period/__init__.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from .simple import SimpleDetector 6 | from .spectrum import SpectrumDetector 7 | from .period_detect import period_detection 8 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/detectors/detector.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | import abc 6 | 7 | 8 | class AnomalyDetector(abc.ABC): 9 | def detect(self, directions, last_value=None): 10 | pass 11 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/util/__init__.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from .fields import * 6 | from .helpers import * 7 | from .stl_helpers import * 8 | from .r_stl import stl, stl_adjust_trend 9 | from .refine import * -------------------------------------------------------------------------------- /src/anomaly_detector/__init__.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from .multivariate.model import MultivariateAnomalyDetector 6 | from .univariate.univariate_anomaly_detection import UnivariateAnomalyDetector, EntireAnomalyDetector, LatestAnomalyDetector 7 | -------------------------------------------------------------------------------- /src/anomaly_detector/common/error_code.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | class ErrorCodes: 6 | # invalid data format 7 | InvalidDataFormat = "Invalid data format. {0}" 8 | InvalidTimestampFormat = "Timestamp should comply with ISO-8601, but not {0}." 9 | InvalidTimeRange = "" 10 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/__init__.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from .util import fit_trend 6 | from .util import get_period_pattern 7 | from .model.detect_model import AnomalyDetectionModel 8 | from .resource.error_message import InvalidJsonFormat, CustomSupportRequired 9 | from .period import period_detection -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/util/enum.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | default_gran_window = { 6 | "daily": 7 * 4, 7 | "minutely": 1440, 8 | "hourly": 24 * 7, 9 | "weekly": 12, 10 | "monthly": 12, 11 | "yearly": 12, 12 | "secondly": 1440, 13 | "microsecond": 1000, 14 | "none": 1440 15 | } 16 | 17 | -------------------------------------------------------------------------------- /src/anomaly_detector/common/constants.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | import enum 6 | 7 | TIMESTAMP = "timestamp" 8 | VALUE = "value" 9 | 10 | 11 | class AlignMode(enum.Enum): 12 | Inner = 1 13 | Outer = 2 14 | 15 | 16 | class FillNAMethod(enum.Enum): 17 | Previous = 1 18 | Subsequent = 2 19 | Linear = 3 20 | Zero = 4 21 | Fixed = 5 22 | -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_last_request_25.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 0}], "maxAnomalyRatio": 0.25, "sensitivity": 95}, "response": {"isAnomaly": true, "expectedValue": 0.9999999999999999}, "type": "last"} -------------------------------------------------------------------------------- /src/anomaly_detector/base.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from abc import abstractmethod 6 | 7 | import mlflow 8 | 9 | 10 | class BaseAnomalyDetector(mlflow.pyfunc.PythonModel): 11 | @abstractmethod 12 | def __init__(self, *args, **kwargs): 13 | pass 14 | 15 | @abstractmethod 16 | def predict(self, context, model_input, params=None): 17 | pass 18 | -------------------------------------------------------------------------------- /tests/testCase/testCase_10000_20/config.yaml: -------------------------------------------------------------------------------- 1 | normal_config: 2 | sliding_window: 200 3 | device: "cpu" 4 | small_window: 5 | sliding_window: 20 6 | device: "cpu" 7 | big_window: 8 | sliding_window: 3000 9 | device: "cpu" 10 | invalid_fillna_config: 11 | sliding_window: 200 12 | device: "cpu" 13 | fill_na_method: "invalid_method" 14 | invalid_fillna_value: 15 | sliding_window: 200 16 | device: "cpu" 17 | fill_na_method: "invalid_value" 18 | invalid_window_value: 19 | sliding_window: "invalid_window_value" 20 | device: "cpu" -------------------------------------------------------------------------------- /.script/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/mirror/docker/library/ubuntu:22.04 2 | ARG DEBIAN_FRONTEND=noninteractive 3 | 4 | RUN apt-get update -qq && \ 5 | apt-get upgrade -y && \ 6 | apt-get install -y gcc build-essential wget python3 python3-pip 7 | RUN rm -rf /var/lib/apt/lists/* 8 | RUN apt-get clean 9 | RUN pip3 install --upgrade pip 10 | RUN pip3 install poetry pytest 11 | WORKDIR /anomaly-detector 12 | COPY . . 13 | RUN cd anomaly-detector && poetry env use 3.10 && \ 14 | poetry lock --no-update && \ 15 | poetry install 16 | RUN pip3 install Cython numpy 17 | RUN cd .. 18 | RUN python3 setup.py build_ext --inplace 19 | RUN cd anomaly-detector/tests -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from setuptools import setup, Extension 6 | from Cython.Build import cythonize 7 | import numpy as np 8 | 9 | 10 | if __name__ == "__main__": 11 | extensions = [ 12 | Extension( 13 | "anomaly_detector.univariate._anomaly_kernel_cython", 14 | ["src/anomaly_detector/univariate/_anomaly_kernel_cython.pyx"], 15 | include_dirs=[np.get_include()] 16 | ) 17 | ] 18 | 19 | setup( 20 | setup_requires=["numpy"], 21 | ext_modules=cythonize(extensions) 22 | ) 23 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/filling_up/__init__.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from anomaly_detector.univariate.util.helpers import get_indices_from_timestamps 6 | from anomaly_detector.univariate.util.fields import FillUpMode, DEFAULT_FILL_UP_MODE 7 | from anomaly_detector.univariate.model.series_compete_processor import fill_up_on_demand, period_detection_with_filled_values 8 | from .fill_up import FillingUpProcess 9 | 10 | FillingUpProcess.fill_up_on_demand = fill_up_on_demand 11 | FillingUpProcess.period_detection_with_filled_values = period_detection_with_filled_values 12 | -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_entire_request_12.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 1}, {"value": 0}], "maxAnomalyRatio": 0.25, "sensitivity": 95}, "response": {"isAnomaly": [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true], "expectedValues": [0.9999999999999999, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999, 0.9999999999999998, 0.9999999999999998, 0.9999999999999998, 0.9999999999999998, 1.0, 0.9999999999999999, 0.9999999999999998, 1.0, 0.9999999999999999, 1.0, 1.0, 0.9999999999999999, 0.9999999999999998, 0.9999999999999998, 0.9999999999999998, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999]}, "type": "entire"} -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/period/period_detect.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from anomaly_detector.univariate.util import DEFAULT_TREND_TYPE, DEFAULT_DETECTOR_TYPE, \ 6 | DEFAULT_GRANULARITY, DEFAULT_PERIOD_THRESH, DEFAULT_MIN_VAR, DEFAULT_INTERVAL 7 | from anomaly_detector.univariate.period import SimpleDetector, SpectrumDetector 8 | 9 | 10 | def period_detection(series, trend_type=DEFAULT_TREND_TYPE, thresh=DEFAULT_PERIOD_THRESH, 11 | min_var=DEFAULT_MIN_VAR, detector_type=DEFAULT_DETECTOR_TYPE, 12 | granularity=DEFAULT_GRANULARITY, interval=DEFAULT_INTERVAL, skip_simple_detector = False, return_period_source = False): 13 | if not skip_simple_detector: 14 | period = SimpleDetector.detect(series, granularity, interval) 15 | if period: 16 | return [period, 0] if return_period_source else period 17 | spectrum_period = SpectrumDetector.detect(series, trend_type=trend_type, thresh=thresh, min_var=min_var, detector_type=detector_type) 18 | return [spectrum_period, 1] if return_period_source else spectrum_period 19 | -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case25_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2022-11-15T01:01:00Z", "value": 287.817}, {"timestamp": "2022-11-15T01:02:00Z", "value": 287.895}, {"timestamp": "2022-11-15T01:03:00Z", "value": 287.742}, {"timestamp": "2022-11-15T01:04:00Z", "value": 287.135}, {"timestamp": "2022-11-15T01:05:00Z", "value": 0.0}, {"timestamp": "2022-11-15T01:06:00Z", "value": 287.689}, {"timestamp": "2022-11-15T01:07:00Z", "value": 287.283}, {"timestamp": "2022-11-15T01:08:00Z", "value": 287.583}, {"timestamp": "2022-11-15T01:09:00Z", "value": 287.155}, {"timestamp": "2022-11-15T01:10:00Z", "value": 287.3}, {"timestamp": "2022-11-15T01:11:00Z", "value": 287.801}, {"timestamp": "2022-11-15T01:12:00Z", "value": 287.724}, {"timestamp": "2022-11-15T01:13:00Z", "value": 287.116}, {"timestamp": "2022-11-15T01:14:00Z", "value": 287.784}, {"timestamp": "2022-11-15T01:15:00Z", "value": 287.934}], "granularity": "minutely", "needDetectorId": true}, "response": {"isAnomaly": [false, false, false, false, true, false, false, false, false, false, false, false, false, false, false], "expectedValues": [287.817, 287.856, 287.818, 287.64725, 287.6055, 287.5799, 287.4575, 287.4257, 287.4297, 287.4020000000001, 287.42440000000005, 287.51260000000013, 287.4192000000001, 287.5450000000001, 287.6718000000001]}, "type": "entire"} -------------------------------------------------------------------------------- /tests/cases/json_last_cases.test_d_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2012-01-01 00:00:00", "value": 1}, {"timestamp": "2012-01-01 00:01:00", "value": 1}, {"timestamp": "2012-01-01 00:02:00", "value": 1}, {"timestamp": "2012-01-01 00:03:00", "value": 1}, {"timestamp": "2012-01-01 00:04:00", "value": 1}, {"timestamp": "2012-01-01 00:05:00", "value": 1}, {"timestamp": "2012-01-01 00:06:00", "value": 1}, {"timestamp": "2012-01-01 00:07:00", "value": 1}, {"timestamp": "2012-01-01 00:08:00", "value": 1}, {"timestamp": "2012-01-01 00:09:00", "value": 1}, {"timestamp": "2012-01-01 00:10:00", "value": 1}, {"timestamp": "2012-01-01 00:11:00", "value": 1}, {"timestamp": "2012-01-01 00:12:00", "value": 1}, {"timestamp": "2012-01-01 00:13:00", "value": 1}, {"timestamp": "2012-01-01 00:14:00", "value": 1}, {"timestamp": "2012-01-01 00:15:00", "value": 1}, {"timestamp": "2012-01-01 00:16:00", "value": 1}, {"timestamp": "2012-01-01 00:17:00", "value": 1}, {"timestamp": "2012-01-01 00:18:00", "value": 1}, {"timestamp": "2012-01-01 00:19:00", "value": 1}, {"timestamp": "2012-01-01 00:20:00", "value": 0}, {"timestamp": "2012-01-01 00:21:00", "value": 0}, {"timestamp": "2012-01-01 00:22:00", "value": 0}], "granularity": "minutely", "sensitivity": 99, "maxAnomalyRatio": 0.49}, "response": {"isAnomaly": true, "expectedValue": 0.4592913792858382}, "type": "last"} -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- 1 | # TODO: The maintainer of this repo has not yet edited this file 2 | 3 | **REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project? 4 | 5 | - **No CSS support:** Fill out this template with information about how to file issues and get help. 6 | - **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps. 7 | - **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide. 8 | 9 | *Then remove this first heading from this SUPPORT.MD file before publishing your repo.* 10 | 11 | # Support 12 | 13 | ## How to file issues and get help 14 | 15 | This project uses GitHub Issues to track bugs and feature requests. Please search the existing 16 | issues before filing new issues to avoid duplicates. For new issues, file your bug or 17 | feature request as a new Issue. 18 | 19 | For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE 20 | FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER 21 | CHANNEL. WHERE WILL YOU HELP PEOPLE?**. 22 | 23 | ## Microsoft Support Policy 24 | 25 | Support for this **PROJECT or PRODUCT** is limited to the resources listed above. 26 | -------------------------------------------------------------------------------- /tests/cases/json_last_cases.test_e_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2012-01-01 00:00:00", "value": 1}, {"timestamp": "2012-01-01 00:01:00", "value": 1}, {"timestamp": "2012-01-01 00:02:00", "value": 1}, {"timestamp": "2012-01-01 00:03:00", "value": 1}, {"timestamp": "2012-01-01 00:04:00", "value": 1}, {"timestamp": "2012-01-01 00:05:00", "value": 1}, {"timestamp": "2012-01-01 00:06:00", "value": 1}, {"timestamp": "2012-01-01 00:07:00", "value": 1}, {"timestamp": "2012-01-01 00:08:00", "value": 1}, {"timestamp": "2012-01-01 00:09:00", "value": 1}, {"timestamp": "2012-01-01 00:10:00", "value": 1}, {"timestamp": "2012-01-01 00:11:00", "value": 1}, {"timestamp": "2012-01-01 00:12:00", "value": 1}, {"timestamp": "2012-01-01 00:13:00", "value": 1}, {"timestamp": "2012-01-01 00:14:00", "value": 1}, {"timestamp": "2012-01-01 00:15:00", "value": 1}, {"timestamp": "2012-01-01 00:16:00", "value": 1}, {"timestamp": "2012-01-01 00:17:00", "value": 1}, {"timestamp": "2012-01-01 00:18:00", "value": 1}, {"timestamp": "2012-01-01 00:19:00", "value": 1}, {"timestamp": "2012-01-01 00:20:00", "value": 0}, {"timestamp": "2012-01-01 00:21:00", "value": 0}, {"timestamp": "2012-01-01 00:22:00", "value": 0}, {"timestamp": "2012-01-01 00:23:00", "value": 1}], "granularity": "minutely", "sensitivity": 99}, "response": {"isAnomaly": false, "expectedValue": 0.2972641780127731}, "type": "last"} -------------------------------------------------------------------------------- /tests/cases/json_spectrum_period.last_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2012-01-01 00:00:00", "value": 1}, {"timestamp": "2012-01-01 00:01:00", "value": 1}, {"timestamp": "2012-01-01 00:02:00", "value": 1}, {"timestamp": "2012-01-01 00:03:00", "value": 1}, {"timestamp": "2012-01-01 00:04:00", "value": 1}, {"timestamp": "2012-01-01 00:05:00", "value": 1}, {"timestamp": "2012-01-01 00:06:00", "value": 1}, {"timestamp": "2012-01-01 00:07:00", "value": 1}, {"timestamp": "2012-01-01 00:08:00", "value": 1}, {"timestamp": "2012-01-01 00:09:00", "value": 1}, {"timestamp": "2012-01-01 00:10:00", "value": 1}, {"timestamp": "2012-01-01 00:11:00", "value": 1}, {"timestamp": "2012-01-01 00:12:00", "value": 1}, {"timestamp": "2012-01-01 00:13:00", "value": 1}, {"timestamp": "2012-01-01 00:14:00", "value": 1}, {"timestamp": "2012-01-01 00:15:00", "value": 1}, {"timestamp": "2012-01-01 00:16:00", "value": 1}, {"timestamp": "2012-01-01 00:17:00", "value": 1}, {"timestamp": "2012-01-01 00:18:00", "value": 1}, {"timestamp": "2012-01-01 00:19:00", "value": 1}, {"timestamp": "2012-01-01 00:20:00", "value": 0}, {"timestamp": "2012-01-01 00:21:00", "value": 0}, {"timestamp": "2012-01-01 00:22:00", "value": 0}], "granularity": "minutely", "sensitivity": 99, "maxAnomalyRatio": 0.49, "needSpectrumPeriod": true}, "response": {"isAnomaly": true, "expectedValue": 0.4592913792858382, "spectrumPeriod": 0}, "type": "last"} -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_last_request_24.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1962-01-01T00:00:00Z", "value": 0}, {"timestamp": "1962-02-01T00:00:00Z", "value": 0}, {"timestamp": "1962-03-01T00:00:00Z", "value": 0}, {"timestamp": "1962-04-01T00:00:00Z", "value": 0}, {"timestamp": "1962-05-01T00:00:00Z", "value": 0}, {"timestamp": "1962-06-01T00:00:00Z", "value": 0}, {"timestamp": "1962-07-01T00:00:00Z", "value": 0}, {"timestamp": "1962-08-01T00:00:00Z", "value": 0}, {"timestamp": "1962-09-01T00:00:00Z", "value": 0}, {"timestamp": "1962-10-01T00:00:00Z", "value": 0}, {"timestamp": "1962-11-01T00:00:00Z", "value": 0}, {"timestamp": "1962-12-01T00:00:00Z", "value": 0}, {"timestamp": "1963-01-01T00:00:00Z", "value": 0}, {"timestamp": "1963-02-01T00:00:00Z", "value": 0}, {"timestamp": "1963-03-01T00:00:00Z", "value": 0}, {"timestamp": "1963-04-01T00:00:00Z", "value": 0}, {"timestamp": "1963-05-01T00:00:00Z", "value": 0}, {"timestamp": "1963-06-01T00:00:00Z", "value": 0}, {"timestamp": "1963-07-01T00:00:00Z", "value": 0}, {"timestamp": "1963-08-01T00:00:00Z", "value": 0}, {"timestamp": "1963-09-01T00:00:00Z", "value": 0}, {"timestamp": "1963-10-01T00:00:00Z", "value": 0}, {"timestamp": "1963-11-01T00:00:00Z", "value": 0}, {"timestamp": "1963-12-01T00:00:00Z", "value": 1}], "granularity": "monthly", "maxAnomalyRatio": 0.25, "sensitivity": 95}, "response": {"isAnomaly": true, "expectedValue": 0.0}, "type": "last"} -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_last_request_20.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1962-01-01T00:00:00Z", "value": 1}, {"timestamp": "1962-02-01T00:00:00Z", "value": 1}, {"timestamp": "1962-03-01T00:00:00Z", "value": 1}, {"timestamp": "1962-04-01T00:00:00Z", "value": 1}, {"timestamp": "1962-05-01T00:00:00Z", "value": 1}, {"timestamp": "1962-06-01T00:00:00Z", "value": 1}, {"timestamp": "1962-07-01T00:00:00Z", "value": 1}, {"timestamp": "1962-08-01T00:00:00Z", "value": 1}, {"timestamp": "1962-09-01T00:00:00Z", "value": 1}, {"timestamp": "1962-10-01T00:00:00Z", "value": 1}, {"timestamp": "1962-11-01T00:00:00Z", "value": 1}, {"timestamp": "1962-12-01T00:00:00Z", "value": 1}, {"timestamp": "1963-01-01T00:00:00Z", "value": 1}, {"timestamp": "1963-02-01T00:00:00Z", "value": 1}, {"timestamp": "1963-03-01T00:00:00Z", "value": 1}, {"timestamp": "1963-04-01T00:00:00Z", "value": 1}, {"timestamp": "1963-05-01T00:00:00Z", "value": 1}, {"timestamp": "1963-06-01T00:00:00Z", "value": 1}, {"timestamp": "1963-07-01T00:00:00Z", "value": 1}, {"timestamp": "1963-08-01T00:00:00Z", "value": 1}, {"timestamp": "1963-09-01T00:00:00Z", "value": 1}, {"timestamp": "1963-10-01T00:00:00Z", "value": 1}, {"timestamp": "1963-11-01T00:00:00Z", "value": 1}, {"timestamp": "1963-12-01T00:00:00Z", "value": 0}], "granularity": "monthly", "maxAnomalyRatio": 0.25, "sensitivity": 95}, "response": {"isAnomaly": true, "expectedValue": 0.9999999999999999}, "type": "last"} -------------------------------------------------------------------------------- /src/anomaly_detector/common/exception.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | class AnomalyDetectorException(Exception): 6 | def __init__(self, code="", message=""): 7 | if code == "": 8 | self.code = self.__class__.__name__ 9 | else: 10 | self.code = code 11 | self.message = message.replace("'", '"') 12 | 13 | def __str__(self): 14 | return f"error_code={self.code}, error_message={self.message}" 15 | 16 | def __repr__(self): 17 | return f"{self.__class__.__name__}('code={self.code}', message={self.message})" 18 | 19 | def to_dict(self): 20 | return {"code": self.code, "message": self.message} 21 | 22 | class AnomalyDetectionRequestError(Exception): 23 | """Raised when an error occurs in the request.""" 24 | def __init__(self, error_msg, error_code=None): 25 | if isinstance(error_msg, type(b'')): 26 | error_msg = error_msg.decode('UTF-8', 'replace') 27 | 28 | super(AnomalyDetectionRequestError, self).__init__( 29 | error_msg 30 | ) 31 | self.message = error_msg 32 | self.code = error_code 33 | 34 | class DataFormatError(AnomalyDetectorException): 35 | pass 36 | 37 | 38 | class InvalidParameterError(AnomalyDetectorException): 39 | pass 40 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/detectors/z_score.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from anomaly_detector.univariate.detectors.detector import AnomalyDetector 6 | import numpy as np 7 | from statsmodels import robust 8 | 9 | from anomaly_detector.univariate.util import Direction 10 | 11 | 12 | class ZScoreDetector(AnomalyDetector): 13 | def __init__(self, series, max_outliers): 14 | self.__series__ = series 15 | self.__max_outliers = max_outliers 16 | self.__median_value = np.median(series) 17 | self.__mad_value = robust.mad(self.__series__) 18 | if self.__mad_value == 0: 19 | self.__mad_value = np.std(self.__series__) 20 | self.__median_value = np.mean(self.__series__) 21 | 22 | def detect(self, direction, last_value=None): 23 | if self.__mad_value == 0: 24 | return [] 25 | detect_data = self.__series__ 26 | if direction == Direction.upper_tail: 27 | detect_data = self.__series__[::-1] 28 | selected_data = detect_data[:self.__max_outliers] 29 | selected_anomaly = selected_data[ 30 | (abs(selected_data - self.__median_value) / self.__mad_value) > 3].index.tolist() 31 | if selected_anomaly is None or len(selected_anomaly) == 0: 32 | return [] 33 | return selected_anomaly 34 | -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_last_request_21.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1962-01-01T00:00:00.001Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.002Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.003Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.004Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.005Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.006Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.007Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.008Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.009Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.010Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.011Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.012Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.013Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.014Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.015Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.016Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.017Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.018Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.019Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.020Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.021Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.022Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.023Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.024Z", "value": 0}], "granularity": "microsecond", "maxAnomalyRatio": 0.25, "sensitivity": 95}, "response": {"isAnomaly": true, "expectedValue": 0.9999999999999999}, "type": "last"} -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_entire_need_fill_request_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1962-01-01T00:00:00Z", "value": 1}, {"timestamp": "1962-02-01T00:00:00Z", "value": 1}, {"timestamp": "1962-03-01T00:00:00Z", "value": 1}, {"timestamp": "1962-04-01T00:00:00Z", "value": 1}, {"timestamp": "1962-05-01T00:00:00Z", "value": 1}, {"timestamp": "1962-06-01T00:00:00Z", "value": 1}, {"timestamp": "1962-07-01T00:00:00Z", "value": 1}, {"timestamp": "1962-08-01T00:00:00Z", "value": 1}, {"timestamp": "1962-09-01T00:00:00Z", "value": 1}, {"timestamp": "1962-10-01T00:00:00Z", "value": 1}, {"timestamp": "1962-11-01T00:00:00Z", "value": 1}, {"timestamp": "1963-05-01T00:00:00Z", "value": 1}, {"timestamp": "1963-06-01T00:00:00Z", "value": 1}, {"timestamp": "1963-07-01T00:00:00Z", "value": 1}, {"timestamp": "1963-08-01T00:00:00Z", "value": 1}, {"timestamp": "1963-09-01T00:00:00Z", "value": 1}, {"timestamp": "1963-10-01T00:00:00Z", "value": 1}, {"timestamp": "1963-11-01T00:00:00Z", "value": 1}, {"timestamp": "1963-12-01T00:00:00Z", "value": 0}], "granularity": "monthly", "maxAnomalyRatio": 0.25, "sensitivity": 95, "imputeStrategy": {"imputeMode": "auto"}}, "response": {"period": 0, "isAnomaly": [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true], "expectedValues": [0.9999999999999999, 0.9999999999999999, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0000000000000002, 1.0000000000000002, 1.0, 1.0000000000000002, 1.0000000000000002, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "type": "entire"} -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_entire_need_fill_request_3.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1962-01-01T00:00:00Z", "value": 1}, {"timestamp": "1962-02-01T00:00:00Z", "value": 1}, {"timestamp": "1962-03-01T00:00:00Z", "value": 1}, {"timestamp": "1962-04-01T00:00:00Z", "value": 1}, {"timestamp": "1962-05-01T00:00:00Z", "value": 1}, {"timestamp": "1962-06-01T00:00:00Z", "value": 1}, {"timestamp": "1962-07-01T00:00:00Z", "value": 1}, {"timestamp": "1962-08-01T00:00:00Z", "value": 1}, {"timestamp": "1962-09-01T00:00:00Z", "value": 1}, {"timestamp": "1962-10-01T00:00:00Z", "value": 1}, {"timestamp": "1962-11-01T00:00:00Z", "value": 1}, {"timestamp": "1963-05-01T00:00:00Z", "value": 1}, {"timestamp": "1963-06-01T00:00:00Z", "value": 1}, {"timestamp": "1963-07-01T00:00:00Z", "value": 1}, {"timestamp": "1963-08-01T00:00:00Z", "value": 1}, {"timestamp": "1963-09-01T00:00:00Z", "value": 1}, {"timestamp": "1963-10-01T00:00:00Z", "value": 1}, {"timestamp": "1963-11-01T00:00:00Z", "value": 1}, {"timestamp": "1963-12-01T00:00:00Z", "value": 0}], "granularity": "monthly", "maxAnomalyRatio": 0.25, "sensitivity": 95, "imputeStrategy": {"imputeMode": "zero"}}, "response": {"period": 0, "isAnomaly": [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true], "expectedValues": [0.9999999999999999, 0.9999999999999999, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0000000000000002, 1.0000000000000002, 1.0, 1.0000000000000002, 1.0000000000000002, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "type": "entire"} -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_entire_need_fill_request_4.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1962-01-01T00:00:00Z", "value": 1}, {"timestamp": "1962-02-01T00:00:00Z", "value": 1}, {"timestamp": "1962-03-01T00:00:00Z", "value": 1}, {"timestamp": "1962-04-01T00:00:00Z", "value": 1}, {"timestamp": "1962-05-01T00:00:00Z", "value": 1}, {"timestamp": "1962-06-01T00:00:00Z", "value": 1}, {"timestamp": "1962-07-01T00:00:00Z", "value": 1}, {"timestamp": "1962-08-01T00:00:00Z", "value": 1}, {"timestamp": "1962-09-01T00:00:00Z", "value": 1}, {"timestamp": "1962-10-01T00:00:00Z", "value": 1}, {"timestamp": "1962-11-01T00:00:00Z", "value": 1}, {"timestamp": "1963-05-01T00:00:00Z", "value": 1}, {"timestamp": "1963-06-01T00:00:00Z", "value": 1}, {"timestamp": "1963-07-01T00:00:00Z", "value": 1}, {"timestamp": "1963-08-01T00:00:00Z", "value": 1}, {"timestamp": "1963-09-01T00:00:00Z", "value": 1}, {"timestamp": "1963-10-01T00:00:00Z", "value": 1}, {"timestamp": "1963-11-01T00:00:00Z", "value": 1}, {"timestamp": "1963-12-01T00:00:00Z", "value": 0}], "granularity": "monthly", "maxAnomalyRatio": 0.25, "sensitivity": 95, "imputeStrategy": {"imputeMode": "previous"}}, "response": {"period": 0, "isAnomaly": [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true], "expectedValues": [0.9999999999999999, 0.9999999999999999, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0000000000000002, 1.0000000000000002, 1.0, 1.0000000000000002, 1.0000000000000002, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "type": "entire"} -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_entire_need_fill_request_5.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1962-01-01T00:00:00Z", "value": 1}, {"timestamp": "1962-02-01T00:00:00Z", "value": 1}, {"timestamp": "1962-03-01T00:00:00Z", "value": 1}, {"timestamp": "1962-04-01T00:00:00Z", "value": 1}, {"timestamp": "1962-05-01T00:00:00Z", "value": 1}, {"timestamp": "1962-06-01T00:00:00Z", "value": 1}, {"timestamp": "1962-07-01T00:00:00Z", "value": 1}, {"timestamp": "1962-08-01T00:00:00Z", "value": 1}, {"timestamp": "1962-09-01T00:00:00Z", "value": 1}, {"timestamp": "1962-10-01T00:00:00Z", "value": 1}, {"timestamp": "1962-11-01T00:00:00Z", "value": 1}, {"timestamp": "1963-05-01T00:00:00Z", "value": 1}, {"timestamp": "1963-06-01T00:00:00Z", "value": 1}, {"timestamp": "1963-07-01T00:00:00Z", "value": 1}, {"timestamp": "1963-08-01T00:00:00Z", "value": 1}, {"timestamp": "1963-09-01T00:00:00Z", "value": 1}, {"timestamp": "1963-10-01T00:00:00Z", "value": 1}, {"timestamp": "1963-11-01T00:00:00Z", "value": 1}, {"timestamp": "1963-12-01T00:00:00Z", "value": 0}], "granularity": "monthly", "maxAnomalyRatio": 0.25, "sensitivity": 95, "imputeStrategy": {"imputeMode": "linear"}}, "response": {"period": 0, "isAnomaly": [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true], "expectedValues": [0.9999999999999999, 0.9999999999999999, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0000000000000002, 1.0000000000000002, 1.0, 1.0000000000000002, 1.0000000000000002, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "type": "entire"} -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_entire_need_fill_request_6.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1962-01-01T00:00:00Z", "value": 1}, {"timestamp": "1962-02-01T00:00:00Z", "value": 1}, {"timestamp": "1962-03-01T00:00:00Z", "value": 1}, {"timestamp": "1962-04-01T00:00:00Z", "value": 1}, {"timestamp": "1962-05-01T00:00:00Z", "value": 1}, {"timestamp": "1962-06-01T00:00:00Z", "value": 1}, {"timestamp": "1962-07-01T00:00:00Z", "value": 1}, {"timestamp": "1962-08-01T00:00:00Z", "value": 1}, {"timestamp": "1962-09-01T00:00:00Z", "value": 1}, {"timestamp": "1962-10-01T00:00:00Z", "value": 1}, {"timestamp": "1962-11-01T00:00:00Z", "value": 1}, {"timestamp": "1963-05-01T00:00:00Z", "value": 1}, {"timestamp": "1963-06-01T00:00:00Z", "value": 1}, {"timestamp": "1963-07-01T00:00:00Z", "value": 1}, {"timestamp": "1963-08-01T00:00:00Z", "value": 1}, {"timestamp": "1963-09-01T00:00:00Z", "value": 1}, {"timestamp": "1963-10-01T00:00:00Z", "value": 1}, {"timestamp": "1963-11-01T00:00:00Z", "value": 1}, {"timestamp": "1963-12-01T00:00:00Z", "value": 0}], "granularity": "monthly", "maxAnomalyRatio": 0.25, "sensitivity": 95, "imputeStrategy": {"imputeMode": "notFill"}}, "response": {"period": 0, "isAnomaly": [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true], "expectedValues": [0.9999999999999999, 0.9999999999999999, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0000000000000002, 1.0000000000000002, 1.0, 1.0000000000000002, 1.0000000000000002, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "type": "entire"} -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_entire_need_fill_request_2.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1962-01-01T00:00:00Z", "value": 1}, {"timestamp": "1962-02-01T00:00:00Z", "value": 1}, {"timestamp": "1962-03-01T00:00:00Z", "value": 1}, {"timestamp": "1962-04-01T00:00:00Z", "value": 1}, {"timestamp": "1962-05-01T00:00:00Z", "value": 1}, {"timestamp": "1962-06-01T00:00:00Z", "value": 1}, {"timestamp": "1962-07-01T00:00:00Z", "value": 1}, {"timestamp": "1962-08-01T00:00:00Z", "value": 1}, {"timestamp": "1962-09-01T00:00:00Z", "value": 1}, {"timestamp": "1962-10-01T00:00:00Z", "value": 1}, {"timestamp": "1962-11-01T00:00:00Z", "value": 1}, {"timestamp": "1963-05-01T00:00:00Z", "value": 1}, {"timestamp": "1963-06-01T00:00:00Z", "value": 1}, {"timestamp": "1963-07-01T00:00:00Z", "value": 1}, {"timestamp": "1963-08-01T00:00:00Z", "value": 1}, {"timestamp": "1963-09-01T00:00:00Z", "value": 1}, {"timestamp": "1963-10-01T00:00:00Z", "value": 1}, {"timestamp": "1963-11-01T00:00:00Z", "value": 1}, {"timestamp": "1963-12-01T00:00:00Z", "value": 0}], "granularity": "monthly", "maxAnomalyRatio": 0.25, "sensitivity": 95, "imputeStrategy": {"imputeMode": "fixed", "imputeValue": 0}}, "response": {"period": 0, "isAnomaly": [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true], "expectedValues": [0.9999999999999999, 0.9999999999999999, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0000000000000002, 1.0000000000000002, 1.0, 1.0000000000000002, 1.0000000000000002, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "type": "entire"} -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_entire_need_fill_request_1.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1962-01-01T00:00:00Z", "value": 1}, {"timestamp": "1962-02-01T00:00:00Z", "value": 1}, {"timestamp": "1962-03-01T00:00:00Z", "value": 1}, {"timestamp": "1962-04-01T00:00:00Z", "value": 1}, {"timestamp": "1962-05-01T00:00:00Z", "value": 1}, {"timestamp": "1962-06-01T00:00:00Z", "value": 1}, {"timestamp": "1962-07-01T00:00:00Z", "value": 1}, {"timestamp": "1962-08-01T00:00:00Z", "value": 1}, {"timestamp": "1962-09-01T00:00:00Z", "value": 1}, {"timestamp": "1962-10-01T00:00:00Z", "value": 1}, {"timestamp": "1962-11-01T00:00:00Z", "value": 1}, {"timestamp": "1963-05-01T00:00:00Z", "value": 1}, {"timestamp": "1963-06-01T00:00:00Z", "value": 1}, {"timestamp": "1963-07-01T00:00:00Z", "value": 1}, {"timestamp": "1963-08-01T00:00:00Z", "value": 1}, {"timestamp": "1963-09-01T00:00:00Z", "value": 1}, {"timestamp": "1963-10-01T00:00:00Z", "value": 1}, {"timestamp": "1963-11-01T00:00:00Z", "value": 1}, {"timestamp": "1963-12-01T00:00:00Z", "value": 0}], "granularity": "monthly", "maxAnomalyRatio": 0.25, "sensitivity": 95, "imputeStrategy": {"imputeMode": "auto"}, "needFillUpConfirm": true}, "response": {"period": 0, "doFillUp": false, "isAnomaly": [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true], "expectedValues": [0.9999999999999999, 0.9999999999999999, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0000000000000002, 1.0000000000000002, 1.0, 1.0000000000000002, 1.0000000000000002, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "type": "entire"} -------------------------------------------------------------------------------- /src/anomaly_detector/common/time_util.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | import math 6 | 7 | from dateutil import parser, tz 8 | 9 | DT_FORMAT = "%Y-%m-%dT%H:%M:%SZ" 10 | DT_FORMAT_WITH_MICROSECOND = "%Y-%m-%dT%H:%M:%S.%fZ" 11 | 12 | 13 | def str_to_dt(s): 14 | """ 15 | This method parses string to datetime. If the string has time zone information, 16 | transpose the datetime to utc time zone. If the string doesn't have time zone 17 | information, assume it's a time in utc time zone. The datetime is returned with 18 | tzinfo as None. 19 | :param s: str 20 | :return: datetime in utc time zone with tzinfo as None 21 | """ 22 | parsed = parser.parse(s) 23 | if parsed.tzinfo is None: 24 | return parsed 25 | else: 26 | parsed = parsed.astimezone(tz.UTC) 27 | return parsed.replace(tzinfo=None) 28 | 29 | 30 | def dt_to_str(dt, fmt=DT_FORMAT): 31 | """ 32 | This method returns string format of datetime in utc time zone. If the datetime 33 | doesn't have time zone information, assume it's a time in utc time zone. 34 | :param dt: datetime 35 | :param fmt: format of str 36 | :return: str 37 | """ 38 | if dt.tzinfo is not None and dt.tzinfo != tz.UTC: 39 | dt = dt.astimezone(tz.UTC) 40 | return dt.strftime(fmt) 41 | 42 | 43 | def dt_to_str_with_microsecond(dt): 44 | return dt_to_str(dt, fmt=DT_FORMAT_WITH_MICROSECOND) 45 | -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_entire_need_fill_request_7.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1962-01-01T00:00:00.01Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.02Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.03Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.04Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.05Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.06Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.07Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.08Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.09Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.10Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.11Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.17Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.18Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.19Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.20Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.21Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.22Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.23Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.24Z", "value": 0}], "granularity": "microsecond", "customInterval": 10, "maxAnomalyRatio": 0.25, "sensitivity": 95, "imputeStrategy": {"imputeMode": "auto"}}, "response": {"period": 0, "isAnomaly": [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true], "expectedValues": [0.9999999999999999, 0.9999999999999999, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0000000000000002, 1.0000000000000002, 1.0, 1.0000000000000002, 1.0000000000000002, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "type": "entire"} -------------------------------------------------------------------------------- /src/anomaly_detector/multivariate/contract.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from dataclasses import dataclass 6 | from typing import List, Optional 7 | 8 | 9 | class MultiADConstants: 10 | TRAIN_CLIP_MIN = 0.0 11 | TRAIN_CLIP_MAX = 1.0 12 | INFERENCE_CLIP_MIN = -1000.0 13 | INFERENCE_CLIP_MAX = 1000.0 14 | ANOMALY_UPPER_THRESHOLD = 0.5 15 | ANOMALY_LOWER_THRESHOLD = 0.3 16 | TOP_ATTENTION_COUNT = 10 17 | DEFAULT_INPUT_SIZE = 200 18 | DEFAULT_THRESHOLD_WINDOW = 200 19 | 20 | 21 | @dataclass 22 | class NormalizeBase: 23 | max_values: List[float] 24 | min_values: List[float] 25 | 26 | 27 | @dataclass 28 | class MultiADConfig: 29 | data_dim: int 30 | cnn_kernel_size: int = 7 31 | input_size: int = MultiADConstants.DEFAULT_INPUT_SIZE # seq length for dataset 32 | gat_window_size: int = 100 # seq length for GAT 33 | hidden_size: int = 100 34 | epochs: int = 100 35 | batch_size: int = 128 36 | seed: int = 42 37 | z_dim: int = 100 38 | enc_dim: int = 100 39 | dec_dim: int = 100 40 | interval: int = 10 41 | lr: float = 1e-3 42 | beta: float = 1.0 43 | gamma: float = 1.0 44 | device: str = "cpu" 45 | horizon: int = 1 46 | dropout: float = 0.5 47 | save: str = "model" 48 | train_ratio: float = 0.8 49 | val_ratio: float = 0.1 50 | normalize_base: Optional[NormalizeBase] = None 51 | threshold_window: int = MultiADConstants.DEFAULT_THRESHOLD_WINDOW 52 | level: float = 0.8 53 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/model/series_compete_processor.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from anomaly_detector.univariate.util import FillUpMode 6 | from anomaly_detector.univariate.period import period_detection 7 | 8 | 9 | def fill_up_on_demand(filling_up_process, mode, fixed_value=None, period=None): 10 | if mode == FillUpMode.previous or mode == FillUpMode.use_last: 11 | return filling_up_process.fill_up(method='last') 12 | if mode == FillUpMode.fixed: 13 | return filling_up_process.fill_up(method='constant', number=fixed_value) 14 | if mode == FillUpMode.linear: 15 | return filling_up_process.fill_up(method='linear') 16 | if mode == FillUpMode.auto: 17 | return filling_up_process.fill_up(method='auto', period=period, if_exception='fill_with_last') 18 | return None, None 19 | 20 | 21 | def period_detection_with_filled_values(filling_up_process, mode, fixed_value=None, **kwargs): 22 | if filling_up_process.need_fill_up: 23 | if mode == FillUpMode.auto: 24 | values_to_detect_period, _ = fill_up_on_demand(filling_up_process, FillUpMode.previous) 25 | else: 26 | values_to_detect_period, _ = fill_up_on_demand(filling_up_process, mode, fixed_value) 27 | if values_to_detect_period is not None: 28 | return period_detection(values_to_detect_period, **kwargs) 29 | 30 | values_to_detect_period = filling_up_process.init_values 31 | return period_detection(values_to_detect_period, **kwargs) 32 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/detectors/dynamic_filter.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from anomaly_detector.univariate.detectors.detector import AnomalyDetector 6 | from anomaly_detector.univariate.util import Direction 7 | from anomaly_detector.univariate._anomaly_kernel_cython import dynamic_threshold 8 | 9 | 10 | class DynamicThreshold(AnomalyDetector): 11 | def __init__(self, series, max_outliers, threshold): 12 | self.__series__ = series 13 | self.__max_outliers = max_outliers 14 | self.__threshold__ = threshold 15 | 16 | def detect(self, direction, last_value=None): 17 | detect_data = self.__series__ 18 | if direction == Direction.upper_tail: 19 | detect_data = self.__series__.iloc[::-1] 20 | last_index = -1 21 | if last_value is not None: 22 | last_index = max(detect_data.index) 23 | last_index = detect_data.index.get_loc(last_index) 24 | selected_anomaly = dynamic_threshold(detect_data.tolist(), detect_data.index.tolist(), 25 | self.__max_outliers, 26 | self.__threshold__, 27 | True if direction == Direction.upper_tail else False, 28 | last_index 29 | ) 30 | 31 | if selected_anomaly is None or len(selected_anomaly) == 0: 32 | return [] 33 | return selected_anomaly 34 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/util/stl_helpers.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | import numpy as np 6 | 7 | MAPE_UB = 0.10 8 | MAPE_LB = 0.05 9 | 10 | 11 | def get_outlier(values, period): 12 | mean = np.mean(values) 13 | std = np.std(values) 14 | if std == 0: 15 | return [] 16 | outlier_index = values[np.abs(values - mean) / std >= 3].index 17 | if len(outlier_index) == 0: 18 | return [] 19 | period_bins = outlier_index % period 20 | unique, counts = np.unique(period_bins, return_counts=True) 21 | invalid_period_bin = unique[np.where(counts <= int((len(values) / period) / 2))] 22 | outlier_index = outlier_index[np.where(np.in1d(period_bins, invalid_period_bin))] 23 | return outlier_index 24 | 25 | 26 | def de_outlier_stl(series, stl_func, period, log_transform): 27 | series_decompose = stl_func(np.asarray(series), np=period, log_transform=log_transform) 28 | series_de_trend = series_decompose['remainder'] + series_decompose['seasonal'] 29 | outlier = get_outlier(series_de_trend, period) 30 | if len(outlier) == 0: 31 | return series_decompose 32 | 33 | series_de_trend[outlier] = np.nan 34 | series_de_trend = series_de_trend.fillna(method='ffill', limit=len(series_de_trend)).fillna(method='bfill') 35 | # series_de_trend = series_de_trend.groupby(series_de_trend.index % period). 36 | # ffill(limit=len(series_de_trend)).bfill() 37 | 38 | return stl_func(series_de_trend + series_decompose['trend'], np=period, log_transform=log_transform) 39 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: CI 4 | 5 | # Controls when the workflow will run 6 | on: 7 | # Triggers the workflow on push or pull request events but only for the "main" branch 8 | push: 9 | branches: ["main", "dev"] 10 | pull_request: 11 | branches: ["main", "dev"] 12 | 13 | # Allows you to run this workflow manually from the Actions tab 14 | workflow_dispatch: 15 | 16 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 17 | jobs: 18 | # This workflow contains a single job called "build" 19 | test: 20 | # The type of runner that the job will run on 21 | runs-on: ${{matrix.os}} 22 | strategy: 23 | matrix: 24 | os: ["ubuntu-latest"] 25 | python-version: ["3.10.8", "3.11", "3.12"] 26 | 27 | # Steps represent a sequence of tasks that will be executed as part of the job 28 | steps: 29 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 30 | - uses: actions/checkout@v4 31 | 32 | - name: setup python ${{matrix.python-version}} 33 | uses: actions/setup-python@v4 34 | with: 35 | python-version: ${{matrix.python-version}} 36 | 37 | - name: install package 38 | run: | 39 | python -m pip install -e . 40 | 41 | - name: run the test script 42 | working-directory: tests 43 | run: | 44 | pip install pytest 45 | pytest test_demo.py 46 | python uvad_test.py 47 | 48 | - name: Analyze code with pylint 49 | run: | 50 | pip install pylint 51 | pylint --exit-zero $(git ls-files '*.py') -------------------------------------------------------------------------------- /tests/e2e_test_uvad.py: -------------------------------------------------------------------------------- 1 | 2 | # --------------------------------------------------------- 3 | # Copyright (c) Microsoft Corporation. All rights reserved. 4 | # --------------------------------------------------------- 5 | 6 | import mlflow 7 | import numpy as np 8 | import pandas as pd 9 | from anomaly_detector import EntireAnomalyDetector 10 | from mlflow.models import infer_signature 11 | 12 | mlflow.set_tracking_uri(uri="http://127.0.0.1:8080") 13 | 14 | 15 | def main(): 16 | mlflow.set_experiment("MLflow Quickstart 2") 17 | 18 | params = { 19 | "granularity": "monthly", 20 | "maxAnomalyRatio": 0.25, 21 | "sensitivity": 95, 22 | "imputeMode": "auto" 23 | } 24 | 25 | with mlflow.start_run(): 26 | mlflow.log_params(params) 27 | mlflow.set_tag("Training Info", "Univariate Anomaly Detector") 28 | 29 | model = EntireAnomalyDetector() 30 | 31 | signature = infer_signature(params=params) 32 | 33 | model_info = mlflow.pyfunc.log_model( 34 | python_model=model, 35 | artifact_path="uvad_artifacts", 36 | registered_model_name="tracking-quickstart", 37 | signature=signature, 38 | ) 39 | print(model_info.model_uri) 40 | loaded_model = mlflow.pyfunc.load_model(model_info.model_uri) 41 | 42 | eval_data = np.ones(20) 43 | eval_data[-1] = 0 44 | eval_data = pd.DataFrame(eval_data, columns=["value"]) 45 | timestamps = pd.date_range(start="1962-01-01", periods=20, freq="M") 46 | eval_data["timestamp"] = timestamps 47 | results = loaded_model.predict( 48 | data=eval_data, 49 | params=params, 50 | ) 51 | print(results) 52 | 53 | 54 | if __name__ == "__main__": 55 | main() 56 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools", 4 | "wheel", 5 | "numpy==1.26.4", 6 | "cython==3.0.10"] 7 | build-backend = "setuptools.build_meta" 8 | 9 | [project] 10 | name = "time-series-anomaly-detector" 11 | version = "0.4.0" 12 | description = "Time Series Anomaly Detector" 13 | readme = "README.md" 14 | requires-python = ">=3.10.0" 15 | license = { file = "LICENSE" } 16 | keywords = ["machine learning", "time series", "anomaly detection"] 17 | authors = [ 18 | { name = "Microsoft Corporation", email = "ad-oss@microsoft.com" } 19 | ] 20 | maintainers = [ 21 | { name = "Microsoft Corporation", email = "ad-oss@microsoft.com" } 22 | ] 23 | classifiers = [ 24 | "Development Status :: 4 - Beta", 25 | "Intended Audience :: Developers", 26 | "Topic :: Scientific/Engineering :: Artificial Intelligence", 27 | "License :: OSI Approved :: MIT License", 28 | "Programming Language :: Python :: 3.10", 29 | "Programming Language :: Python :: 3.11", 30 | "Programming Language :: Python :: 3.12" 31 | ] 32 | 33 | dependencies = [ 34 | "numpy<2, >=1.23.5", 35 | "pandas<=2.2.3, >=1.3.5", 36 | "seasonal>=0.3.1", 37 | "scipy<1.13.0, >=1.9.3", 38 | "pytz>=2018.9", 39 | "rstl>=0.1.3", 40 | "psutil>=6.1.1", 41 | "statsmodels>=0.14.1", 42 | "scikit-learn>=1.3.2", 43 | "arch>=6.3.0", 44 | "torch>=1.13.1", 45 | "tqdm>=4.66.1", 46 | "mlflow>=3.1.0,<4.0", 47 | "pytest>=7.4.4", 48 | ] 49 | 50 | [project.urls] 51 | "Homepage" = "https://github.com/microsoft/anomaly-detector" 52 | 53 | [tool.setuptools.packages.find] 54 | where = ["src"] 55 | 56 | [tool.cibuildwheel] 57 | skip = ["pp*", "*-musllinux*"] 58 | test-command = [ 59 | "python {project}/tests/uvad_test.py", 60 | "python {project}/tests/test_demo.py" 61 | ] -------------------------------------------------------------------------------- /tests/cases/json_profiling_cases.case_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2019-04-07 00:00:00", "value": 66.52747252747254}, {"timestamp": "2019-04-07 00:05:00", "value": 65.8262012012012}, {"timestamp": "2019-04-07 00:10:00", "value": 65.32795698924731}, {"timestamp": "2019-04-07 00:15:00", "value": 63.86878182530357}, {"timestamp": "2019-04-07 00:20:00", "value": 63.80686352753392}, {"timestamp": "2019-04-07 00:25:00", "value": 64.12922951487974}, {"timestamp": "2019-04-07 00:30:00", "value": 64.26258843112775}, {"timestamp": "2019-04-07 00:35:00", "value": 64.08354537743851}, {"timestamp": "2019-04-07 00:40:00", "value": 64.77029360967185}, {"timestamp": "2019-04-07 00:45:00", "value": 65.5095090667846}, {"timestamp": "2019-04-07 00:50:00", "value": 65.98459447213412}, {"timestamp": "2019-04-07 00:55:00", "value": 66.6232018561485}, {"timestamp": "2019-04-07 01:00:00", "value": 66.91733585262163}, {"timestamp": "2019-04-07 01:05:00", "value": 67.28543783260764}, {"timestamp": "2019-04-07 01:10:00", "value": 68.35384615384615}, {"timestamp": "2019-04-07 01:15:00", "value": 68.9026727181039}, {"timestamp": "2019-04-07 01:20:00", "value": 69.30270546197039}, {"timestamp": "2019-04-07 01:25:00", "value": 70.09588327253779}, {"timestamp": "2019-04-07 01:30:00", "value": 70.60721868365181}, {"timestamp": "2019-04-07 01:35:00", "value": 71.34017278617709}], "granularity": "minutely", "maxAnomalyRatio": 0.01, "sensitivity": 95, "customInterval": 5}, "response": {"expectedValues": [66.52747252747254, 66.17683686433688, 65.89387690597368, 65.38760313580616, 65.07145521415171, 64.59180661163315, 64.27908405761846, 64.0302017352567, 64.21050409213035, 64.5510331999805, 64.92210619143138, 65.39422887643553, 65.96098697147215, 66.4640158160593, 67.03288323347161, 67.61649888266557, 68.15239960382993, 68.78810908781318, 69.45246525802202, 70.04973058448823]}, "type": "entire"} -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case4_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2020-01-29 00:00:00", "value": 42517838}, {"timestamp": "2020-01-30 00:00:00", "value": 40162799}, {"timestamp": "2020-01-31 00:00:00", "value": 35489472}, {"timestamp": "2020-02-01 00:00:00", "value": 15946425}, {"timestamp": "2020-02-02 00:00:00", "value": 14674424}, {"timestamp": "2020-02-03 00:00:00", "value": 37493883}, {"timestamp": "2020-02-04 00:00:00", "value": 40287775}, {"timestamp": "2020-02-05 00:00:00", "value": 40177038}, {"timestamp": "2020-02-06 00:00:00", "value": 38627206}, {"timestamp": "2020-02-07 00:00:00", "value": 34559069}, {"timestamp": "2020-02-08 00:00:00", "value": 15593195}, {"timestamp": "2020-02-09 00:00:00", "value": 15288930}, {"timestamp": "2020-02-10 00:00:00", "value": 40719843}, {"timestamp": "2020-02-11 00:00:00", "value": 41044248}, {"timestamp": "2020-02-12 00:00:00", "value": 39737054}, {"timestamp": "2020-02-13 00:00:00", "value": 38212400}, {"timestamp": "2020-02-14 00:00:00", "value": 32798622}, {"timestamp": "2020-02-15 00:00:00", "value": 15208345}, {"timestamp": "2020-02-16 00:00:00", "value": 14652225}, {"timestamp": "2020-02-17 00:00:00", "value": 38463952}, {"timestamp": "2020-02-18 00:00:00", "value": 40238478}, {"timestamp": "2020-02-19 00:00:00", "value": 39468397}, {"timestamp": "2020-02-20 00:00:00", "value": 37914518}, {"timestamp": "2020-02-21 00:00:00", "value": 33020449}, {"timestamp": "2020-02-22 00:00:00", "value": 14951144}, {"timestamp": "2020-02-23 00:00:00", "value": 13993547}, {"timestamp": "2020-02-24 00:00:00", "value": 34367660}, {"timestamp": "2020-02-25 00:00:00", "value": 34875573}, {"timestamp": "2020-02-26 00:00:00", "value": 38093022}], "granularity": "daily", "customInterval": 1, "sensitivity": 99, "maxAnomalyRatio": 0.15, "needDetectorId": true, "period": 7}, "response": {"expectedValue": 38093022}, "type": "last"} -------------------------------------------------------------------------------- /tests/cases/json_last_cases.test_b_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2019-08-10T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-11T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-12T00:00:00+00:00", "value": 16500}, {"timestamp": "2019-08-13T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-14T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-15T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-16T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-17T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-18T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-19T00:00:00+00:00", "value": 10000}, {"timestamp": "2019-08-20T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-21T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-22T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-23T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-24T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-25T00:00:00+00:00", "value": 16500}, {"timestamp": "2019-08-26T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-27T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-28T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-29T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-30T00:00:00+00:00", "value": 9000}, {"timestamp": "2019-08-31T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-09-01T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-09-02T00:00:00+00:00", "value": 16500}, {"timestamp": "2019-09-03T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-09-04T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-09-05T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-09-06T00:00:00+00:00", "value": 9500}, {"timestamp": "2019-09-07T00:00:00+00:00", "value": 14500}], "granularity": "daily", "sensitivity": 99}, "response": {"isAnomaly": false, "expectedValue": 13143.764784523093}, "type": "last"} -------------------------------------------------------------------------------- /tests/cases/json_last_cases.test_a_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2019-08-09T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-10T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-11T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-12T00:00:00+00:00", "value": 16500}, {"timestamp": "2019-08-13T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-14T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-15T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-16T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-17T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-18T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-19T00:00:00+00:00", "value": 10000}, {"timestamp": "2019-08-20T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-21T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-22T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-23T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-24T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-25T00:00:00+00:00", "value": 16500}, {"timestamp": "2019-08-26T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-27T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-28T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-29T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-08-30T00:00:00+00:00", "value": 9000}, {"timestamp": "2019-08-31T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-09-01T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-09-02T00:00:00+00:00", "value": 16500}, {"timestamp": "2019-09-03T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-09-04T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-09-05T00:00:00+00:00", "value": 14500}, {"timestamp": "2019-09-06T00:00:00+00:00", "value": 9500}], "granularity": "daily", "sensitivity": 99, "maxAnomalyRatio": 0.35}, "response": {"isAnomaly": true, "expectedValue": 14589.515479829284}, "type": "last"} -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case6_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2020-01-31 00:00:00", "value": 5300384343}, {"timestamp": "2020-02-01 00:00:00", "value": 1497619653}, {"timestamp": "2020-02-02 00:00:00", "value": 1426069716}, {"timestamp": "2020-02-03 00:00:00", "value": 6117033424}, {"timestamp": "2020-02-04 00:00:00", "value": 6260968596}, {"timestamp": "2020-02-05 00:00:00", "value": 6030784237}, {"timestamp": "2020-02-06 00:00:00", "value": 5903108570}, {"timestamp": "2020-02-07 00:00:00", "value": 5261308245}, {"timestamp": "2020-02-08 00:00:00", "value": 1447697667}, {"timestamp": "2020-02-09 00:00:00", "value": 1457367461}, {"timestamp": "2020-02-10 00:00:00", "value": 6285313007}, {"timestamp": "2020-02-11 00:00:00", "value": 6234744975}, {"timestamp": "2020-02-12 00:00:00", "value": 6220011806}, {"timestamp": "2020-02-13 00:00:00", "value": 6025899838}, {"timestamp": "2020-02-14 00:00:00", "value": 5193291136}, {"timestamp": "2020-02-15 00:00:00", "value": 1484423106}, {"timestamp": "2020-02-16 00:00:00", "value": 1446584139}, {"timestamp": "2020-02-17 00:00:00", "value": 5746018210}, {"timestamp": "2020-02-18 00:00:00", "value": 6349255639}, {"timestamp": "2020-02-19 00:00:00", "value": 6123592815}, {"timestamp": "2020-02-20 00:00:00", "value": 5978575236}, {"timestamp": "2020-02-21 00:00:00", "value": 5146719593}, {"timestamp": "2020-02-22 00:00:00", "value": 1442216387}, {"timestamp": "2020-02-23 00:00:00", "value": 1414381347}, {"timestamp": "2020-02-24 00:00:00", "value": 5886306580}, {"timestamp": "2020-02-25 00:00:00", "value": 5913229731}, {"timestamp": "2020-02-26 00:00:00", "value": 6118602640}, {"timestamp": "2020-02-27 00:00:00", "value": 6170260175}, {"timestamp": "2020-02-28 00:00:00", "value": 5501728886}], "granularity": "daily", "customInterval": 1, "sensitivity": 99, "maxAnomalyRatio": 0.15, "needDetectorId": true, "period": 7}, "response": {"expectedValue": 5501728886.0}, "type": "last"} -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case3_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2020-01-29 00:00:00", "value": 42517838.0}, {"timestamp": "2020-01-30 00:00:00", "value": 40162799.0}, {"timestamp": "2020-01-31 00:00:00", "value": 35489472.0}, {"timestamp": "2020-02-01 00:00:00", "value": 15946425.0}, {"timestamp": "2020-02-02 00:00:00", "value": 14674424.0}, {"timestamp": "2020-02-03 00:00:00", "value": 37493883.0}, {"timestamp": "2020-02-04 00:00:00", "value": 40287775.0}, {"timestamp": "2020-02-05 00:00:00", "value": 40177038.0}, {"timestamp": "2020-02-06 00:00:00", "value": 38627206.0}, {"timestamp": "2020-02-07 00:00:00", "value": 34559069.0}, {"timestamp": "2020-02-08 00:00:00", "value": 15593195.0}, {"timestamp": "2020-02-09 00:00:00", "value": 15288930.0}, {"timestamp": "2020-02-10 00:00:00", "value": 40719843.0}, {"timestamp": "2020-02-11 00:00:00", "value": 41044248.0}, {"timestamp": "2020-02-12 00:00:00", "value": 39737054.0}, {"timestamp": "2020-02-13 00:00:00", "value": 38212400.0}, {"timestamp": "2020-02-14 00:00:00", "value": 32798622.0}, {"timestamp": "2020-02-15 00:00:00", "value": 15208345.0}, {"timestamp": "2020-02-16 00:00:00", "value": 14652225.0}, {"timestamp": "2020-02-17 00:00:00", "value": 38463952.0}, {"timestamp": "2020-02-18 00:00:00", "value": 40238478.0}, {"timestamp": "2020-02-19 00:00:00", "value": 39468397.0}, {"timestamp": "2020-02-20 00:00:00", "value": 37914518.0}, {"timestamp": "2020-02-21 00:00:00", "value": 33020449.0}, {"timestamp": "2020-02-22 00:00:00", "value": 14951144.0}, {"timestamp": "2020-02-23 00:00:00", "value": 13993547.0}, {"timestamp": "2020-02-24 00:00:00", "value": 34367660.0}, {"timestamp": "2020-02-25 00:00:00", "value": 34875573.0}, {"timestamp": "2020-02-26 00:00:00", "value": 38093022.0}], "granularity": "daily", "customInterval": 1, "sensitivity": 99, "maxAnomalyRatio": 0.15, "needDetectorId": true}, "response": {"isAnomaly": false, "expectedValue": 38079076.35677312}, "type": "last"} -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case5_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2020-01-30 00:00:00", "value": 42141722.0}, {"timestamp": "2020-01-31 00:00:00", "value": 38543285.625}, {"timestamp": "2020-02-01 00:00:00", "value": 11487517.25}, {"timestamp": "2020-02-02 00:00:00", "value": 9435094.875}, {"timestamp": "2020-02-03 00:00:00", "value": 43890432.5}, {"timestamp": "2020-02-04 00:00:00", "value": 43094278.125}, {"timestamp": "2020-02-05 00:00:00", "value": 40721800.75}, {"timestamp": "2020-02-06 00:00:00", "value": 39529350.375}, {"timestamp": "2020-02-07 00:00:00", "value": 34263852.0}, {"timestamp": "2020-02-08 00:00:00", "value": 5163465.625}, {"timestamp": "2020-02-09 00:00:00", "value": 3769636.250000001}, {"timestamp": "2020-02-10 00:00:00", "value": 38596704.875}, {"timestamp": "2020-02-11 00:00:00", "value": 35557030.5}, {"timestamp": "2020-02-12 00:00:00", "value": 35340864.125}, {"timestamp": "2020-02-13 00:00:00", "value": 33789747.75}, {"timestamp": "2020-02-14 00:00:00", "value": 27984867.375}, {"timestamp": "2020-02-15 00:00:00", "value": 672171.0}, {"timestamp": "2020-02-16 00:00:00", "value": -188625.375}, {"timestamp": "2020-02-17 00:00:00", "value": 31250461.25}, {"timestamp": "2020-02-18 00:00:00", "value": 32044669.875000004}, {"timestamp": "2020-02-19 00:00:00", "value": 30468912.500000004}, {"timestamp": "2020-02-20 00:00:00", "value": 28981138.125}, {"timestamp": "2020-02-21 00:00:00", "value": 23320436.75}, {"timestamp": "2020-02-22 00:00:00", "value": -3741173.624999998}, {"timestamp": "2020-02-23 00:00:00", "value": -6075720.0}, {"timestamp": "2020-02-24 00:00:00", "value": 23523623.625}, {"timestamp": "2020-02-25 00:00:00", "value": 26484273.25}, {"timestamp": "2020-02-26 00:00:00", "value": 26266574.875}, {"timestamp": "2020-02-27 00:00:00", "value": 25193122.5}], "granularity": "daily", "customInterval": 1, "sensitivity": 99, "maxAnomalyRatio": 0.15, "needDetectorId": true, "period": 7}, "response": {"expectedValue": 25193122.5}, "type": "last"} -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_entire_request_5.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1962-01-01T00:00:00Z", "value": 1}, {"timestamp": "1962-02-01T00:00:00Z", "value": 1}, {"timestamp": "1962-03-01T00:00:00Z", "value": 1}, {"timestamp": "1962-04-01T00:00:00Z", "value": 1}, {"timestamp": "1962-05-01T00:00:00Z", "value": 1}, {"timestamp": "1962-06-01T00:00:00Z", "value": 1}, {"timestamp": "1962-07-01T00:00:00Z", "value": 1}, {"timestamp": "1962-08-01T00:00:00Z", "value": 1}, {"timestamp": "1962-09-01T00:00:00Z", "value": 1}, {"timestamp": "1962-10-01T00:00:00Z", "value": 1}, {"timestamp": "1962-11-01T00:00:00Z", "value": 1}, {"timestamp": "1962-12-01T00:00:00Z", "value": 1}, {"timestamp": "1963-01-01T00:00:00Z", "value": 1}, {"timestamp": "1963-02-01T00:00:00Z", "value": 1}, {"timestamp": "1963-03-01T00:00:00Z", "value": 1}, {"timestamp": "1963-04-01T00:00:00Z", "value": 1}, {"timestamp": "1963-05-01T00:00:00Z", "value": 1}, {"timestamp": "1963-06-01T00:00:00Z", "value": 1}, {"timestamp": "1963-07-01T00:00:00Z", "value": 1}, {"timestamp": "1963-08-01T00:00:00Z", "value": 1}, {"timestamp": "1963-09-01T00:00:00Z", "value": 1}, {"timestamp": "1963-10-01T00:00:00Z", "value": 1}, {"timestamp": "1963-11-01T00:00:00Z", "value": 1}, {"timestamp": "1963-12-01T00:00:00Z", "value": 0}], "granularity": "monthly", "maxAnomalyRatio": 0.25, "sensitivity": 95}, "response": {"isAnomaly": [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true], "expectedValues": [0.9999999999999999, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999, 0.9999999999999998, 0.9999999999999998, 0.9999999999999998, 0.9999999999999998, 1.0, 0.9999999999999999, 0.9999999999999998, 1.0, 0.9999999999999999, 1.0, 1.0, 0.9999999999999999, 0.9999999999999998, 0.9999999999999998, 0.9999999999999998, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999]}, "type": "entire"} -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_entire_request_9.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1962-01-01T00:00:00Z", "value": 1}, {"timestamp": "1962-02-01T00:00:00Z", "value": 1}, {"timestamp": "1962-03-01T00:00:00Z", "value": 1}, {"timestamp": "1962-04-01T00:00:00Z", "value": 1}, {"timestamp": "1962-05-01T00:00:00Z", "value": 1}, {"timestamp": "1962-06-01T00:00:00Z", "value": 0}, {"timestamp": "1962-07-01T00:00:00Z", "value": 1}, {"timestamp": "1962-08-01T00:00:00Z", "value": 1}, {"timestamp": "1962-09-01T00:00:00Z", "value": 1}, {"timestamp": "1962-10-01T00:00:00Z", "value": 1}, {"timestamp": "1962-11-01T00:00:00Z", "value": 1}, {"timestamp": "1962-12-01T00:00:00Z", "value": 1}, {"timestamp": "1963-01-01T00:00:00Z", "value": 1}, {"timestamp": "1963-02-01T00:00:00Z", "value": 1}, {"timestamp": "1963-03-01T00:00:00Z", "value": 1}, {"timestamp": "1963-04-01T00:00:00Z", "value": 1}, {"timestamp": "1963-05-01T00:00:00Z", "value": 1}, {"timestamp": "1963-06-01T00:00:00Z", "value": 1}, {"timestamp": "1963-07-01T00:00:00Z", "value": 1}, {"timestamp": "1963-08-01T00:00:00Z", "value": 1}, {"timestamp": "1963-09-01T00:00:00Z", "value": 1}, {"timestamp": "1963-10-01T00:00:00Z", "value": 1}, {"timestamp": "1963-11-01T00:00:00Z", "value": 1}, {"timestamp": "1963-12-01T00:00:00Z", "value": 1}], "granularity": "monthly", "maxAnomalyRatio": 0.25, "sensitivity": 95}, "response": {"isAnomaly": [false, false, false, false, false, true, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false], "expectedValues": [0.9999999999999999, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999, 0.9999999999999998, 0.9999999999999998, 0.9999999999999998, 0.9999999999999998, 1.0, 0.9999999999999999, 0.9999999999999998, 1.0, 0.9999999999999999, 1.0, 1.0, 0.9999999999999999, 0.9999999999999998, 0.9999999999999998, 0.9999999999999998, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999]}, "type": "entire"} -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/model/detect.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | import pandas as pd 6 | 7 | from anomaly_detector.univariate.util import Direction, IsPositiveAnomaly, IsNegativeAnomaly, IsAnomaly 8 | 9 | 10 | def detect(directions, detectors, max_outliers, num_obs, last_value=None): 11 | anomalies = {} 12 | for direction in directions: 13 | anomaly = {} 14 | for i in range(len(detectors)): 15 | selected_anomaly = detectors[i].detect(direction=direction, last_value=last_value) 16 | if selected_anomaly is None or len(selected_anomaly) == 0: 17 | continue 18 | for k in range(len(selected_anomaly)): 19 | index = selected_anomaly[k] 20 | if index in anomaly: 21 | anomaly[index] += k 22 | else: 23 | anomaly[index] = k + i * num_obs 24 | 25 | sorted_anomaly = sorted(anomaly, key=anomaly.get) 26 | anomalies[direction] = pd.DataFrame(sorted_anomaly[:min(max_outliers, len(sorted_anomaly))], 27 | columns=['Idx']) 28 | if direction == Direction.upper_tail: 29 | anomalies[direction][IsPositiveAnomaly] = True 30 | else: 31 | anomalies[direction][IsNegativeAnomaly] = True 32 | 33 | anomalies[direction].set_index('Idx', inplace=True) 34 | 35 | anomaly_frame = pd.DataFrame() 36 | for direction in anomalies: 37 | anomaly_frame = anomaly_frame.join(anomalies[direction], how='outer') 38 | 39 | if len(anomaly_frame) != 0: 40 | anomaly_frame[IsAnomaly] = True 41 | else: 42 | return pd.DataFrame(columns=[IsAnomaly, IsPositiveAnomaly, 43 | IsNegativeAnomaly, 44 | 'Idx']) 45 | return anomaly_frame 46 | -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case21_0.json: -------------------------------------------------------------------------------- 1 | {"request":{"boundaryVersion":"V3","series":[{"timestamp":"2021-09-28T00:00:00Z","value":0.7395937094377659},{"timestamp":"2021-09-29T00:00:00Z","value":0.739444310338565},{"timestamp":"2021-09-30T00:00:00Z","value":0.7430499500163679},{"timestamp":"2021-10-01T00:00:00Z","value":0.7418095766949304},{"timestamp":"2021-10-02T00:00:00Z","value":0.695133513888751},{"timestamp":"2021-10-03T00:00:00Z","value":0.6834289585832579},{"timestamp":"2021-10-04T00:00:00Z","value":0.7387050476849135},{"timestamp":"2021-10-05T00:00:00Z","value":0.7417863667707081},{"timestamp":"2021-10-06T00:00:00Z","value":0.7429737416603015},{"timestamp":"2021-10-07T00:00:00Z","value":0.7439187242213674},{"timestamp":"2021-10-08T00:00:00Z","value":0.7431277403034221},{"timestamp":"2021-10-09T00:00:00Z","value":0.702743635913485},{"timestamp":"2021-10-10T00:00:00Z","value":0.689069573466121},{"timestamp":"2021-10-11T00:00:00Z","value":0.7425535300340591},{"timestamp":"2021-10-12T00:00:00Z","value":0.741774507061165},{"timestamp":"2021-10-13T00:00:00Z","value":0.7397328019579146},{"timestamp":"2021-10-14T00:00:00Z","value":0.741853027056},{"timestamp":"2021-10-15T00:00:00Z","value":0.7389734121919656},{"timestamp":"2021-10-16T00:00:00Z","value":0.6900559320121721},{"timestamp":"2021-10-17T00:00:00Z","value":0.6890958284992494},{"timestamp":"2021-10-18T00:00:00Z","value":0.7437165482786},{"timestamp":"2021-10-19T00:00:00Z","value":0.7456919615602259},{"timestamp":"2021-10-20T00:00:00Z","value":0.746957831987582},{"timestamp":"2021-10-21T00:00:00Z","value":0.4644057970590395},{"timestamp":"2021-10-22T00:00:00Z","value":0.380992851189023},{"timestamp":"2021-10-23T00:00:00Z","value":0.376012984356072},{"timestamp":"2021-10-24T00:00:00Z","value":0.208564593827555},{"timestamp":"2021-10-25T00:00:00Z","value":0.3748380814304175},{"timestamp":"2021-10-26T00:00:00Z","value":0.3814443870744565}],"granularity":"daily","customInterval":1,"sensitivity":100,"maxAnomalyRatio":0.15},"response":{"isAnomaly":true,"expectedValue":0.7485865843482159},"type":"last"} -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case16_0.json: -------------------------------------------------------------------------------- 1 | {"request":{"boundaryVersion":"V3","series":[{"timestamp":"2021-09-28T00:00:00Z","value":0.9800753512864412},{"timestamp":"2021-09-29T00:00:00Z","value":0.9801664153295204},{"timestamp":"2021-09-30T00:00:00Z","value":0.9799775507179591},{"timestamp":"2021-10-01T00:00:00Z","value":0.9802284237840944},{"timestamp":"2021-10-02T00:00:00Z","value":0.976753243730052},{"timestamp":"2021-10-03T00:00:00Z","value":0.9764127056057769},{"timestamp":"2021-10-04T00:00:00Z","value":0.973740990243866},{"timestamp":"2021-10-05T00:00:00Z","value":0.9742229500499541},{"timestamp":"2021-10-06T00:00:00Z","value":0.9756134001687335},{"timestamp":"2021-10-07T00:00:00Z","value":0.9747427654065849},{"timestamp":"2021-10-08T00:00:00Z","value":0.9741433918297701},{"timestamp":"2021-10-09T00:00:00Z","value":0.9716815984786841},{"timestamp":"2021-10-10T00:00:00Z","value":0.9687453536891504},{"timestamp":"2021-10-11T00:00:00Z","value":0.9729850988295106},{"timestamp":"2021-10-12T00:00:00Z","value":0.9736862978660851},{"timestamp":"2021-10-13T00:00:00Z","value":0.9757201526997891},{"timestamp":"2021-10-14T00:00:00Z","value":0.9764136134129336},{"timestamp":"2021-10-15T00:00:00Z","value":0.9752048474019921},{"timestamp":"2021-10-16T00:00:00Z","value":0.9694904323786341},{"timestamp":"2021-10-17T00:00:00Z","value":0.9661898148165371},{"timestamp":"2021-10-18T00:00:00Z","value":0.9735246928687616},{"timestamp":"2021-10-19T00:00:00Z","value":0.971867851116305},{"timestamp":"2021-10-20T00:00:00Z","value":0.9740651049043331},{"timestamp":"2021-10-21T00:00:00Z","value":0.9465614109934096},{"timestamp":"2021-10-22T00:00:00Z","value":0.9360677958161591},{"timestamp":"2021-10-23T00:00:00Z","value":0.930935515613098},{"timestamp":"2021-10-24T00:00:00Z","value":0.914690122118312},{"timestamp":"2021-10-25T00:00:00Z","value":0.9383310980663321},{"timestamp":"2021-10-26T00:00:00Z","value":0.9386255147224905}],"granularity":"daily","customInterval":1,"sensitivity":100,"maxAnomalyRatio":0.15},"response":{"isAnomaly":true,"expectedValue":0.9470176133851806},"type":"last"} -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case17_0.json: -------------------------------------------------------------------------------- 1 | {"request":{"boundaryVersion":"V3","series":[{"timestamp":"2021-09-28T00:00:00Z","value":0.620968559737307},{"timestamp":"2021-09-29T00:00:00Z","value":0.613117434726291},{"timestamp":"2021-09-30T00:00:00Z","value":0.6121646464479885},{"timestamp":"2021-10-01T00:00:00Z","value":0.610501669267898},{"timestamp":"2021-10-02T00:00:00Z","value":0.6049507957340575},{"timestamp":"2021-10-03T00:00:00Z","value":0.5863605799708445},{"timestamp":"2021-10-04T00:00:00Z","value":0.6201456633636735},{"timestamp":"2021-10-05T00:00:00Z","value":0.6022374943291321},{"timestamp":"2021-10-06T00:00:00Z","value":0.5916215789278225},{"timestamp":"2021-10-07T00:00:00Z","value":0.600197266038055},{"timestamp":"2021-10-08T00:00:00Z","value":0.6065954403907521},{"timestamp":"2021-10-09T00:00:00Z","value":0.6266568642832175},{"timestamp":"2021-10-10T00:00:00Z","value":0.6015331287026765},{"timestamp":"2021-10-11T00:00:00Z","value":0.6070306581162951},{"timestamp":"2021-10-12T00:00:00Z","value":0.608429755113444},{"timestamp":"2021-10-13T00:00:00Z","value":0.5908665774234465},{"timestamp":"2021-10-14T00:00:00Z","value":0.5805518060598345},{"timestamp":"2021-10-15T00:00:00Z","value":0.5853119306100105},{"timestamp":"2021-10-16T00:00:00Z","value":0.5816932956376606},{"timestamp":"2021-10-17T00:00:00Z","value":0.5810927945004549},{"timestamp":"2021-10-18T00:00:00Z","value":0.5918606231985455},{"timestamp":"2021-10-19T00:00:00Z","value":0.6050963262011164},{"timestamp":"2021-10-20T00:00:00Z","value":0.5946676803978046},{"timestamp":"2021-10-21T00:00:00Z","value":0.7210365644044701},{"timestamp":"2021-10-22T00:00:00Z","value":0.7526715969792679},{"timestamp":"2021-10-23T00:00:00Z","value":0.7421365785245415},{"timestamp":"2021-10-24T00:00:00Z","value":0.8017941094613165},{"timestamp":"2021-10-25T00:00:00Z","value":0.7599276009247564},{"timestamp":"2021-10-26T00:00:00Z","value":0.7572599562700529}],"granularity":"daily","customInterval":1,"sensitivity":100,"maxAnomalyRatio":0.15},"response":{"isAnomaly":true,"expectedValue":0.6370890026914026},"type":"last"} -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case18_0.json: -------------------------------------------------------------------------------- 1 | {"request":{"boundaryVersion":"V3","series":[{"timestamp":"2021-09-28T00:00:00Z","value":0.43195802946842893},{"timestamp":"2021-09-29T00:00:00Z","value":0.437816233602729},{"timestamp":"2021-09-30T00:00:00Z","value":0.437744779051428},{"timestamp":"2021-10-01T00:00:00Z","value":0.442340562796227},{"timestamp":"2021-10-02T00:00:00Z","value":0.4515530179417255},{"timestamp":"2021-10-03T00:00:00Z","value":0.4746069416233985},{"timestamp":"2021-10-04T00:00:00Z","value":0.426536672044024},{"timestamp":"2021-10-05T00:00:00Z","value":0.435976773507495},{"timestamp":"2021-10-06T00:00:00Z","value":0.44622878622548795},{"timestamp":"2021-10-07T00:00:00Z","value":0.43596019467327707},{"timestamp":"2021-10-08T00:00:00Z","value":0.42776392504766},{"timestamp":"2021-10-09T00:00:00Z","value":0.405591420498323},{"timestamp":"2021-10-10T00:00:00Z","value":0.43692528406175896},{"timestamp":"2021-10-11T00:00:00Z","value":0.4273399738796515},{"timestamp":"2021-10-12T00:00:00Z","value":0.4251113114225589},{"timestamp":"2021-10-13T00:00:00Z","value":0.44691382817403},{"timestamp":"2021-10-14T00:00:00Z","value":0.456976370235797},{"timestamp":"2021-10-15T00:00:00Z","value":0.4511143013810125},{"timestamp":"2021-10-16T00:00:00Z","value":0.459389234883034},{"timestamp":"2021-10-17T00:00:00Z","value":0.4589358105048915},{"timestamp":"2021-10-18T00:00:00Z","value":0.4443352033087665},{"timestamp":"2021-10-19T00:00:00Z","value":0.42846421097406395},{"timestamp":"2021-10-20T00:00:00Z","value":0.44089606898917794},{"timestamp":"2021-10-21T00:00:00Z","value":0.32425843242688107},{"timestamp":"2021-10-22T00:00:00Z","value":0.2914673632847505},{"timestamp":"2021-10-23T00:00:00Z","value":0.3032958314255045},{"timestamp":"2021-10-24T00:00:00Z","value":0.2382732127656725},{"timestamp":"2021-10-25T00:00:00Z","value":0.2801725365330385},{"timestamp":"2021-10-26T00:00:00Z","value":0.2856556771700195}],"granularity":"daily","customInterval":1,"sensitivity":100,"maxAnomalyRatio":0.15},"response":{"isAnomaly":true,"expectedValue":0.31390199252365714},"type":"last"} -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case20_0.json: -------------------------------------------------------------------------------- 1 | {"request":{"boundaryVersion":"V3","series":[{"timestamp":"2021-09-28T00:00:00Z","value":0.9800753512864412},{"timestamp":"2021-09-29T00:00:00Z","value":0.9801664153295204},{"timestamp":"2021-09-30T00:00:00Z","value":0.9799775507179591},{"timestamp":"2021-10-01T00:00:00Z","value":0.9802284237840944},{"timestamp":"2021-10-02T00:00:00Z","value":0.976753243730052},{"timestamp":"2021-10-03T00:00:00Z","value":0.9764127056057769},{"timestamp":"2021-10-04T00:00:00Z","value":0.973740990243866},{"timestamp":"2021-10-05T00:00:00Z","value":0.9742229500499541},{"timestamp":"2021-10-06T00:00:00Z","value":0.9756134001687335},{"timestamp":"2021-10-07T00:00:00Z","value":0.9747427654065849},{"timestamp":"2021-10-08T00:00:00Z","value":0.9741433918297701},{"timestamp":"2021-10-09T00:00:00Z","value":0.9716815984786841},{"timestamp":"2021-10-10T00:00:00Z","value":0.9687453536891504},{"timestamp":"2021-10-11T00:00:00Z","value":0.9729850988295106},{"timestamp":"2021-10-12T00:00:00Z","value":0.9736862978660851},{"timestamp":"2021-10-13T00:00:00Z","value":0.9757201526997891},{"timestamp":"2021-10-14T00:00:00Z","value":0.9764136134129336},{"timestamp":"2021-10-15T00:00:00Z","value":0.9752048474019921},{"timestamp":"2021-10-16T00:00:00Z","value":0.9694904323786341},{"timestamp":"2021-10-17T00:00:00Z","value":0.9661898148165371},{"timestamp":"2021-10-18T00:00:00Z","value":0.9735246928687616},{"timestamp":"2021-10-19T00:00:00Z","value":0.971867851116305},{"timestamp":"2021-10-20T00:00:00Z","value":0.9740651049043331},{"timestamp":"2021-10-21T00:00:00Z","value":0.9465614109934096},{"timestamp":"2021-10-22T00:00:00Z","value":0.9360677958161591},{"timestamp":"2021-10-23T00:00:00Z","value":0.930935515613098},{"timestamp":"2021-10-24T00:00:00Z","value":0.914690122118312},{"timestamp":"2021-10-25T00:00:00Z","value":0.9383310980663321},{"timestamp":"2021-10-26T00:00:00Z","value":0.9386255147224905}],"granularity":"daily","customInterval":1,"sensitivity":100,"maxAnomalyRatio":0.15},"response":{"isAnomaly":true,"expectedValue":0.9470176133851806},"type":"last"} -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case22_0.json: -------------------------------------------------------------------------------- 1 | {"request":{"boundaryVersion":"V3","series":[{"timestamp":"2021-09-28T00:00:00Z","value":0.727889762805614},{"timestamp":"2021-09-29T00:00:00Z","value":0.7278689328301715},{"timestamp":"2021-09-30T00:00:00Z","value":0.7313505588385341},{"timestamp":"2021-10-01T00:00:00Z","value":0.7302217175882455},{"timestamp":"2021-10-02T00:00:00Z","value":0.6824460001366225},{"timestamp":"2021-10-03T00:00:00Z","value":0.6712920784482079},{"timestamp":"2021-10-04T00:00:00Z","value":0.7233669897409705},{"timestamp":"2021-10-05T00:00:00Z","value":0.727400200394371},{"timestamp":"2021-10-06T00:00:00Z","value":0.7294915553638315},{"timestamp":"2021-10-07T00:00:00Z","value":0.7298403111530479},{"timestamp":"2021-10-08T00:00:00Z","value":0.7286188185298306},{"timestamp":"2021-10-09T00:00:00Z","value":0.6880191421007655},{"timestamp":"2021-10-10T00:00:00Z","value":0.6728103630531139},{"timestamp":"2021-10-11T00:00:00Z","value":0.7272929420379565},{"timestamp":"2021-10-12T00:00:00Z","value":0.7270107092643191},{"timestamp":"2021-10-13T00:00:00Z","value":0.726129961973518},{"timestamp":"2021-10-14T00:00:00Z","value":0.7288434571700559},{"timestamp":"2021-10-15T00:00:00Z","value":0.7248771672666495},{"timestamp":"2021-10-16T00:00:00Z","value":0.6745186831331185},{"timestamp":"2021-10-17T00:00:00Z","value":0.6712113743378905},{"timestamp":"2021-10-18T00:00:00Z","value":0.7286986714320959},{"timestamp":"2021-10-19T00:00:00Z","value":0.7297117908915329},{"timestamp":"2021-10-20T00:00:00Z","value":0.7322569822862486},{"timestamp":"2021-10-21T00:00:00Z","value":0.4489616014708615},{"timestamp":"2021-10-22T00:00:00Z","value":0.367199740547456},{"timestamp":"2021-10-23T00:00:00Z","value":0.3591673378745245},{"timestamp":"2021-10-24T00:00:00Z","value":0.196433886061607},{"timestamp":"2021-10-25T00:00:00Z","value":0.3618543722822455},{"timestamp":"2021-10-26T00:00:00Z","value":0.3685714623163745}],"granularity":"daily","customInterval":1,"sensitivity":100,"maxAnomalyRatio":0.15},"response":{"isAnomaly":true,"expectedValue":0.7332318727210403},"type":"last"} -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case15_0.json: -------------------------------------------------------------------------------- 1 | {"request":{"boundaryVersion":"V3","series":[{"timestamp":"2021-09-28T00:00:00Z","value":0.7726658035667016},{"timestamp":"2021-09-29T00:00:00Z","value":0.7725834876861519},{"timestamp":"2021-09-30T00:00:00Z","value":0.7737944293805915},{"timestamp":"2021-10-01T00:00:00Z","value":0.7719476278975435},{"timestamp":"2021-10-02T00:00:00Z","value":0.7199824900306171},{"timestamp":"2021-10-03T00:00:00Z","value":0.7270034459068729},{"timestamp":"2021-10-04T00:00:00Z","value":0.7555488627879754},{"timestamp":"2021-10-05T00:00:00Z","value":0.7838747565238935},{"timestamp":"2021-10-06T00:00:00Z","value":0.7867461138043821},{"timestamp":"2021-10-07T00:00:00Z","value":0.7843751494581235},{"timestamp":"2021-10-08T00:00:00Z","value":0.7846630123708445},{"timestamp":"2021-10-09T00:00:00Z","value":0.743634564288573},{"timestamp":"2021-10-10T00:00:00Z","value":0.7390248817864655},{"timestamp":"2021-10-11T00:00:00Z","value":0.7901259720631815},{"timestamp":"2021-10-12T00:00:00Z","value":0.7827474288720441},{"timestamp":"2021-10-13T00:00:00Z","value":0.7817293505019984},{"timestamp":"2021-10-14T00:00:00Z","value":0.787506023683206},{"timestamp":"2021-10-15T00:00:00Z","value":0.7890152997878905},{"timestamp":"2021-10-16T00:00:00Z","value":0.7430159805885905},{"timestamp":"2021-10-17T00:00:00Z","value":0.7460383089591645},{"timestamp":"2021-10-18T00:00:00Z","value":0.7890079806593765},{"timestamp":"2021-10-19T00:00:00Z","value":0.7858479932094545},{"timestamp":"2021-10-20T00:00:00Z","value":0.7871560324950155},{"timestamp":"2021-10-21T00:00:00Z","value":0.4877803223004515},{"timestamp":"2021-10-22T00:00:00Z","value":0.40494416404746597},{"timestamp":"2021-10-23T00:00:00Z","value":0.4017907633264285},{"timestamp":"2021-10-24T00:00:00Z","value":0.2365925334950035},{"timestamp":"2021-10-25T00:00:00Z","value":0.4006865143855665},{"timestamp":"2021-10-26T00:00:00Z","value":0.4061264112883175}],"granularity":"daily","customInterval":1,"sensitivity":100,"maxAnomalyRatio":0.15},"response":{"isAnomaly":true,"expectedValue":0.7869841085145047},"type":"last"} -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case19_0.json: -------------------------------------------------------------------------------- 1 | {"request":{"boundaryVersion":"V3","series":[{"timestamp":"2021-09-28T00:00:00Z","value":0.36447689853345},{"timestamp":"2021-09-29T00:00:00Z","value":0.3701632340344815},{"timestamp":"2021-09-30T00:00:00Z","value":0.3699233798812815},{"timestamp":"2021-10-01T00:00:00Z","value":0.37312817197322207},{"timestamp":"2021-10-02T00:00:00Z","value":0.3714053158534085},{"timestamp":"2021-10-03T00:00:00Z","value":0.3922931885896795},{"timestamp":"2021-10-04T00:00:00Z","value":0.366432179522393},{"timestamp":"2021-10-05T00:00:00Z","value":0.38516709676461897},{"timestamp":"2021-10-06T00:00:00Z","value":0.39586796982186345},{"timestamp":"2021-10-07T00:00:00Z","value":0.38696716476364595},{"timestamp":"2021-10-08T00:00:00Z","value":0.3816770869866295},{"timestamp":"2021-10-09T00:00:00Z","value":0.35712411572331004},{"timestamp":"2021-10-10T00:00:00Z","value":0.3832546898391935},{"timestamp":"2021-10-11T00:00:00Z","value":0.3813375827006715},{"timestamp":"2021-10-12T00:00:00Z","value":0.379586591015611},{"timestamp":"2021-10-13T00:00:00Z","value":0.3975383837827435},{"timestamp":"2021-10-14T00:00:00Z","value":0.4078828942400825},{"timestamp":"2021-10-15T00:00:00Z","value":0.4034911884930285},{"timestamp":"2021-10-16T00:00:00Z","value":0.404224353927564},{"timestamp":"2021-10-17T00:00:00Z","value":0.405565089406058},{"timestamp":"2021-10-18T00:00:00Z","value":0.3955075984148745},{"timestamp":"2021-10-19T00:00:00Z","value":0.3823480207691225},{"timestamp":"2021-10-20T00:00:00Z","value":0.39285202500392896},{"timestamp":"2021-10-21T00:00:00Z","value":0.2678511412803895},{"timestamp":"2021-10-22T00:00:00Z","value":0.23612960552393697},{"timestamp":"2021-10-23T00:00:00Z","value":0.24347262378489104},{"timestamp":"2021-10-24T00:00:00Z","value":0.18702162111220103},{"timestamp":"2021-10-25T00:00:00Z","value":0.2292806605933465},{"timestamp":"2021-10-26T00:00:00Z","value":0.231611787397198}],"granularity":"daily","customInterval":1,"sensitivity":100,"maxAnomalyRatio":0.15},"response":{"isAnomaly":true,"expectedValue":0.25703186393329375},"type":"last"} -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_entire_request_6.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1962-01-01T00:00:00.001Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.002Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.003Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.004Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.005Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.006Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.007Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.008Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.009Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.010Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.011Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.012Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.013Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.014Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.015Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.016Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.017Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.018Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.019Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.020Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.021Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.022Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.023Z", "value": 1}, {"timestamp": "1962-01-01T00:00:00.024Z", "value": 0}], "granularity": "microsecond", "maxAnomalyRatio": 0.25, "sensitivity": 95}, "response": {"isAnomaly": [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true], "expectedValues": [0.9999999999999999, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999, 0.9999999999999998, 0.9999999999999998, 0.9999999999999998, 0.9999999999999998, 1.0, 0.9999999999999999, 0.9999999999999998, 1.0, 0.9999999999999999, 1.0, 1.0, 0.9999999999999999, 0.9999999999999998, 0.9999999999999998, 0.9999999999999998, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999]}, "type": "entire"} -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case7_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2020-01-30 00:00:00", "value": 5585229670.0}, {"timestamp": "2020-01-31 00:00:00", "value": 5388507098.214286}, {"timestamp": "2020-02-01 00:00:00", "value": 1673865163.4285715}, {"timestamp": "2020-02-02 00:00:00", "value": 1690437981.642857}, {"timestamp": "2020-02-03 00:00:00", "value": 6469524444.857143}, {"timestamp": "2020-02-04 00:00:00", "value": 6701582372.071428}, {"timestamp": "2020-02-05 00:00:00", "value": 6559520768.285714}, {"timestamp": "2020-02-06 00:00:00", "value": 6519967856.5}, {"timestamp": "2020-02-07 00:00:00", "value": 5966290286.714285}, {"timestamp": "2020-02-08 00:00:00", "value": 2240802463.928571}, {"timestamp": "2020-02-09 00:00:00", "value": 2338595013.142857}, {"timestamp": "2020-02-10 00:00:00", "value": 7254663314.357142}, {"timestamp": "2020-02-11 00:00:00", "value": 7292218037.571428}, {"timestamp": "2020-02-12 00:00:00", "value": 7365607623.785715}, {"timestamp": "2020-02-13 00:00:00", "value": 7259618411.0}, {"timestamp": "2020-02-14 00:00:00", "value": 6515132464.214286}, {"timestamp": "2020-02-15 00:00:00", "value": 2894387189.428571}, {"timestamp": "2020-02-16 00:00:00", "value": 2944670977.642857}, {"timestamp": "2020-02-17 00:00:00", "value": 7332227803.857143}, {"timestamp": "2020-02-18 00:00:00", "value": 8023587988.071428}, {"timestamp": "2020-02-19 00:00:00", "value": 7886047919.285714}, {"timestamp": "2020-02-20 00:00:00", "value": 7829153095.5}, {"timestamp": "2020-02-21 00:00:00", "value": 7085420207.714286}, {"timestamp": "2020-02-22 00:00:00", "value": 3469039756.9285717}, {"timestamp": "2020-02-23 00:00:00", "value": 3529327472.142857}, {"timestamp": "2020-02-24 00:00:00", "value": 8089375460.357142}, {"timestamp": "2020-02-25 00:00:00", "value": 8204421366.571428}, {"timestamp": "2020-02-26 00:00:00", "value": 8497917030.785714}, {"timestamp": "2020-02-27 00:00:00", "value": 8637697321.0}], "granularity": "daily", "customInterval": 1, "sensitivity": 99, "maxAnomalyRatio": 0.15, "needDetectorId": true, "period": 7}, "response": {"expectedValue": 8637697321.0}, "type": "last"} -------------------------------------------------------------------------------- /tests/cases/json_profiling_cases.case_3.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2019-05-20T00:00:00Z", "value": 5.0}, {"timestamp": "2019-05-21T00:00:00Z", "value": 3.0}, {"timestamp": "2019-05-22T00:00:00Z", "value": 4.0}, {"timestamp": "2019-05-23T00:00:00Z", "value": 2.0}, {"timestamp": "2019-05-24T00:00:00Z", "value": 2.0}, {"timestamp": "2019-05-25T00:00:00Z", "value": 2.0}, {"timestamp": "2019-05-26T00:00:00Z", "value": 2.0}, {"timestamp": "2019-05-27T00:00:00Z", "value": 2.0}, {"timestamp": "2019-05-28T00:00:00Z", "value": 2.0}, {"timestamp": "2019-05-29T00:00:00Z", "value": 2.0}, {"timestamp": "2019-05-30T00:00:00Z", "value": 2.0}, {"timestamp": "2019-05-31T00:00:00Z", "value": 2.0}, {"timestamp": "2019-06-01T00:00:00Z", "value": 2.0}, {"timestamp": "2019-06-02T00:00:00Z", "value": 2.0}, {"timestamp": "2019-06-03T00:00:00Z", "value": 2.0}, {"timestamp": "2019-06-04T00:00:00Z", "value": 2.0}, {"timestamp": "2019-06-05T00:00:00Z", "value": 2.0}, {"timestamp": "2019-06-06T00:00:00Z", "value": 2.0}, {"timestamp": "2019-06-07T00:00:00Z", "value": 2.0}, {"timestamp": "2019-06-08T00:00:00Z", "value": 2.0}, {"timestamp": "2019-06-09T00:00:00Z", "value": 2.0}, {"timestamp": "2019-06-10T00:00:00Z", "value": 2.0}, {"timestamp": "2019-06-11T00:00:00Z", "value": 2.0}, {"timestamp": "2019-06-12T00:00:00Z", "value": 2.0}, {"timestamp": "2019-06-13T00:00:00Z", "value": 2.0}, {"timestamp": "2019-06-14T00:00:00Z", "value": 2.0}, {"timestamp": "2019-06-15T00:00:00Z", "value": 2.0}, {"timestamp": "2019-06-16T00:00:00Z", "value": 2.0}, {"timestamp": "2019-06-17T00:00:00Z", "value": 1.0}], "granularity": "daily", "maxAnomalyRatio": 0.25}, "response": {"expectedValues": [1.9999999999999984, 1.9999999999999987, 1.999999999999999, 1.999999999999999, 1.9999999999999991, 1.9999999999999993, 1.9999999999999996, 1.9999999999999998, 2.0, 2.0000000000000004, 2.0000000000000004, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.0000000000000013, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.0000000000000004, 2.0000000000000004, 2.0000000000000004, 2.0000000000000004, 2.0000000000000004]}, "type": "entire"} -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_extreme_data_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1952-01-01 00:00:00", "value": -9e+99}, {"timestamp": "1952-01-02 00:00:00", "value": -7e+99}, {"timestamp": "1952-01-03 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-04 00:00:00", "value": 6e+99}, {"timestamp": "1952-01-05 00:00:00", "value": 6e+99}, {"timestamp": "1952-01-06 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-07 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-08 00:00:00", "value": -9e+99}, {"timestamp": "1952-01-09 00:00:00", "value": -7e+99}, {"timestamp": "1952-01-10 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-11 00:00:00", "value": 6e+99}, {"timestamp": "1952-01-12 00:00:00", "value": 6e+99}, {"timestamp": "1952-01-13 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-14 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-15 00:00:00", "value": -9e+99}, {"timestamp": "1952-01-16 00:00:00", "value": -7e+99}, {"timestamp": "1952-01-17 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-18 00:00:00", "value": 6e+99}, {"timestamp": "1952-01-19 00:00:00", "value": 6e+99}, {"timestamp": "1952-01-20 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-21 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-22 00:00:00", "value": -9e+99}, {"timestamp": "1952-01-23 00:00:00", "value": -7e+99}, {"timestamp": "1952-01-24 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-25 00:00:00", "value": 6e+99}, {"timestamp": "1952-01-26 00:00:00", "value": 6e+99}, {"timestamp": "1952-01-27 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-28 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-29 00:00:00", "value": -9e+99}, {"timestamp": "1952-01-30 00:00:00", "value": -7e+99}], "granularity": "daily", "maxAnomalyRatio": 0.25, "sensitivity": 95, "period": 7}, "response": {"expectedValues": [-8.999999999999998e+99, -7e+99, 5.000000000000001e+99, 6.000000000000001e+99, 6.000000000000001e+99, 5e+99, 4.999999999999999e+99, -9e+99, -7.0000000000000005e+99, 5.000000000000001e+99, 6.000000000000001e+99, 6e+99, 5e+99, 5e+99, -9e+99, -7.0000000000000005e+99, 5e+99, 6e+99, 6e+99, 5.000000000000001e+99, 5.000000000000001e+99, -8.999999999999998e+99, -7.0000000000000005e+99, 5e+99, 6e+99, 5.999999999999999e+99, 4.999999999999998e+99, 4.999999999999998e+99, -9e+99, -7.000000000000002e+99]}, "type": "entire"} -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_extreme_data_2.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1952-01-01 00:00:00", "value": -9e+99}, {"timestamp": "1952-01-02 00:00:00", "value": -7e+99}, {"timestamp": "1952-01-03 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-04 00:00:00", "value": 6e+99}, {"timestamp": "1952-01-05 00:00:00", "value": 6e+99}, {"timestamp": "1952-01-06 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-07 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-08 00:00:00", "value": -9e+99}, {"timestamp": "1952-01-09 00:00:00", "value": -7e+99}, {"timestamp": "1952-01-10 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-11 00:00:00", "value": 6e+99}, {"timestamp": "1952-01-12 00:00:00", "value": 6e+99}, {"timestamp": "1952-01-13 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-14 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-15 00:00:00", "value": -9e+99}, {"timestamp": "1952-01-16 00:00:00", "value": -7e+99}, {"timestamp": "1952-01-17 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-18 00:00:00", "value": 6e+99}, {"timestamp": "1952-01-19 00:00:00", "value": 6e+99}, {"timestamp": "1952-01-20 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-21 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-22 00:00:00", "value": -9e+99}, {"timestamp": "1952-01-23 00:00:00", "value": -7e+99}, {"timestamp": "1952-01-24 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-25 00:00:00", "value": 6e+99}, {"timestamp": "1952-01-26 00:00:00", "value": 6e+99}, {"timestamp": "1952-01-27 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-28 00:00:00", "value": 5e+99}, {"timestamp": "1952-01-29 00:00:00", "value": -9e+99}, {"timestamp": "1952-01-30 00:00:00", "value": -7e+99}], "granularity": "daily", "maxAnomalyRatio": 0.25, "sensitivity": 95, "period": 7}, "response": {"expectedValues": [-8.999999999999998e+99, -7e+99, 5.000000000000001e+99, 6.000000000000001e+99, 6.000000000000001e+99, 5e+99, 4.999999999999999e+99, -9e+99, -7.0000000000000005e+99, 5.000000000000001e+99, 6.000000000000001e+99, 6e+99, 5e+99, 5e+99, -9e+99, -7.0000000000000005e+99, 5e+99, 6e+99, 6e+99, 5.000000000000001e+99, 5.000000000000001e+99, -8.999999999999998e+99, -7.0000000000000005e+99, 5e+99, 6e+99, 5.999999999999999e+99, 4.999999999999998e+99, 4.999999999999998e+99, -9e+99, -7.000000000000002e+99]}, "type": "entire"} -------------------------------------------------------------------------------- /tests/e2e_test_mvad.py: -------------------------------------------------------------------------------- 1 | 2 | # --------------------------------------------------------- 3 | # Copyright (c) Microsoft Corporation. All rights reserved. 4 | # --------------------------------------------------------- 5 | 6 | import mlflow 7 | import numpy as np 8 | import pandas as pd 9 | from anomaly_detector import MultivariateAnomalyDetector 10 | 11 | import json 12 | from pprint import pprint 13 | 14 | mlflow.set_tracking_uri(uri="http://127.0.0.1:8080") 15 | 16 | 17 | def main(): 18 | mlflow.set_experiment("MLflow Quickstart") 19 | training_data = np.random.randn(10000, 20) 20 | columns = [f"variable_{i}" for i in range(20)] 21 | training_data = pd.DataFrame(training_data, columns=columns) 22 | timestamps = pd.date_range(start="2023-01-03", periods=10000, freq="H") 23 | training_data["timestamp"] = timestamps.strftime("%Y-%m-%dT%H:%M:%SZ") 24 | 25 | training_data = training_data.set_index("timestamp", drop=True) 26 | params = {"sliding_window": 200, "device": "cpu", "abcd": 10} 27 | with mlflow.start_run(): 28 | mlflow.log_params(params) 29 | mlflow.set_tag("Training Info", "Basic multivariate anomaly detector") 30 | 31 | 32 | model = MultivariateAnomalyDetector() 33 | 34 | model.fit(training_data, params=params) 35 | 36 | model_info = mlflow.pyfunc.log_model( 37 | python_model=model, 38 | artifact_path="mvad_artifacts", 39 | registered_model_name="tracking-quickstart", 40 | ) 41 | print(model_info.model_uri) 42 | loaded_model = mlflow.pyfunc.load_model(model_info.model_uri) 43 | 44 | eval_data = np.random.randn(201, 20) 45 | eval_data[-1, :] += 100 46 | eval_data = pd.DataFrame(eval_data, columns=columns) 47 | timestamps = pd.date_range(start="2023-01-03", periods=201, freq="H") 48 | eval_data["timestamp"] = timestamps.strftime("%Y-%m-%dT%H:%M:%SZ") 49 | 50 | eval_data = eval_data.set_index("timestamp", drop=True) 51 | results = loaded_model.predict(data=eval_data) 52 | pprint(results) 53 | with open("result1.json", "w") as f: 54 | json.dump(results, f) 55 | 56 | eval_data = np.random.randn(201, 20) 57 | eval_data[-1, :] += 100 58 | eval_data = pd.DataFrame(eval_data, columns=columns) 59 | results = loaded_model.predict(data=eval_data) 60 | pprint(results) 61 | with open("result2.json", "w") as f: 62 | json.dump(results, f) 63 | 64 | 65 | if __name__ == "__main__": 66 | main() 67 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/detectors/esd_filter.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from anomaly_detector.univariate.detectors.detector import AnomalyDetector 6 | from anomaly_detector.univariate.util import Direction, get_critical, EPS 7 | from anomaly_detector.univariate._anomaly_kernel_cython import generalized_esd_test 8 | 9 | 10 | class ESD(AnomalyDetector): 11 | def __init__(self, series, max_outliers, majority_value, alpha): 12 | self.__series__ = series 13 | self.__max_outliers = max_outliers 14 | self.__alpha = alpha 15 | self.__majority_value = majority_value 16 | self.__critical_values__ = get_critical(self.__alpha, len(series), max_outliers) \ 17 | if majority_value is None else None 18 | 19 | def detect(self, direction, last_value=None): 20 | last_index = -1 21 | if last_value is not None: 22 | last_index = max(self.__series__.index) 23 | 24 | detect_data = self.__series__ 25 | if direction == Direction.upper_tail: 26 | detect_data = self.__series__.iloc[::-1] 27 | if self.__majority_value is not None: 28 | detect_data = detect_data[:next(i for i in reversed(range(len(detect_data))) if 29 | abs(detect_data.iloc[i] - self.__majority_value) < EPS) + 1] 30 | 31 | if last_index != -1: 32 | if last_index not in detect_data.index: 33 | return [] 34 | else: 35 | last_index = detect_data.index.get_loc(last_index) 36 | 37 | critical_values = get_critical(self.__alpha, len(detect_data), self.__max_outliers) \ 38 | if self.__critical_values__ is None else self.__critical_values__ 39 | 40 | selected_anomaly = generalized_esd_test(detect_data.tolist(), detect_data.index.tolist(), 41 | self.__max_outliers, 42 | critical_values, 43 | True if direction == Direction.upper_tail else False, 44 | last_index 45 | ) 46 | 47 | if selected_anomaly is None or len(selected_anomaly) == 0: 48 | return [] 49 | return selected_anomaly 50 | -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case26_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2022-10-10T00:00:00Z", "value": 0}, {"timestamp": "2022-10-11T00:00:00Z", "value": 0}, {"timestamp": "2022-10-12T00:00:00Z", "value": 0}, {"timestamp": "2022-10-13T00:00:00Z", "value": 0}, {"timestamp": "2022-10-14T00:00:00Z", "value": 0}, {"timestamp": "2022-10-15T00:00:00Z", "value": 0}, {"timestamp": "2022-10-16T00:00:00Z", "value": 0}, {"timestamp": "2022-10-17T00:00:00Z", "value": 0}, {"timestamp": "2022-10-18T00:00:00Z", "value": 0}, {"timestamp": "2022-10-19T00:00:00Z", "value": 0}, {"timestamp": "2022-10-20T00:00:00Z", "value": 0}, {"timestamp": "2022-10-21T00:00:00Z", "value": 0}, {"timestamp": "2022-10-22T00:00:00Z", "value": 0}, {"timestamp": "2022-10-23T00:00:00Z", "value": 0}, {"timestamp": "2022-10-24T00:00:00Z", "value": 0}, {"timestamp": "2022-10-25T00:00:00Z", "value": 6072}, {"timestamp": "2022-10-26T00:00:00Z", "value": 5928}, {"timestamp": "2022-10-27T00:00:00Z", "value": 7197}, {"timestamp": "2022-10-28T00:00:00Z", "value": 7202}, {"timestamp": "2022-10-29T00:00:00Z", "value": 7202}, {"timestamp": "2022-10-30T00:00:00Z", "value": 7199}, {"timestamp": "2022-10-31T00:00:00Z", "value": 7199}, {"timestamp": "2022-11-01T00:00:00Z", "value": 7200}, {"timestamp": "2022-11-02T00:00:00Z", "value": 7203}, {"timestamp": "2022-11-03T00:00:00Z", "value": 7198}, {"timestamp": "2022-11-04T00:00:00Z", "value": 7201}, {"timestamp": "2022-11-05T00:00:00Z", "value": 7200}, {"timestamp": "2022-11-06T00:00:00Z", "value": 7199}, {"timestamp": "2022-11-07T00:00:00Z", "value": 7201}, {"timestamp": "2022-11-08T00:00:00Z", "value": 7200}, {"timestamp": "2022-11-09T00:00:00Z", "value": 7200}], "granularity": "daily", "needDetectorId": true, "boundaryVersion": "V3"}, "response": {"expectedValues": [-164.95247431691698, -119.84059616505664, -74.72871801319631, -29.61683986133599, 73.67575994421279, 149.8435300314922, 140.7057487468137, 12.697451450064674, -169.2832984445664, -315.72268050931564, -337.1068743164186, -143.92205943811143, 353.3455845533699, 1208.2627096214765, 2330.6072938724087, 3594.2101469480504, 4872.902078490291, 6040.513898141015, 6970.8764155421095, 7569.131592906123, 7865.6660027282705, 7922.177370074418, 7800.363420010443, 7561.921877602218, 7268.5504679156165, 6981.94691601651, 6763.808946970775, 6675.834285844283, 6714.639003743286, 6753.443721642288, 6792.24843954129], "isAnomaly": [true, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, false, false, true, true, false, false, false, true, false, true, true, false, true, true, true]}, "type": "entire"} -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case27_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2022-10-10T00:00:00Z", "value": 5.0}, {"timestamp": "2022-10-11T00:00:00Z", "value": 5.0}, {"timestamp": "2022-10-12T00:00:00Z", "value": 5.0}, {"timestamp": "2022-10-13T00:00:00Z", "value": 5.0}, {"timestamp": "2022-10-14T00:00:00Z", "value": 5.0}, {"timestamp": "2022-10-15T00:00:00Z", "value": 0.0}, {"timestamp": "2022-10-16T00:00:00Z", "value": 0.0}, {"timestamp": "2022-10-17T00:00:00Z", "value": 5.0}, {"timestamp": "2022-10-18T00:00:00Z", "value": 5.0}, {"timestamp": "2022-10-19T00:00:00Z", "value": 5.0}, {"timestamp": "2022-10-20T00:00:00Z", "value": 5.0}, {"timestamp": "2022-10-21T00:00:00Z", "value": 5.0}, {"timestamp": "2022-10-22T00:00:00Z", "value": 0.0}, {"timestamp": "2022-10-23T00:00:00Z", "value": 37.0}, {"timestamp": "2022-10-24T00:00:00Z", "value": 5.0}, {"timestamp": "2022-10-25T00:00:00Z", "value": 6077.0}, {"timestamp": "2022-10-26T00:00:00Z", "value": 5933.0}, {"timestamp": "2022-10-27T00:00:00Z", "value": 7202.0}, {"timestamp": "2022-10-28T00:00:00Z", "value": 7207.0}, {"timestamp": "2022-10-29T00:00:00Z", "value": 7202.0}, {"timestamp": "2022-10-30T00:00:00Z", "value": 7199.0}, {"timestamp": "2022-10-31T00:00:00Z", "value": 7204.0}, {"timestamp": "2022-11-01T00:00:00Z", "value": 7205.0}, {"timestamp": "2022-11-02T00:00:00Z", "value": 7208.0}, {"timestamp": "2022-11-03T00:00:00Z", "value": 7203.0}, {"timestamp": "2022-11-04T00:00:00Z", "value": 7206.0}, {"timestamp": "2022-11-05T00:00:00Z", "value": 7200.0}, {"timestamp": "2022-11-06T00:00:00Z", "value": 7199.0}, {"timestamp": "2022-11-07T00:00:00Z", "value": 7206.0}, {"timestamp": "2022-11-08T00:00:00Z", "value": 7205.0}, {"timestamp": "2022-11-09T00:00:00Z", "value": 7205.0}], "granularity": "daily", "needDetectorId": true, "boundaryVersion": "V3"}, "response": {"expectedValues": [-156.625213055791, -112.342166039315, -68.05911902283898, -23.776072006362988, 77.88157625391129, 152.56912953174364, 142.91198658333565, 15.92183388773587, -163.84449118462067, -307.44386354045287, -325.9331580864788, -130.36924972941745, 368.190986624013, 1222.9082498696155, 2343.813534113283, 3605.155407263427, 4881.182437228463, 6046.143191916802, 6974.28623923686, 7571.076126062209, 7866.841315127082, 7923.126248130864, 7801.475366772947, 7563.433112752719, 7270.543927769573, 6984.352253522895, 6766.402531712076, 6678.2392040365075, 6716.513515127953, 6754.787826219397, 6793.062137310842], "isAnomaly": [true, true, true, true, true, true, true, false, false, false, false, false, true, false, false, false, false, false, true, false, false, true, true, true, false, true, false, false, true, true, true]}, "type": "entire"} -------------------------------------------------------------------------------- /src/anomaly_detector/common/data_processor.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from typing import List, Union 6 | 7 | import pandas as pd 8 | from anomaly_detector.common.constants import TIMESTAMP, FillNAMethod 9 | from anomaly_detector.common.exception import * 10 | from anomaly_detector.common.time_util import DT_FORMAT, dt_to_str 11 | 12 | 13 | class MultiADDataProcessor: 14 | def __init__( 15 | self, 16 | *, 17 | fill_na_method: str = FillNAMethod.Linear.name, 18 | fill_na_value: float = 0.0, 19 | ): 20 | if not hasattr(FillNAMethod, fill_na_method): 21 | raise InvalidParameterError( 22 | f"fill_na_method {fill_na_method} is not supported." 23 | ) 24 | if not isinstance(fill_na_value, float): 25 | raise InvalidParameterError(f"fill_na_value must be a float number.") 26 | 27 | self.fill_na_method = FillNAMethod[fill_na_method] 28 | self.fill_na_value = fill_na_value 29 | 30 | def process(self, data: pd.DataFrame): 31 | if not isinstance(data, pd.DataFrame): 32 | raise DataFormatError(f"data must be a pandas.DataFrame") 33 | data = data.sort_index() # sort indices 34 | data = data[sorted(data.columns)] # sort columns 35 | data = self.fill_na(data) 36 | return data 37 | 38 | def fill_na(self, data: pd.DataFrame): 39 | if not isinstance(data, pd.DataFrame): 40 | raise DataFormatError(f"data must be a pandas.DataFrame") 41 | try: 42 | data = data.astype(float) 43 | except Exception as e: 44 | raise DataFormatError(f"Cannot convert values to float. {str(e)}") 45 | if self.fill_na_method == FillNAMethod.Previous: 46 | output_series = data.fillna(method="ffill", limit=len(data)).fillna( 47 | method="bfill", limit=len(data) 48 | ) 49 | elif self.fill_na_method == FillNAMethod.Subsequent: 50 | output_series = data.fillna(method="bfill", limit=len(data)).fillna( 51 | method="ffill", limit=len(data) 52 | ) 53 | elif self.fill_na_method == FillNAMethod.Linear: 54 | output_series = data.interpolate( 55 | method="linear", limit_direction="both", axis=0, limit=len(data) 56 | ) 57 | elif self.fill_na_method == FillNAMethod.Fixed: 58 | output_series = data.fillna(self.fill_na_value) 59 | else: 60 | output_series = data 61 | return output_series.fillna(0) 62 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_extreme_data_1.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1952-01-01 00:00:00", "value": 1.2255063829647773e+29}, {"timestamp": "1952-01-01 00:01:00", "value": 9.923804137914923e+29}, {"timestamp": "1952-01-01 00:02:00", "value": 1.3647134772021297e+29}, {"timestamp": "1952-01-01 00:03:00", "value": 1.1953437539075973e+29}, {"timestamp": "1952-01-01 00:04:00", "value": 8.721955645494106e+28}, {"timestamp": "1952-01-01 00:05:00", "value": 8.174812340453537e+29}, {"timestamp": "1952-01-01 00:06:00", "value": 2.334424672415003e+29}, {"timestamp": "1952-01-01 00:07:00", "value": 5.292379774098937e+29}, {"timestamp": "1952-01-01 00:08:00", "value": 3.036823921122237e+29}, {"timestamp": "1952-01-01 00:09:00", "value": 2.2680065994114598e+29}, {"timestamp": "1952-01-01 00:10:00", "value": 8.881298958324887e+29}, {"timestamp": "1952-01-01 00:11:00", "value": 7.64142045977671e+29}, {"timestamp": "1952-01-01 00:12:00", "value": 9.381774944234387e+29}, {"timestamp": "1952-01-01 00:13:00", "value": 7.969575111017113e+28}, {"timestamp": "1952-01-01 00:14:00", "value": 8.247229778263617e+29}, {"timestamp": "1952-01-01 00:15:00", "value": 5.92049414620417e+28}, {"timestamp": "1952-01-01 00:16:00", "value": 1.1518018625289073e+29}, {"timestamp": "1952-01-01 00:17:00", "value": 5.835105912564154e+28}, {"timestamp": "1952-01-01 00:18:00", "value": 4.471247904688437e+29}, {"timestamp": "1952-01-01 00:19:00", "value": 5.957474525665639e+29}, {"timestamp": "1952-01-01 00:20:00", "value": 9.584770930150502e+29}, {"timestamp": "1952-01-01 00:21:00", "value": 6.924989567343242e+28}, {"timestamp": "1952-01-01 00:22:00", "value": 6.572593972069977e+29}, {"timestamp": "1952-01-01 00:23:00", "value": 4.096365104829516e+29}, {"timestamp": "1952-01-01 00:24:00", "value": 2.999735338659988e+29}, {"timestamp": "1952-01-01 00:25:00", "value": 2.466835876195408e+29}, {"timestamp": "1952-01-01 00:26:00", "value": 2.012290409966855e+29}, {"timestamp": "1952-01-01 00:27:00", "value": 2.751335594117911e+29}, {"timestamp": "1952-01-01 00:28:00", "value": 6.205382228942574e+29}, {"timestamp": "1952-01-01 00:29:00", "value": 4.7497839470469174e+29}], "granularity": "minutely", "maxAnomalyRatio": 0.25, "sensitivity": 95, "period": 0}, "response": {"expectedValues": [1.2255063829647773e+29, 1.2255063829647773e+29, 1.2719087477105614e+29, 1.2527674992598203e+29, 1.1766531123177384e+29, 2.5665143038154904e+29, 2.788297961705536e+29, 3.5738312210848975e+29, 3.942127254527825e+29, 4.221289461500235e+29, 4.362586785074504e+29, 5.423985942546846e+29, 6.2418649765739365e+29, 5.793891694569831e+29, 6.989736330340262e+29, 5.331886421599369e+29, 4.033962702149808e+29, 2.2743098315542142e+29, 3.0091679102715594e+29, 2.5512168597519646e+29, 4.34976116285798e+29, 4.2579005816990634e+29, 5.455717257861775e+29, 5.380740697889992e+29, 4.789192860488862e+29, 3.365605849697844e+29, 3.629564140344348e+29, 2.8653124647539362e+29, 3.287115889576548e+29, 3.637125611253935e+29]}, "type": "entire"} -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_extreme_data_3.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "1952-01-01 00:00:00", "value": 4.8891175062095475e+29}, {"timestamp": "1952-01-01 00:01:00", "value": 2.6599880363364824e+29}, {"timestamp": "1952-01-01 00:02:00", "value": 9.242242335097177e+29}, {"timestamp": "1952-01-01 00:03:00", "value": 4.273879018262541e+29}, {"timestamp": "1952-01-01 00:04:00", "value": 4.941772579095512e+29}, {"timestamp": "1952-01-01 00:05:00", "value": 7.459841260682113e+29}, {"timestamp": "1952-01-01 00:06:00", "value": 1.0562722086351973e+29}, {"timestamp": "1952-01-01 00:07:00", "value": 5.441588994257814e+29}, {"timestamp": "1952-01-01 00:08:00", "value": 8.037703993793022e+28}, {"timestamp": "1952-01-01 00:09:00", "value": 2.4075788996516446e+29}, {"timestamp": "1952-01-01 00:10:00", "value": 7.722171126267556e+29}, {"timestamp": "1952-01-01 00:11:00", "value": 1.1035870998440057e+29}, {"timestamp": "1952-01-01 00:12:00", "value": 1.9201388520377273e+29}, {"timestamp": "1952-01-01 00:13:00", "value": 4.830055913415437e+29}, {"timestamp": "1952-01-01 00:14:00", "value": 2.8982119444067433e+29}, {"timestamp": "1952-01-01 00:15:00", "value": 3.901222221956895e+29}, {"timestamp": "1952-01-01 00:16:00", "value": 9.532790412780196e+29}, {"timestamp": "1952-01-01 00:17:00", "value": 6.597479530290773e+27}, {"timestamp": "1952-01-01 00:18:00", "value": 6.340215383581113e+29}, {"timestamp": "1952-01-01 00:19:00", "value": 2.7627757670183217e+28}, {"timestamp": "1952-01-01 00:20:00", "value": 1.5502976060345364e+29}, {"timestamp": "1952-01-01 00:21:00", "value": 2.799956365220937e+29}, {"timestamp": "1952-01-01 00:22:00", "value": 3.3050155016444136e+29}, {"timestamp": "1952-01-01 00:23:00", "value": 5.5720196555326005e+29}, {"timestamp": "1952-01-01 00:24:00", "value": 8.916149881641319e+29}, {"timestamp": "1952-01-01 00:25:00", "value": 1.0129409839452642e+29}, {"timestamp": "1952-01-01 00:26:00", "value": 7.561809428456247e+29}, {"timestamp": "1952-01-01 00:27:00", "value": 1.3123161382244242e+29}, {"timestamp": "1952-01-01 00:28:00", "value": 1.2931421174603997e+28}, {"timestamp": "1952-01-01 00:29:00", "value": 9.184746169547731e+29}], "granularity": "minutely", "maxAnomalyRatio": 0.25, "sensitivity": 95, "period": 0}, "response": {"expectedValues": [4.8891175062095475e+29, 3.774552771273015e+29, 5.597115959214402e+29, 5.2663067239764365e+29, 5.201399895000252e+29, 5.7155446458947644e+29, 5.394801480354508e+29, 4.6346708121866356e+29, 3.9406490884099876e+29, 3.433810352521214e+29, 3.486276325638303e+29, 3.495739303880064e+29, 2.791449275436046e+29, 3.596706378243274e+29, 3.694832987194294e+29, 2.9306432063321625e+29, 3.349594197787344e+29, 2.9787613864403813e+29, 3.280793280473517e+29, 2.7564064069325337e+29, 2.2862214837480607e+29, 2.2065443453682662e+29, 2.854352486636566e+29, 2.7007133410268624e+29, 4.428687802014759e+29, 4.321216477596904e+29, 5.273587090243965e+29, 4.8750472175599666e+29, 4.460834170801898e+29, 4.514553428383181e+29]}, "type": "entire"} -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Environment 2 | Tested on 3 | [![Python 3.10](https://img.shields.io/badge/python-3.10-blue.svg)](https://www.python.org/downloads/release/python-3100/) 4 | [![Python 3.11](https://img.shields.io/badge/python-3.11-blue.svg)](https://www.python.org/downloads/release/python-3110/) 5 | [![Python 3.12](https://img.shields.io/badge/python-3.12-blue.svg)](https://www.python.org/downloads/release/python-3120/) 6 | 7 | 8 | # Getting Started 9 | 10 | ## Installing from pip 11 | 12 | ```bash 13 | # install time-series-anomaly-detector 14 | pip install time-series-anomaly-detector==0.4.0 15 | ``` 16 | 17 | ## Installing from Source 18 | 19 | 20 | ```bash 21 | git clone https://github.com/microsoft/anomaly-detector.git 22 | pip install -e . 23 | ``` 24 | 25 | ## Test 26 | 27 | ```bash 28 | cd anomaly-detector/tests 29 | python uvad_test.py 30 | python test_demo.py 31 | ``` 32 | 33 | # Project 34 | 35 | > This repo has been populated by an initial template to help get you started. Please 36 | > make sure to update the content to build a great experience for community-building. 37 | 38 | As the maintainer of this project, please make a few updates: 39 | 40 | - Improving this README.MD file to provide a great experience 41 | - Updating SUPPORT.MD with content about this project's support experience 42 | - Understanding the security reporting process in SECURITY.MD 43 | - Remove this section from the README 44 | 45 | ## Contributing 46 | 47 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 48 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 49 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 50 | 51 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 52 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 53 | provided by the bot. You will only need to do this once across all repos using our CLA. 54 | 55 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 56 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 57 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 58 | 59 | ## Trademarks 60 | 61 | This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 62 | trademarks or logos is subject to and must follow 63 | [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). 64 | Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. 65 | Any use of third-party trademarks or logos are subject to those third-party's policies. 66 | -------------------------------------------------------------------------------- /src/anomaly_detector/multivariate/util.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | import numpy as np 6 | import pandas as pd 7 | import torch 8 | 9 | 10 | class AverageMeter: 11 | def __init__(self): 12 | self.sum = None 13 | self.avg = None 14 | self.count = None 15 | self.reset() 16 | 17 | def reset(self): 18 | self.sum = 0.0 19 | self.avg = 0.0 20 | self.count = 0 21 | 22 | def update(self, val, num, is_sum=True): 23 | self.sum += val if is_sum else val * num 24 | self.count += num 25 | self.avg = self.sum / self.count 26 | 27 | 28 | def get_threshold(scores, p=0.95): 29 | if len(scores) == 0: 30 | return 0 31 | data = np.sort(scores) 32 | return data[int(len(data) * p)] 33 | 34 | 35 | def minmax_normalize(data, min_val, max_val, clip_min, clip_max): 36 | data_normalized = (data - min_val) / (max_val - min_val + 1e-8) 37 | if isinstance(data_normalized, np.ndarray): 38 | data_normalized = np.clip(data_normalized, clip_min, clip_max) 39 | elif isinstance(data_normalized, torch.Tensor): 40 | data_normalized = torch.clamp(data_normalized, clip_min, clip_max) 41 | else: 42 | raise NotImplementedError 43 | return data_normalized 44 | 45 | 46 | def get_multiple_variables_pct_weight_score(data, window, timestamp_col="timestamp"): 47 | max_pct_weight = 1.9 48 | if isinstance(data, pd.DataFrame): 49 | if timestamp_col in data.columns: 50 | data = data.drop(timestamp_col, axis=1) 51 | data = data[sorted(data.columns)] 52 | data = data.values 53 | elif isinstance(data, np.ndarray): 54 | data = data 55 | else: 56 | raise TypeError(f"Unsupported type {type(data)}") 57 | variables_num = data.shape[1] 58 | pct_weight = np.empty(variables_num, float) 59 | i = 0 60 | for i in range(variables_num): 61 | s_data = pd.Series(data[:, i] + 0.0001) 62 | data1 = np.maximum( 63 | np.abs((s_data.shift(1) / s_data) - 1), 64 | np.abs((s_data.shift(-1) / s_data) - 1), 65 | ) 66 | data2 = np.clip(data1, a_max=2, a_min=0) 67 | pct_weight[i] = data2.rolling(window).max().mean() 68 | i = i + 1 69 | reweight_num = np.count_nonzero(pct_weight > max_pct_weight) 70 | reweight_value = reweight_num / variables_num 71 | for i in range(variables_num): 72 | if pct_weight[i] > max_pct_weight: 73 | pct_weight[i] = reweight_value 74 | else: 75 | pct_weight[i] = 1 76 | return pct_weight.tolist() 77 | 78 | 79 | def append_result(result, batch): 80 | if result is None: 81 | result = batch 82 | else: 83 | result = np.concatenate([result, batch]) 84 | return result 85 | 86 | 87 | def compute_anomaly_scores(rmse, prob, gamma): 88 | return (prob * gamma + rmse) / (1 + gamma) 89 | 90 | 91 | def compute_severity(inference_scores): 92 | return inference_scores / (np.e - 1) 93 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/model/dynamic_threshold.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | import pandas as pd 6 | import numpy as np 7 | import math 8 | 9 | from anomaly_detector.univariate.detectors import ESD 10 | from anomaly_detector.univariate.detectors import ZScoreDetector 11 | from anomaly_detector.univariate.model.detect import detect 12 | from anomaly_detector.univariate.util import Value, Direction, ExpectedValue, IsAnomaly, IsNegativeAnomaly, IsPositiveAnomaly, Trend, \ 13 | ModelType, get_verified_majority_value, normalize 14 | from anomaly_detector.univariate.util import interp, trend_detection 15 | 16 | 17 | def dynamic_threshold_detection(series, trend_values, alpha, max_anomaly_ratio, need_trend, last_value=None): 18 | directions = [Direction.upper_tail, Direction.lower_tail] 19 | _series = pd.Series(series) 20 | data = pd.Series(normalize(_series)) 21 | anomalies, model_id = detect_anomaly(data=data, alpha=alpha, ratio=max_anomaly_ratio, 22 | directions=directions, last_value=last_value) 23 | if len(anomalies) != 0: 24 | de_anomaly_series = pd.Series(series) 25 | de_anomaly_series[anomalies.index] = np.nan 26 | trend_values = trend_detection(interp(de_anomaly_series)) 27 | 28 | expected_values = _series.to_frame(name=Value) 29 | expected_values[ExpectedValue] = trend_values 30 | 31 | if need_trend: 32 | expected_values[Trend] = trend_values 33 | 34 | if len(anomalies) != 0: 35 | anomalies = anomalies.join(expected_values, how='inner') 36 | anomalies[IsPositiveAnomaly], anomalies[IsNegativeAnomaly] = \ 37 | zip(*anomalies.apply(lambda x: modify_anomaly_status(x), axis=1)) 38 | 39 | anomalies = anomalies[[IsAnomaly, IsPositiveAnomaly, IsNegativeAnomaly]] 40 | 41 | merged_result = expected_values.join(anomalies, how='left') 42 | merged_result.fillna(False, inplace=True) 43 | 44 | return merged_result, model_id 45 | 46 | 47 | def detect_anomaly(data, alpha, ratio, directions, last_value=None): 48 | sorted_data = data.sort_values(ascending=True) 49 | num_obs = len(data) 50 | max_outliers = min(max(math.ceil(num_obs * ratio), 1), len(data) // 2 - 1) 51 | majority_value = get_verified_majority_value(sorted_data, num_obs) 52 | detectors = [ESD(series=sorted_data, alpha=alpha, max_outliers=max_outliers, majority_value=majority_value), 53 | ZScoreDetector(series=sorted_data, max_outliers=max_outliers)] 54 | 55 | model_id = ModelType.DynamicThreshold 56 | if majority_value is not None: 57 | model_id = ModelType.DynamicThresholdMad 58 | 59 | return detect(directions=directions, detectors=detectors, max_outliers=max_outliers, num_obs=num_obs, 60 | last_value=last_value), model_id 61 | 62 | 63 | def modify_anomaly_status(x): 64 | if not x[IsAnomaly]: 65 | return False, False 66 | if x[ExpectedValue] > x[Value]: 67 | return False, True 68 | else: 69 | return True, False 70 | -------------------------------------------------------------------------------- /src/anomaly_detector/multivariate/dataset.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from typing import List, Union 6 | 7 | import numpy as np 8 | import pandas as pd 9 | import torch 10 | from anomaly_detector.multivariate.util import minmax_normalize 11 | from torch.utils.data import Dataset 12 | 13 | 14 | class MultiADDataset(Dataset): 15 | """ 16 | Dataset for Multi-AD model 17 | """ 18 | 19 | def __init__( 20 | self, 21 | data: np.ndarray, 22 | window_size: int, 23 | interval: int, 24 | horizon: int, 25 | max_values: Union[np.ndarray, List[float]], 26 | min_values: Union[np.ndarray, List[float]], 27 | clip_min: float = 0.0, 28 | clip_max: float = 1.0, 29 | ): 30 | """ 31 | :param data: 2-d array, shape (num_timestamps, num_variables) 32 | :param window_size: size of the sliding window 33 | :param interval: step size of the sliding window 34 | :param horizon: horizon of the prediction 35 | :param max_values: max value of each variable 36 | :param min_values: min value of each variable 37 | :param clip_min: min value of the normalized data 38 | :param clip_max: max value of the normalized data 39 | """ 40 | self.window_size = window_size 41 | self.interval = interval 42 | self.horizon = horizon 43 | self.max_values = np.array(max_values) 44 | self.min_values = np.array(min_values) 45 | self.data = data 46 | self.data_length = len(self.data) 47 | self.clip_min = clip_min 48 | self.clip_max = clip_max 49 | self.x_end_idx = self.get_x_end_idx() 50 | 51 | def get_x_end_idx(self): 52 | # each element `hi` in `x_index_set` is an upper bound for getting data 53 | # range: [lo, hi), lo = hi - window_size 54 | x_index_set = range(self.window_size, self.data_length - self.horizon + 1) 55 | x_end_idx = [ 56 | x_index_set[j * self.interval] 57 | for j in range((len(x_index_set)) // self.interval) 58 | ] 59 | return x_end_idx 60 | 61 | def __len__(self): 62 | return len(self.x_end_idx) 63 | 64 | def __getitem__(self, idx): 65 | """ 66 | :param idx: int, index of the sample 67 | :return x: 2-d array, shape (window_size, num_variables) 68 | :return y: 1-d array, shape (num_variables,) 69 | """ 70 | hi = self.x_end_idx[idx] 71 | lo = hi - self.window_size 72 | train_data = self.data[lo:hi] 73 | target_data = self.data[hi - 1 + self.horizon] 74 | train_data = minmax_normalize( 75 | train_data, self.min_values, self.max_values, self.clip_min, self.clip_max 76 | ) 77 | target_data = minmax_normalize( 78 | target_data, self.min_values, self.max_values, self.clip_min, self.clip_max 79 | ) 80 | x = torch.from_numpy(train_data).type(torch.float) 81 | y = torch.from_numpy(target_data).type(torch.float) 82 | return x, y 83 | -------------------------------------------------------------------------------- /tests/cases/function_module_test.test_last_request_15.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2018-05-01T00:00:00Z", "value": 0.0}, {"timestamp": "2018-05-02T00:00:00Z", "value": 0.49999999999999994}, {"timestamp": "2018-05-03T00:00:00Z", "value": 0.8660254037844386}, {"timestamp": "2018-05-04T00:00:00Z", "value": 1.0}, {"timestamp": "2018-05-05T00:00:00Z", "value": 0.8660254037844387}, {"timestamp": "2018-05-06T00:00:00Z", "value": 0.49999999999999994}, {"timestamp": "2018-05-07T00:00:00Z", "value": 1.2246467991473532e-16}, {"timestamp": "2018-05-08T00:00:00Z", "value": -0.5000000000000001}, {"timestamp": "2018-05-09T00:00:00Z", "value": -0.8660254037844384}, {"timestamp": "2018-05-10T00:00:00Z", "value": -1.0}, {"timestamp": "2018-05-11T00:00:00Z", "value": -0.8660254037844386}, {"timestamp": "2018-05-12T00:00:00Z", "value": -0.5000000000000004}, {"timestamp": "2018-05-13T00:00:00Z", "value": -2.4492935982947064e-16}, {"timestamp": "2018-05-14T00:00:00Z", "value": 0.4999999999999993}, {"timestamp": "2018-05-15T00:00:00Z", "value": 0.8660254037844388}, {"timestamp": "2018-05-16T00:00:00Z", "value": 1.0}, {"timestamp": "2018-05-17T00:00:00Z", "value": 0.8660254037844392}, {"timestamp": "2018-05-18T00:00:00Z", "value": 0.4999999999999998}, {"timestamp": "2018-05-19T00:00:00Z", "value": 3.6739403974420594e-16}, {"timestamp": "2018-05-20T00:00:00Z", "value": -0.49999999999999917}, {"timestamp": "2018-05-21T00:00:00Z", "value": -0.8660254037844387}, {"timestamp": "2018-05-22T00:00:00Z", "value": -1.0}, {"timestamp": "2018-05-23T00:00:00Z", "value": -0.8660254037844392}, {"timestamp": "2018-05-24T00:00:00Z", "value": -0.4999999999999999}, {"timestamp": "2018-05-25T00:00:00Z", "value": -4.898587196589413e-16}, {"timestamp": "2018-05-26T00:00:00Z", "value": 0.5000000000000006}, {"timestamp": "2018-05-27T00:00:00Z", "value": 0.8660254037844378}, {"timestamp": "2018-05-28T00:00:00Z", "value": 1.0}, {"timestamp": "2018-05-29T00:00:00Z", "value": 0.8660254037844384}, {"timestamp": "2018-05-30T00:00:00Z", "value": 0.5000000000000016}, {"timestamp": "2018-05-31T00:00:00Z", "value": 6.123233995736766e-16}, {"timestamp": "2018-06-01T00:00:00Z", "value": -0.49999999999999895}, {"timestamp": "2018-06-02T00:00:00Z", "value": -0.8660254037844377}, {"timestamp": "2018-06-03T00:00:00Z", "value": -1.0}, {"timestamp": "2018-06-04T00:00:00Z", "value": -0.8660254037844384}, {"timestamp": "2018-06-05T00:00:00Z", "value": -0.5000000000000001}, {"timestamp": "2018-06-06T00:00:00Z", "value": -7.347880794884119e-16}, {"timestamp": "2018-06-07T00:00:00Z", "value": 0.49999999999999883}, {"timestamp": "2018-06-08T00:00:00Z", "value": 0.8660254037844377}, {"timestamp": "2018-06-09T00:00:00Z", "value": 1.0}, {"timestamp": "2018-06-10T00:00:00Z", "value": 0.8660254037844385}, {"timestamp": "2018-06-11T00:00:00Z", "value": 0.5000000000000002}, {"timestamp": "2018-06-12T00:00:00Z", "value": 8.572527594031472e-16}, {"timestamp": "2018-06-13T00:00:00Z", "value": -0.4999999999999987}, {"timestamp": "2018-06-14T00:00:00Z", "value": -0.8660254037844376}, {"timestamp": "2018-06-15T00:00:00Z", "value": -1.0}, {"timestamp": "2018-06-16T00:00:00Z", "value": -0.8660254037844385}, {"timestamp": "2018-06-17T00:00:00Z", "value": -0.5000000000000003}, {"timestamp": "2018-06-18T00:00:00Z", "value": 0}], "granularity": "daily"}, "response": {"period": 12, "isAnomaly": false, "expectedValue": -7.275383368488165e-17}, "type": "last"} -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/util/fields.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from enum import Enum 6 | 7 | IsAnomaly = "is_anomaly" 8 | IsNegativeAnomaly = "is_negative_anomaly" 9 | IsPositiveAnomaly = "is_positive_anomaly" 10 | Period = "period" 11 | ExpectedValue = "expected_value" 12 | ESDRank = 'ESDRank' 13 | StandardFilterRank = 'DeviationRank' 14 | Reminder = 'Reminder' 15 | Value = 'Value' 16 | AnomalyId = "Id" 17 | AnomalyScore = "Score" 18 | Trend = "Trend" 19 | Severity = 'Severity' 20 | BoundaryUnit = 'BoundaryUnit' 21 | UpperMargin = 'upper_margin' 22 | LowerMargin = 'lower_margin' 23 | SuggestedWindow = 'suggested_window' 24 | 25 | DEFAULT_TREND_TYPE = "spline" 26 | DEFAULT_PERIOD_THRESH = 0.9 27 | DEFAULT_MIN_VAR = 0.20 28 | DEFAULT_MAX_RATIO = 0.25 29 | DEFAULT_ALPHA = 0.05 30 | DEFAULT_MARGIN_FACTOR = 99 31 | DEFAULT_NORMAL_MARGIN_FACTOR = 99 32 | DEFAULT_MAXIMUM_FILLUP_LENGTH = 8640 * 2 33 | DEFAULT_MAX_RATIO = 0.25 34 | DEFAULT_THRESHOLD = 3.5 35 | 36 | VALUE_LOWER_BOUND = -1.0e100 37 | VALUE_UPPER_BOUND = 1.0e100 38 | 39 | EPS = 1e-8 40 | 41 | SKELETON_POINT_SCORE_THRESHOLD = 1.0 42 | MIN_SR_RAW_SCORE = 3.5 43 | MAX_SR_RAW_SCORE = 15.0 44 | 45 | 46 | class Direction(Enum): 47 | upper_tail = 1 48 | lower_tail = 2 49 | 50 | 51 | class ModelType(Enum): 52 | Unknown = 0 53 | AnomalyDetector = 10 54 | SpectralResidual = 20 55 | SpectralResidual_ZScore = 21 56 | Predict = 30 57 | DynamicThreshold = 40 58 | AnomalyDetectorMad = 11 59 | DynamicThresholdMad = 41 60 | PeriodDetection = 50 61 | ChangePointDetection = 60 62 | HbosNonseasonal = 70 63 | HbosSeasonal = 71 64 | PatternDetection = 80 65 | 66 | class DetectType(Enum): 67 | ENTIRE = 0 68 | LATEST = 1 69 | 70 | class Granularity(Enum): 71 | none = -1 72 | yearly = 0 73 | monthly = 1 74 | weekly = 2 75 | daily = 3 76 | hourly = 4 77 | minutely = 5 78 | secondly = 6 79 | microsecond = 7 80 | 81 | 82 | DEFAULT_GRANULARITY_NONE = Granularity.none 83 | 84 | 85 | class BoundaryVersion(Enum): 86 | V1 = 1 87 | V2 = 2 88 | V3 = 3 89 | 90 | 91 | class FillUpMode(Enum): 92 | # these are to be compatible with current implementation 93 | # TODO: remove them 94 | use_last = 'last' 95 | no = 'no' 96 | 97 | auto = 'auto' 98 | previous = 'previous' 99 | linear = 'linear' 100 | zero = 'zero' 101 | fixed = 'fixed' 102 | notFill = 'notFill' 103 | 104 | 105 | DEFAULT_FILL_UP_MODE = FillUpMode.auto 106 | 107 | YEAR_SECOND = 12 * 4 * 7 * 24 * 60 * 60 108 | MONTH_SECOND = 4 * 7 * 24 * 60 * 60 109 | WEEK_SECOND = 7 * 24 * 60 * 60 110 | DAY_SECOND = 24 * 60 * 60 111 | HOUR_SECOND = 60 * 60 112 | MINUTE_SECOND = 60 113 | SECOND = 1 114 | MICRO_SECOND = 0.001 115 | 116 | DEFAULT_CHANGEPOINT_THRESHOLD = { 117 | "yearly": 0.9, 118 | "monthly": 0.9, 119 | "weekly": 0.9, 120 | "daily": 0.9, 121 | "hourly": 0.91, 122 | "minutely": 0.9, 123 | "secondly": 0.9, 124 | "microsecond": 0.9, 125 | "none": 0.9 126 | } 127 | 128 | DEFAULT_CHANGEPOINT_WINDOW = { 129 | "yearly": 3, 130 | "monthly": 6, 131 | "weekly": 3, 132 | "daily": 5, 133 | "hourly": 24, 134 | "minutely": 72, 135 | "secondly": 72, 136 | "microsecond": 100, 137 | "none": 100 138 | } 139 | 140 | DEFAULT_MIN_WINDOW = 3 141 | DEFAULT_PERIOD = None 142 | 143 | TREND_TYPES = ["median", "mean", "line", "spline"] 144 | 145 | DEFAULT_DETECTOR_TYPE = "correlogram" 146 | # DEFAULT_DETECTOR_TYPE = "periodogram" 147 | DEFAULT_GRANULARITY = Granularity.minutely 148 | DEFAULT_INTERVAL = 1 149 | MIN_PERIOD = 4 150 | -------------------------------------------------------------------------------- /.github/workflows/cd.yml: -------------------------------------------------------------------------------- 1 | name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'v*' 7 | tags: 8 | - '*' 9 | 10 | jobs: 11 | build_wheels: 12 | name: Build wheels on ${{ matrix.os }} 📦 13 | runs-on: ${{ matrix.os }} 14 | strategy: 15 | matrix: 16 | os: 17 | ['ubuntu-latest', 'windows-latest'] 18 | 19 | 20 | steps: 21 | - uses: actions/checkout@v4 22 | 23 | - uses: actions/setup-python@v5 24 | 25 | - name: Install cibuildwheel 26 | run: python -m pip install cibuildwheel==2.19.2 27 | 28 | - name: Build wheels 29 | env: 30 | CIBW_ARCHS_WINDOWS: AMD64 31 | CIBW_ARCHS_LINUX: x86_64 32 | CIBW_BEFORE_TEST: > 33 | python -m pip install --upgrade pip setuptools wheel && 34 | python -m pip install --only-binary=:all: pyarrow 35 | run: python -m cibuildwheel --output-dir wheelhouse 36 | 37 | - uses: actions/upload-artifact@v4 38 | with: 39 | name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} 40 | path: ./wheelhouse/*.whl 41 | 42 | build_sdist: 43 | name: Build source distribution 44 | runs-on: ubuntu-latest 45 | steps: 46 | - uses: actions/checkout@v4 47 | 48 | - name: Build sdist 49 | run: pipx run build --sdist 50 | 51 | - uses: actions/upload-artifact@v4 52 | with: 53 | name: cibw-sdist 54 | path: dist/*.tar.gz 55 | 56 | publish-to-testpypi: 57 | name: Publish Python 🐍 distribution 📦 to TestPyPI 58 | needs: 59 | - build_wheels 60 | - build_sdist 61 | runs-on: ubuntu-latest 62 | 63 | environment: 64 | name: release 65 | url: https://test.pypi.org/p/time-series-anomaly-detector 66 | 67 | permissions: 68 | id-token: write # IMPORTANT: mandatory for trusted publishing 69 | 70 | steps: 71 | - name: Download all the dists 72 | uses: actions/download-artifact@v4 73 | with: 74 | pattern: cibw-* 75 | path: dist 76 | merge-multiple: true 77 | 78 | - name: Publish distribution 📦 to TestPyPI 79 | uses: pypa/gh-action-pypi-publish@release/v1 80 | with: 81 | repository-url: https://test.pypi.org/legacy/ 82 | 83 | - name: Store the distribution packages 84 | uses: actions/upload-artifact@v4 85 | with: 86 | name: cibw-all-files 87 | path: dist 88 | 89 | 90 | github-release: 91 | name: >- 92 | Sign the Python 🐍 distribution 📦 with Sigstore 93 | and upload them to GitHub Release 94 | needs: 95 | - publish-to-testpypi 96 | runs-on: ubuntu-latest 97 | 98 | permissions: 99 | contents: write # IMPORTANT: mandatory for making GitHub Releases 100 | id-token: write # IMPORTANT: mandatory for sigstore 101 | 102 | steps: 103 | - name: Download all the dists 104 | uses: actions/download-artifact@v4 105 | with: 106 | name: cibw-all-files 107 | path: dist 108 | 109 | - name: Sign the dists with Sigstore 110 | uses: sigstore/gh-action-sigstore-python@v3.0.0 111 | with: 112 | inputs: >- 113 | ./dist/*.tar.gz 114 | ./dist/*.whl 115 | - name: Create GitHub Release 116 | env: 117 | GITHUB_TOKEN: ${{ github.token }} 118 | run: >- 119 | gh release create 120 | '${{ github.ref_name }}' 121 | --repo '${{ github.repository }}' 122 | --notes "" 123 | - name: Upload artifact signatures to GitHub Release 124 | env: 125 | GITHUB_TOKEN: ${{ github.token }} 126 | # Upload to GitHub Release using the `gh` CLI. 127 | # `dist/` contains the built packages, and the 128 | # sigstore-produced signatures and certificates. 129 | run: >- 130 | gh release upload 131 | '${{ github.ref_name }}' ./dist/** 132 | --repo '${{ github.repository }}' -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/model/spectral_residual_model.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | from anomaly_detector.univariate.detectors.spectral_residual import SpectralResidual 6 | from anomaly_detector.univariate.util import average_filter, Value, ExpectedValue, IsAnomaly, IsNegativeAnomaly, IsPositiveAnomaly, Trend, \ 7 | AnomalyScore, MIN_SR_RAW_SCORE, MAX_SR_RAW_SCORE, SKELETON_POINT_SCORE_THRESHOLD 8 | from anomaly_detector.univariate.util import interp, trend_detection 9 | import numpy as np 10 | import pandas as pd 11 | 12 | 13 | def spectral_residual_detection(series, threshold, max_anomaly_ratio, need_trend, last_value=None): 14 | num_obs = len(series) 15 | max_outliers = max(int(num_obs * max_anomaly_ratio), 1) 16 | detector = SpectralResidual(series=series, max_outliers=max_outliers, threshold=threshold) 17 | anomalies, model_id = detector.detect(last_value=last_value) 18 | merged_result = pd.DataFrame({Value: series}) 19 | merged_result[AnomalyScore] = anomalies[AnomalyScore] 20 | 21 | anomalies_index = anomalies[anomalies[IsAnomaly]].index 22 | anomalies = anomalies.loc[anomalies_index] if len(anomalies_index) != 0 else \ 23 | pd.DataFrame(columns=[IsAnomaly, IsPositiveAnomaly, IsNegativeAnomaly]) 24 | 25 | # calculate expected value for each point not belong to skeleton points 26 | skeleton_points_bool_index = merged_result[AnomalyScore] <= SKELETON_POINT_SCORE_THRESHOLD 27 | expected_values = np.copy(series) 28 | 29 | if last_value is not None: 30 | skeleton_points = merged_result[skeleton_points_bool_index] 31 | expected_values[-1] = np.mean(skeleton_points[len(skeleton_points)//2:][Value]) 32 | else: 33 | skeleton_points_partial_cnt = np.cumsum(skeleton_points_bool_index) 34 | skeleton_points_partial_cnt = np.concatenate(([0], skeleton_points_partial_cnt), axis=0) 35 | skeleton_points_partial_sum = np.cumsum(np.multiply(merged_result[Value].values, skeleton_points_bool_index)) 36 | skeleton_points_partial_sum = np.concatenate(([0], skeleton_points_partial_sum), axis=0) 37 | for i, v in enumerate(skeleton_points_bool_index): 38 | if v is not True: 39 | skeleton_point_cnt = skeleton_points_partial_cnt[i+1] - skeleton_points_partial_cnt[i//2] 40 | if skeleton_point_cnt == 0: 41 | expected_values[i] = np.mean(series[:i+1]) 42 | else: 43 | expected_values[i] = (skeleton_points_partial_sum[i+1] - skeleton_points_partial_sum[i//2]) / \ 44 | skeleton_point_cnt 45 | 46 | expected_values = average_filter(expected_values, 5) 47 | 48 | merged_result[ExpectedValue] = expected_values 49 | 50 | if need_trend: 51 | merged_result[Trend] = trend_detection(expected_values) 52 | 53 | # normailize the anomaly score to be in region [0,1] and 0 represents normal points 54 | merged_result[AnomalyScore] = np.clip(merged_result[AnomalyScore].values - MIN_SR_RAW_SCORE / 55 | (MAX_SR_RAW_SCORE - MIN_SR_RAW_SCORE), 0.0, 1.0) 56 | 57 | if len(anomalies) != 0: 58 | anomalies = anomalies.sort_values(by=AnomalyScore, ascending=False) 59 | anomalies = anomalies[[IsAnomaly]].head(min(max_outliers, len(anomalies))) 60 | anomalies = anomalies.join(merged_result, how='inner') 61 | anomalies[IsPositiveAnomaly], anomalies[IsNegativeAnomaly] = \ 62 | zip(*anomalies.apply(lambda x: modify_anomaly_status(x), axis=1)) 63 | 64 | anomalies = anomalies[[IsAnomaly, IsPositiveAnomaly, IsNegativeAnomaly]] 65 | 66 | merged_result = merged_result.join(anomalies, how='left') 67 | merged_result.fillna(False, inplace=True) 68 | 69 | return merged_result, model_id 70 | 71 | 72 | def modify_anomaly_status(x): 73 | if not x[IsAnomaly]: 74 | return False, False 75 | if x[ExpectedValue] > x[Value]: 76 | return False, True 77 | else: 78 | return True, False 79 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/util/r_stl.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | # -*- coding: utf-8 -*- 6 | 7 | import pandas 8 | import numpy as npy 9 | from rstl import STL 10 | 11 | 12 | def stl_core(data, np=None): 13 | """ 14 | Seasonal-Trend decomposition procedure based on LOESS 15 | 16 | data : pandas.Series 17 | 18 | ns : int 19 | Length of the seasonal smoother. 20 | The value of ns should be an odd integer greater than or equal to 3. 21 | A value ns>6 is recommended. As ns increases the values of the 22 | seasonal component at a given point in the seasonal cycle (e.g., January 23 | values of a monthly series with a yearly cycle) become smoother. 24 | 25 | np : int 26 | Period of the seasonal component. 27 | For example, if the time series is monthly with a yearly cycle, then 28 | np=12. 29 | If no value is given, then the period will be determined from the 30 | ``data`` timeseries. 31 | """ 32 | res_ts = STL(data, np, "periodic", robust=True) 33 | return pandas.DataFrame({"seasonal": res_ts.seasonal, "trend": res_ts.trend, "remainder": res_ts.remainder}) 34 | 35 | 36 | def stl_log(data, np=None): 37 | """ 38 | Seasonal-Trend decomposition procedure based on LOESS for data with log transformation 39 | 40 | data : pandas.Series 41 | 42 | np : int 43 | Period of the seasonal component. 44 | For example, if the time series is monthly with a yearly cycle, then 45 | np=12. 46 | If no value is given, then the period will be determined from the 47 | ``data`` timeseries. 48 | """ 49 | base = min(data) 50 | if base < 1: 51 | data = npy.subtract(data, base) 52 | data = data + 1 # add 1 along in case value scale in _data is extreme compared with 1 53 | 54 | result = STL(npy.log(data), np, "periodic", robust=True) 55 | trend_log = result.trend 56 | seasonal_log = result.seasonal 57 | 58 | trend = npy.exp(trend_log) 59 | seasonal = npy.exp(trend_log + seasonal_log) - trend 60 | remainder = data - trend - seasonal 61 | 62 | if base < 1: 63 | trend = trend - 1 64 | trend = trend + base 65 | 66 | try: 67 | res_ts = pandas.DataFrame({"seasonal": seasonal, 68 | "trend": trend, 69 | "remainder": remainder}) 70 | except Exception as e: 71 | raise e 72 | 73 | return res_ts 74 | 75 | 76 | def stl(data, np=None, log_transform=False): 77 | if log_transform: 78 | return stl_log(data.copy(), np) 79 | else: 80 | return stl_core(data.copy(), np) 81 | 82 | 83 | def stl_adjust_trend(data, np, log_transform=False): 84 | """ 85 | extend one point at the end of data to make the stl decompose result more robust 86 | :param data: pandas.Series 87 | :param np: period 88 | :return: 89 | """ 90 | # make sure that data doesn't start or end with nan 91 | _data = data.copy() 92 | 93 | stl_func = stl_log if log_transform else stl_core 94 | 95 | # Append one point to the end of the array to make the stl decompose more robust. 96 | # The value of the point is the median of points in the same phrase in the cycle. 97 | extended_data = npy.append(_data, [npy.median(_data[-np::-np])]) 98 | 99 | origin_stl_result = stl_func(_data, np=np) 100 | adjust_stl_result = stl_func(extended_data, np=np) 101 | 102 | origin_diff = npy.abs(origin_stl_result['remainder'].values[-1]) 103 | adjust_diff = npy.abs(adjust_stl_result['remainder'].values[-2]) 104 | 105 | if origin_diff <= adjust_diff: 106 | return origin_stl_result 107 | else: 108 | return pandas.DataFrame({"seasonal": adjust_stl_result['seasonal'][:len(data)], 109 | "trend": adjust_stl_result['trend'][:len(data)], 110 | "remainder": adjust_stl_result['remainder'][:len(data)]}) 111 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/period/simple.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | import numpy as np 6 | import math 7 | from anomaly_detector.univariate._anomaly_kernel_cython import gcv, median_filter, remove_anomaly_in_bucket 8 | from anomaly_detector.univariate.util import Granularity, normalize, smooth_spikes, MIN_PERIOD 9 | 10 | 11 | class SimpleDetector: 12 | 13 | @staticmethod 14 | def detect(series, granularity, interval): 15 | if series is None: 16 | return None 17 | series_array = np.array(series) 18 | 19 | period = SimpleDetector.verify_period(series_array, granularity, interval) 20 | return period 21 | 22 | @staticmethod 23 | def verify_period(values, granularity, interval): 24 | periods = SimpleDetector.guess_period(granularity, interval) 25 | if len(periods) == 0: 26 | return None 27 | 28 | verified = None 29 | for period in periods: 30 | if len(values) <= period * 2 or period < MIN_PERIOD: 31 | continue 32 | if SimpleDetector.is_valid_period(values, period): 33 | verified = period 34 | break 35 | if verified: 36 | std_period = SimpleDetector.standard_period(granularity, interval) 37 | if len(values) <= std_period * 2 or std_period % verified != 0: 38 | return verified 39 | return std_period 40 | return None 41 | 42 | @staticmethod 43 | def is_valid_period(values, period): 44 | normed_values = normalize(values) 45 | removed_spike = smooth_spikes(normed_values) 46 | if np.isclose(removed_spike.var(), 0.0): 47 | return False 48 | if SimpleDetector.check_period(removed_spike, period, False): 49 | return True 50 | median_trend = median_filter(normed_values, period, True) 51 | detrended = normed_values - median_trend 52 | detrended = smooth_spikes(detrended) 53 | detrended = remove_anomaly_in_bucket(detrended, period) 54 | if np.isclose(detrended.var(), 0.0): 55 | return False 56 | return SimpleDetector.check_period(detrended, period, True) 57 | 58 | @staticmethod 59 | def check_period(values, period, detrend): 60 | config_mse = SimpleDetector.get_period_config(period) if not detrend else SimpleDetector.get_period_detrend_config(period) 61 | var = values.var() 62 | cv_mse, cv_seasons = gcv(values, period) 63 | if np.isclose(cv_mse, 0.0): 64 | mse = 1 65 | else: 66 | mse = 1 - cv_mse / var 67 | return mse > config_mse 68 | 69 | @staticmethod 70 | def get_period_config(period): 71 | if period <= 24: 72 | return 0.35 73 | if period <= 168: 74 | return 0.15 75 | return 0.1 76 | 77 | @staticmethod 78 | def get_period_detrend_config(period): 79 | if period <= 24: 80 | return 0.35 81 | if period <= 168: 82 | return 0.65 83 | return 0.7 84 | 85 | @staticmethod 86 | def guess_period(granularity, interval): 87 | interval = interval if interval else 1 88 | periods = { 89 | Granularity.yearly: [], 90 | Granularity.none: [], 91 | Granularity.daily: [7], 92 | Granularity.hourly: [168 // interval, 24 // interval], 93 | Granularity.minutely: [1440 * 7 // interval, 1440 // interval, 1440 * 2 // interval], 94 | Granularity.weekly: [4 * 3, 4], 95 | Granularity.monthly: [12], 96 | Granularity.secondly: [86400 * 7 // interval, 86400 // interval, 86400 * 2 // interval], 97 | Granularity.microsecond: [1000] 98 | } 99 | return periods[granularity] 100 | 101 | @staticmethod 102 | def standard_period(granularity, interval): 103 | interval = interval if interval else 1 104 | period = { 105 | Granularity.daily: 7, 106 | Granularity.hourly: 168 // interval, 107 | Granularity.minutely: 1440 * 7 // interval, 108 | Granularity.weekly: 12, 109 | Granularity.monthly: 12, 110 | Granularity.secondly: 86400 * 7 // interval, 111 | Granularity.yearly: 0, 112 | Granularity.none: 0 113 | } 114 | return period[granularity] 115 | -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case14_0.json: -------------------------------------------------------------------------------- 1 | {"request":{"sensitivity":100,"series":[{"timestamp":"12/16/2021 4:39:32 AM ","value":7422018},{"timestamp":"12/16/2021 6:01:23 AM ","value":7422314},{"timestamp":"12/16/2021 7:23:14 AM ","value":7422447},{"timestamp":"12/16/2021 8:45:05 AM ","value":7422657},{"timestamp":"12/16/2021 10:06:56 AM ","value":7422683},{"timestamp":"12/16/2021 11:28:47 AM ","value":7422913},{"timestamp":"12/16/2021 12:50:38 PM ","value":7423185},{"timestamp":"12/16/2021 2:12:29 PM ","value":7423515},{"timestamp":"12/16/2021 3:34:20 PM ","value":7423863},{"timestamp":"12/16/2021 4:56:11 PM ","value":7424146},{"timestamp":"12/16/2021 6:18:02 PM ","value":7424437},{"timestamp":"12/16/2021 7:39:53 PM ","value":7424551},{"timestamp":"12/16/2021 9:01:44 PM ","value":7424619},{"timestamp":"12/16/2021 10:23:35 PM ","value":7426591},{"timestamp":"12/16/2021 11:45:26 PM ","value":7426583},{"timestamp":"12/17/2021 1:07:17 AM ","value":7426585},{"timestamp":"12/17/2021 2:29:08 AM ","value":7426614},{"timestamp":"12/17/2021 3:50:59 AM ","value":7426692},{"timestamp":"12/17/2021 5:12:50 AM ","value":7426718},{"timestamp":"12/17/2021 6:34:41 AM ","value":7426728},{"timestamp":"12/17/2021 7:56:32 AM ","value":7426743},{"timestamp":"12/17/2021 9:18:23 AM ","value":7426755},{"timestamp":"12/17/2021 10:40:14 AM ","value":7426795},{"timestamp":"12/17/2021 12:02:05 PM ","value":7426948},{"timestamp":"12/17/2021 1:23:56 PM ","value":7426881},{"timestamp":"12/17/2021 2:45:47 PM ","value":7426777},{"timestamp":"12/17/2021 4:07:38 PM ","value":7427683},{"timestamp":"12/17/2021 5:29:29 PM ","value":7427766},{"timestamp":"12/17/2021 6:51:20 PM ","value":7427847},{"timestamp":"12/17/2021 8:13:11 PM ","value":7428475},{"timestamp":"12/17/2021 9:35:02 PM ","value":7428597},{"timestamp":"12/17/2021 10:56:53 PM ","value":7428910},{"timestamp":"12/18/2021 12:18:44 AM ","value":7429223},{"timestamp":"12/18/2021 1:40:35 AM ","value":7429535},{"timestamp":"12/18/2021 3:02:26 AM ","value":7429848},{"timestamp":"12/18/2021 4:24:17 AM ","value":7430161},{"timestamp":"12/18/2021 5:46:08 AM ","value":7430474},{"timestamp":"12/18/2021 7:07:59 AM ","value":7430786},{"timestamp":"12/18/2021 8:29:50 AM ","value":7431099},{"timestamp":"12/18/2021 9:51:41 AM ","value":7431412},{"timestamp":"12/18/2021 11:13:32 AM ","value":7431489},{"timestamp":"12/18/2021 12:35:23 PM ","value":7431572},{"timestamp":"12/18/2021 1:57:14 PM ","value":7431668},{"timestamp":"12/18/2021 3:19:05 PM ","value":7431740},{"timestamp":"12/18/2021 4:40:56 PM ","value":7432046},{"timestamp":"12/18/2021 6:02:47 PM ","value":7432352},{"timestamp":"12/18/2021 7:24:38 PM ","value":7432658},{"timestamp":"12/18/2021 8:46:29 PM ","value":7432790},{"timestamp":"12/18/2021 10:08:20 PM ","value":7433006},{"timestamp":"12/18/2021 11:30:11 PM ","value":7433090},{"timestamp":"12/19/2021 12:52:02 AM ","value":7433314},{"timestamp":"12/19/2021 2:13:53 AM ","value":7433539},{"timestamp":"12/19/2021 3:35:44 AM ","value":7433763},{"timestamp":"12/19/2021 4:57:35 AM ","value":7433987},{"timestamp":"12/19/2021 6:19:26 AM ","value":7434211},{"timestamp":"12/19/2021 7:41:17 AM ","value":7434435},{"timestamp":"12/19/2021 9:03:08 AM ","value":7434486},{"timestamp":"12/19/2021 10:24:59 AM ","value":7434577},{"timestamp":"12/19/2021 11:46:50 AM ","value":7434657},{"timestamp":"12/19/2021 1:08:41 PM ","value":7434690},{"timestamp":"12/19/2021 2:30:32 PM ","value":7434797},{"timestamp":"12/19/2021 3:52:23 PM ","value":7434868},{"timestamp":"12/19/2021 5:14:14 PM ","value":7434902},{"timestamp":"12/19/2021 6:36:05 PM ","value":7434974},{"timestamp":"12/19/2021 7:57:56 PM ","value":7435252},{"timestamp":"12/19/2021 9:19:47 PM ","value":7435286},{"timestamp":"12/19/2021 10:41:38 PM ","value":7435376},{"timestamp":"12/20/2021 12:03:29 AM ","value":7435487},{"timestamp":"12/20/2021 1:25:20 AM ","value":7435536},{"timestamp":"12/20/2021 2:47:11 AM ","value":7435596},{"timestamp":"12/20/2021 4:09:02 AM ","value":7435705},{"timestamp":"12/20/2021 5:30:53 AM ","value":7435877},{"timestamp":"12/20/2021 6:52:44 AM ","value":7436315},{"timestamp":"12/20/2021 8:14:35 AM ","value":7437051},{"timestamp":"12/20/2021 9:36:26 AM ","value":7437636},{"timestamp":"12/20/2021 10:58:17 AM ","value":7437690},{"timestamp":"12/20/2021 12:20:08 PM ","value":7437903},{"timestamp":"12/20/2021 1:41:59 PM ","value":7438109},{"timestamp":"12/20/2021 3:03:50 PM ","value":7438225},{"timestamp":"12/20/2021 4:25:41 PM ","value":7438635},{"timestamp":"12/20/2021 5:47:32 PM ","value":7438995},{"timestamp":"12/20/2021 7:09:23 PM ","value":7439121},{"timestamp":"12/20/2021 8:31:14 PM ","value":7439121},{"timestamp":"12/20/2021 9:53:05 PM ","value":7439121},{"timestamp":"12/20/2021 11:14:56 PM ","value":7439121},{"timestamp":"12/21/2021 12:36:47 AM ","value":7439121},{"timestamp":"12/21/2021 1:58:38 AM ","value":7442906}],"granularity":"secondly","customInterval":4911},"response":{"isAnomaly":true,"expectedValue":7439505.041953438},"type":"last"} -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/util/date_utils.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | # -*- coding: utf-8 -*- 6 | from datetime import datetime, date 7 | from re import match 8 | import pytz 9 | from calendar import monthrange 10 | 11 | 12 | def json_serial(obj): 13 | if isinstance(obj, (date, datetime)): 14 | return obj.strftime("%Y-%m-%dT%H:%M:%SZ") 15 | 16 | 17 | def datetime_from_ts(column): 18 | return column.map( 19 | lambda date_str: datetime.fromtimestamp(int(date_str), tz=pytz.utc)) 20 | 21 | 22 | def date_format(column, format_str): 23 | return column.map(lambda date_str: datetime.strptime(date_str, format_str)) 24 | 25 | 26 | def format_timestamp_str(date_str): 27 | if match("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2} \\+\\d{4}$", 28 | date_str): 29 | return datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") 30 | elif match("^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$", date_str): 31 | return datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%SZ") 32 | elif match("^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", date_str): 33 | return datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S") 34 | elif match("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$", date_str): 35 | return datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") 36 | elif match("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}$", date_str): 37 | return datetime.strptime(date_str, "%Y-%m-%d %H:%M") 38 | elif match("^\\d{2}/\\d{2}/\\d{2}$", date_str): 39 | return datetime.strptime(date_str, "%m/%d/%y") 40 | elif match("^\\d{2}/\\d{2}/\\d{4}$", date_str): 41 | return datetime.strptime(date_str, "%Y%m%d") 42 | elif match("^\\d{4}\\d{2}\\d{2}$", date_str): 43 | return datetime.strptime(date_str, "%Y/%m/%d/%H") 44 | else: 45 | raise ValueError("timestamp format currently not supported.") 46 | 47 | 48 | def format_timestamp(df, index=0): 49 | column = df.iloc[:, index] 50 | 51 | if match("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2} \\+\\d{4}$", 52 | column[0]): 53 | column = date_format(column, "%Y-%m-%d %H:%M:%S") 54 | elif match("^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$", column[0]): 55 | column = date_format(column, "%Y-%m-%dT%H:%M:%SZ") 56 | elif match("^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", column[0]): 57 | column = date_format(column, "%Y-%m-%dT%H:%M:%S") 58 | elif match("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$", column[0]): 59 | column = date_format(column, "%Y-%m-%d %H:%M:%S") 60 | elif match("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}$", column[0]): 61 | column = date_format(column, "%Y-%m-%d %H:%M") 62 | elif match("^\\d{2}/\\d{2}/\\d{2}$", column[0]): 63 | column = date_format(column, "%m/%d/%y") 64 | elif match("^\\d{2}/\\d{2}/\\d{4}$", column[0]): 65 | column = date_format(column, "%Y%m%d") 66 | elif match("^\\d{4}\\d{2}\\d{2}$", column[0]): 67 | column = date_format(column, "%Y/%m/%d/%H") 68 | elif match("^\\d{10}$", column[0]): 69 | column = datetime_from_ts(column) 70 | else: 71 | raise ValueError("timestamp format currently not supported.") 72 | 73 | df.iloc[:, index] = column 74 | 75 | return df 76 | 77 | 78 | def format_timestamp_to_str(df): 79 | df['Timestamp'] = df['Timestamp'].map(lambda x: x.strftime("%Y-%m-%dT%H:%M:%SZ")) 80 | return df 81 | 82 | 83 | class DateDifference: 84 | def __init__(self, years=0, months=0, days=0): 85 | self.years = years 86 | self.months = months 87 | self.days = days 88 | 89 | def __str__(self): 90 | return f'years={self.years},months={self.months},days={self.days}' 91 | 92 | 93 | def get_days_in_month(year, month): 94 | return monthrange(year, month)[1] 95 | 96 | 97 | def get_date_difference(a, b): 98 | factor = 1 99 | if a < b: 100 | tmp = a 101 | a = b 102 | b = tmp 103 | factor = -1 104 | 105 | a_days_in_month = get_days_in_month(a.year, a.month) 106 | b_days_in_month = get_days_in_month(b.year, b.month) 107 | 108 | diff_day = 0 109 | diff_month = 0 110 | diff_year = 0 111 | 112 | if a.year == b.year and a.month == b.month: 113 | diff_day = a.day - b.day 114 | elif (a.day == b.day) or (a.day == a_days_in_month and b.day == b_days_in_month) \ 115 | or (a_days_in_month != b_days_in_month and ( 116 | a.day == a_days_in_month and b.day > a.day or b.day == b_days_in_month and a.day > b.day)): 117 | diff_month = a.month - b.month 118 | else: 119 | if a.day > b.day: 120 | diff_day = a.day - b.day 121 | diff_month = a.month - b.month 122 | else: 123 | diff_day = b_days_in_month - b.day + a.day 124 | diff_month = a.month - b.month - 1 125 | diff_year = a.year - b.year 126 | if diff_month < 0: 127 | diff_year -= 1 128 | diff_month += 12 129 | return DateDifference(years=factor * diff_year, months=factor * diff_month, days=factor * diff_day) 130 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/period/spectrum.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | import numpy as np 6 | from anomaly_detector.univariate._anomaly_kernel_cython import max_gcv, gcv 7 | from seasonal import periodogram_peaks 8 | import statsmodels.api as sm 9 | from anomaly_detector.univariate.util import fit_trend, normalize, smooth_spikes, MIN_PERIOD 10 | 11 | 12 | class SpectrumDetector: 13 | 14 | @staticmethod 15 | def detect(series, trend_type, thresh, min_var, detector_type): 16 | if len(series) < 12: 17 | raise ValueError("Series length cannot be less than 12 for period detection.") 18 | 19 | series_array = np.array(series) 20 | 21 | series_array, s_max, s_min = normalize(series_array, min_max=True) 22 | 23 | period, seasons, trend = SpectrumDetector.calculate_period( 24 | series=series_array, trend_type=trend_type, 25 | thresh=thresh, min_var=min_var, detector_type=detector_type 26 | ) 27 | 28 | if period == 0: 29 | return period 30 | 31 | verified_period = period 32 | while verified_period != 0: 33 | series_array = series_array[0::verified_period] 34 | verified_period, _, _ = SpectrumDetector.calculate_period( 35 | series_array, trend_type, thresh, min_var, detector_type) 36 | if verified_period != 0: 37 | period = period * verified_period 38 | 39 | return period 40 | 41 | @staticmethod 42 | def calculate_period(series, trend_type, thresh, min_var, detector_type): 43 | if len(series) < 12: 44 | return 0, None, None 45 | 46 | seasons, trend = SpectrumDetector.fit_seasons( 47 | series, trend_type=trend_type, period_gram_thresh=thresh, 48 | min_ev=min_var, detector_type=detector_type, 49 | ) 50 | if seasons is None or len(seasons) == 0: 51 | return 0, seasons, trend 52 | 53 | period = len(seasons) 54 | 55 | cycles = len(series) / period + 1 56 | if cycles <= 3: 57 | return 0, seasons, trend 58 | 59 | return period, seasons, trend 60 | 61 | @staticmethod 62 | def is_valid_period(first, second): 63 | if first >= second: 64 | return first % second == 0 65 | 66 | return second % first == 0 67 | 68 | @staticmethod 69 | def fit_seasons(data, trend_type="spline", period=None, min_ev=0.05, 70 | period_gram_thresh=0.5, detector_type="periodogram"): 71 | data = smooth_spikes(data) 72 | 73 | if trend_type is None: 74 | trend = np.zeros(len(data)) 75 | elif not isinstance(trend_type, np.ndarray): 76 | trend = fit_trend(data, kind=trend_type, period=period) 77 | else: 78 | assert isinstance(trend_type, np.ndarray) 79 | data = data - trend 80 | var = data.var() 81 | if np.isclose(var, 0.0): 82 | return None, trend 83 | 84 | if period: 85 | cv_mse, cv_seasons = gcv(data, period) 86 | fev = 1 - cv_mse / var 87 | if np.isclose(cv_mse, 0.0) or fev >= min_ev: 88 | return cv_seasons, trend 89 | else: 90 | return None, trend 91 | 92 | if detector_type == "periodogram": 93 | periods = SpectrumDetector.periodogram_detector(data, period_gram_thresh) 94 | else: 95 | periods = SpectrumDetector.correlogram_detector(data) 96 | 97 | if len(periods) == 0: 98 | return None, trend 99 | cv_mse, cv_seasons = max_gcv(data, np.array(periods, dtype='i')) 100 | if np.isclose(cv_mse, 0.0) or min_ev <= 1 - cv_mse / var: 101 | return cv_seasons, trend 102 | else: 103 | return None, trend 104 | 105 | @staticmethod 106 | def periodogram_detector(data, period_gram_thresh): 107 | if period_gram_thresh: 108 | peaks = periodogram_peaks(data, thresh=period_gram_thresh) 109 | if peaks is None: 110 | return [] 111 | peaks = sorted(peaks) 112 | else: 113 | peaks = [(0, 0, 4, len(data) // 2)] 114 | periods = [] 115 | period = 0 116 | for peak in peaks: 117 | periods.extend(range(max(period, peak[2]), peak[3] + 1)) 118 | period = peak[3] + 1 119 | return periods 120 | 121 | @staticmethod 122 | def correlogram_detector(data, min_period=MIN_PERIOD, max_period=None, corr_thresh=0.1): 123 | if max_period is None: 124 | max_period = int(min(len(data) / 3.0, 2880 * 2)) 125 | 126 | acf, conf = sm.tsa.acf(data, nlags=max_period, fft=False, alpha=0.1) 127 | acf = acf[1:] 128 | conf = conf[1:] 129 | periods = [] 130 | 131 | while True: 132 | peak_i = acf.argmax() 133 | ub = conf[peak_i, 1] - acf[peak_i] 134 | if acf[peak_i] < ub or acf[peak_i] < corr_thresh: 135 | break 136 | 137 | acf[peak_i] = 0 138 | if min_period < peak_i + 1 < max_period: 139 | periods.append(peak_i + 1) 140 | return periods 141 | -------------------------------------------------------------------------------- /anomaly-detector-sample-code.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "a2f4ea7a79dcc5e", 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "source": [ 10 | "## Installing from pip" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "id": "a0f996e3f768461f", 17 | "metadata": { 18 | "collapsed": false, 19 | "vscode": { 20 | "languageId": "powershell" 21 | } 22 | }, 23 | "outputs": [], 24 | "source": "%pip install time-series-anomaly-detector==0.2.6" 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "id": "8346a504", 29 | "metadata": {}, 30 | "source": [ 31 | "## MVAD" 32 | ] 33 | }, 34 | { 35 | "cell_type": "markdown", 36 | "id": "4f25e1b2", 37 | "metadata": {}, 38 | "source": [ 39 | "### Train model\n", 40 | "To train a mvad model, the type of training data should be DataFrame type. And you must specify the sliding_window and device in params. \n", 41 | "\n", 42 | "Moreover, please note that in mvad, timestamp of the training data is optional." 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "id": "d5da3531", 49 | "metadata": {}, 50 | "outputs": [], 51 | "source": [ 52 | "import numpy as np\n", 53 | "import pandas as pd\n", 54 | "from anomaly_detector import MultivariateAnomalyDetector\n", 55 | "\n", 56 | "import json\n", 57 | "from pprint import pprint\n", 58 | "\n", 59 | "data_size = 1000\n", 60 | "var_num = 20\n", 61 | "\n", 62 | "training_data = np.random.randn(data_size, var_num)\n", 63 | "columns = [f\"variable_{i}\" for i in range(var_num)]\n", 64 | "training_data = pd.DataFrame(training_data, columns=columns)\n", 65 | "\n", 66 | "# Optional\n", 67 | "timestamps = pd.date_range(start=\"2023-01-03\", periods=data_size, freq=\"H\")\n", 68 | "training_data[\"timestamp\"] = timestamps.strftime(\"%Y-%m-%dT%H:%M:%SZ\")\n", 69 | "training_data = training_data.set_index(\"timestamp\", drop=True)\n", 70 | "\n", 71 | "params = {\"sliding_window\": 200, \"device\": \"cpu\"}\n", 72 | "\n", 73 | "model = MultivariateAnomalyDetector()\n", 74 | "\n", 75 | "# Train model\n", 76 | "model.fit(training_data, params=params)" 77 | ] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "id": "b918d943", 82 | "metadata": {}, 83 | "source": [ 84 | "### Prediction" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": null, 90 | "id": "f3a010d7", 91 | "metadata": {}, 92 | "outputs": [], 93 | "source": [ 94 | "eval_data = np.random.randn(201, var_num)\n", 95 | "eval_data[-1, :] += 100\n", 96 | "eval_data = pd.DataFrame(eval_data, columns=columns)\n", 97 | "\n", 98 | "# Optional\n", 99 | "timestamps = pd.date_range(start=\"2023-01-03\", periods=201, freq=\"H\")\n", 100 | "eval_data[\"timestamp\"] = timestamps.strftime(\"%Y-%m-%dT%H:%M:%SZ\")\n", 101 | "eval_data = eval_data.set_index(\"timestamp\", drop=True)\n", 102 | "\n", 103 | "# prediction\n", 104 | "results = model.predict(data=eval_data, context=None)\n", 105 | "\n", 106 | "pprint(results)" 107 | ] 108 | }, 109 | { 110 | "cell_type": "markdown", 111 | "id": "1cc129ba", 112 | "metadata": {}, 113 | "source": [ 114 | "## UVAD" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "id": "5d12be4c", 120 | "metadata": {}, 121 | "source": [ 122 | "### Prediction\n", 123 | "\n", 124 | "Please note that the uvad does not need to train before prediction, and timestamp of the eval_data must be specified." 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "id": "232963b5", 131 | "metadata": {}, 132 | "outputs": [], 133 | "source": [ 134 | "import numpy as np\n", 135 | "import pandas as pd\n", 136 | "from anomaly_detector import EntireAnomalyDetector\n", 137 | "\n", 138 | "params = {\n", 139 | " \"granularity\": \"monthly\", \n", 140 | " \"maxAnomalyRatio\": 0.25, \n", 141 | " \"sensitivity\": 95, \n", 142 | " \"imputeMode\": \"auto\"\n", 143 | "}\n", 144 | "\n", 145 | "\n", 146 | "model = EntireAnomalyDetector()\n", 147 | "\n", 148 | "eval_data = np.ones(20)\n", 149 | "eval_data[-1] = 0\n", 150 | "eval_data = pd.DataFrame(eval_data, columns=[\"value\"])\n", 151 | "\n", 152 | "timestamps = pd.date_range(start=\"1962-01-01\", periods=20, freq=\"ME\")\n", 153 | "eval_data[\"timestamp\"] = timestamps\n", 154 | "\n", 155 | "results = model.predict(\n", 156 | " data=eval_data,\n", 157 | " params=params,\n", 158 | " context=None\n", 159 | ")\n", 160 | "print(results)\n" 161 | ] 162 | } 163 | ], 164 | "metadata": { 165 | "kernelspec": { 166 | "display_name": "Python 3", 167 | "language": "python", 168 | "name": "python3" 169 | }, 170 | "language_info": { 171 | "codemirror_mode": { 172 | "name": "ipython", 173 | "version": 3 174 | }, 175 | "file_extension": ".py", 176 | "mimetype": "text/x-python", 177 | "name": "python", 178 | "nbconvert_exporter": "python", 179 | "pygments_lexer": "ipython3", 180 | "version": "3.10.13" 181 | } 182 | }, 183 | "nbformat": 4, 184 | "nbformat_minor": 5 185 | } 186 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/model/hbos_detection.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | import numpy as np 6 | import pandas as pd 7 | from pyod.models.hbos import HBOS 8 | from anomaly_detector.univariate.util import stl, stl_adjust_trend, trend_detection, interp, de_outlier_stl, normalize, MAPE_LB, MAPE_UB 9 | from anomaly_detector.univariate.util import Value, ExpectedValue, IsAnomaly, IsNegativeAnomaly, IsPositiveAnomaly, AnomalyScore, \ 10 | ModelType, Trend 11 | 12 | 13 | def hbos_detection_seasonal(series, period, outlier_fraction, threshold, adjust_trend=False, need_trend=False, last_value=None): 14 | if outlier_fraction > 0.49: 15 | length = len(series) 16 | raise ValueError( 17 | ("max_anomaly_ratio must be less than 50% of " 18 | "the data points (max_anomaly_ratio =%f data_points =%s).") 19 | % (round(outlier_fraction * length, 0), length)) 20 | 21 | num_obs = len(series) 22 | 23 | clamp = (1 / float(num_obs)) 24 | if outlier_fraction < clamp: 25 | outlier_fraction = clamp 26 | 27 | if num_obs < period * 2 + 1: 28 | raise ValueError("Anomaly detection needs at least 2 periods worth of data") 29 | 30 | input_series = series.copy() 31 | stl_func = stl_adjust_trend if adjust_trend else stl 32 | decompose_result = de_outlier_stl(input_series, stl_func=stl_func, period=period, log_transform=False) 33 | 34 | mape = np.mean(np.abs(decompose_result['remainder'] / input_series)) 35 | 36 | if mape > MAPE_UB: 37 | decompose_result_log = de_outlier_stl(input_series, stl_func=stl_func, period=period, 38 | log_transform=True) 39 | mape_log = np.mean(np.abs(decompose_result_log['remainder'] / input_series)) 40 | if mape_log < MAPE_LB: 41 | decompose_result = decompose_result_log 42 | 43 | decompose_trend = decompose_result['trend'].values 44 | decompose_season = decompose_result['seasonal'].values 45 | 46 | values_to_detect = normalize(series - decompose_trend - decompose_season).reshape(-1, 1) 47 | 48 | model = HBOS(contamination=outlier_fraction) 49 | model.fit(values_to_detect) 50 | scores = model.predict_proba(values_to_detect)[:, 1] 51 | isAnomaly = scores > threshold 52 | 53 | if np.any(isAnomaly) and np.sum(isAnomaly) < len(series): 54 | decompose_trend[isAnomaly] = np.nan 55 | decompose_trend = interp(decompose_trend) 56 | 57 | p = { 58 | ExpectedValue: decompose_trend + decompose_season, 59 | Value: series, 60 | IsAnomaly: isAnomaly 61 | } 62 | 63 | if need_trend: 64 | p[Trend] = decompose_trend 65 | 66 | results = pd.DataFrame(p) 67 | if np.any(isAnomaly): 68 | results[IsPositiveAnomaly], results[IsNegativeAnomaly] = \ 69 | zip(*results.apply(lambda x: modify_anomaly_status(x), axis=1)) 70 | 71 | results.fillna(False, inplace=True) 72 | 73 | return results, ModelType.HbosSeasonal 74 | 75 | 76 | def hbos_detection_nonseasonal(series, threshold, outlier_fraction, need_trend=False, last_value=None): 77 | series = np.asarray(series) 78 | num_obs = len(series) 79 | 80 | clamp = (1 / float(num_obs)) 81 | if outlier_fraction < clamp: 82 | outlier_fraction = clamp 83 | 84 | values = series.reshape(-1, 1) 85 | model = HBOS(contamination=outlier_fraction) 86 | model.fit(values) 87 | model.predict_proba(values) 88 | scores = model.predict_proba(values)[:, 1] 89 | isAnomaly = scores > threshold 90 | 91 | result = pd.DataFrame({ 92 | Value: series, 93 | AnomalyScore: scores, 94 | IsAnomaly: isAnomaly 95 | }) 96 | 97 | if np.any(isAnomaly) and np.sum(isAnomaly) < np.sum(isAnomaly): 98 | de_anomaly_series = pd.Series(series) 99 | de_anomaly_series[de_anomaly_series[isAnomaly]] = np.nan 100 | trend_values = trend_detection(interp(de_anomaly_series)) 101 | else: 102 | trend_values = trend_detection(series) 103 | 104 | result[ExpectedValue] = trend_values 105 | 106 | if need_trend: 107 | result[Trend] = trend_values 108 | 109 | if np.any(isAnomaly): 110 | result[IsPositiveAnomaly], result[IsNegativeAnomaly] = \ 111 | zip(*result.apply(lambda x: modify_anomaly_status(x), axis=1)) 112 | 113 | result.fillna(False, inplace=True) 114 | 115 | return result, ModelType.HbosNonseasonal 116 | 117 | 118 | def hbos_detection(series, period, threshold, outlier_fraction, need_trend=False, last_value=None): 119 | if period > 0: 120 | adjust_trend = True if last_value is not None else False 121 | return hbos_detection_seasonal(series, period, 122 | outlier_fraction=outlier_fraction, threshold=threshold, adjust_trend=adjust_trend, 123 | need_trend=need_trend, last_value=last_value) 124 | else: 125 | return hbos_detection_nonseasonal(series, threshold=threshold, outlier_fraction=outlier_fraction, 126 | need_trend=need_trend, last_value=last_value) 127 | 128 | 129 | def modify_anomaly_status(x): 130 | if not x[IsAnomaly]: 131 | return False, False 132 | if x[ExpectedValue] > x[Value]: 133 | return False, True 134 | else: 135 | return True, False 136 | -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case28_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2020-06-01 05:15:00", "value": 0.35069}, {"timestamp": "2020-06-01 05:30:00", "value": 0.42636}, {"timestamp": "2020-06-01 05:45:00", "value": 0.51682}, {"timestamp": "2020-06-01 06:00:00", "value": 0.59226}, {"timestamp": "2020-06-01 06:15:00", "value": 0.65255}, {"timestamp": "2020-06-01 06:30:00", "value": 0.7084699999999999}, {"timestamp": "2020-06-01 06:45:00", "value": 0.76547}, {"timestamp": "2020-06-01 07:00:00", "value": 0.81307}, {"timestamp": "2020-06-01 07:15:00", "value": 0.45321999999999996}, {"timestamp": "2020-06-01 07:30:00", "value": 0.53332}, {"timestamp": "2020-06-01 07:45:00", "value": 0.5969}, {"timestamp": "2020-06-01 08:00:00", "value": 0.6321399999999999}, {"timestamp": "2020-06-01 08:15:00", "value": 0.6280100000000001}, {"timestamp": "2020-06-01 08:30:00", "value": 0.59958}, {"timestamp": "2020-06-01 08:45:00", "value": 0.70529}, {"timestamp": "2020-06-01 09:00:00", "value": 0.67765}, {"timestamp": "2020-06-01 09:15:00", "value": 0.56775}, {"timestamp": "2020-06-01 09:30:00", "value": 0.46647}, {"timestamp": "2020-06-01 09:45:00", "value": 0.21895}, {"timestamp": "2020-06-01 10:00:00", "value": 0.19449}, {"timestamp": "2020-06-01 10:15:00", "value": 0.42573}, {"timestamp": "2020-06-01 10:30:00", "value": 0.75398}, {"timestamp": "2020-06-01 10:45:00", "value": 0.35562}, {"timestamp": "2020-06-01 11:00:00", "value": 0.12061}, {"timestamp": "2020-06-01 11:15:00", "value": 0.08352000000000001}, {"timestamp": "2020-06-01 11:30:00", "value": 0.08357}, {"timestamp": "2020-06-01 11:45:00", "value": 0.03404}, {"timestamp": "2020-06-01 12:00:00", "value": 0.020880000000000003}, {"timestamp": "2020-06-01 12:15:00", "value": 0.0129}, {"timestamp": "2020-06-01 12:30:00", "value": 0.008190000000000001}, {"timestamp": "2020-06-01 12:45:00", "value": 0.005379999999999999}, {"timestamp": "2020-06-01 13:00:00", "value": 0.00361}, {"timestamp": "2020-06-01 13:15:00", "value": 0.0043100000000000005}, {"timestamp": "2020-06-01 13:30:00", "value": 0.01556}, {"timestamp": "2020-06-01 13:45:00", "value": 0.0849}, {"timestamp": "2020-06-01 14:00:00", "value": 0.18217}, {"timestamp": "2020-06-01 14:15:00", "value": 0.20656999999999998}, {"timestamp": "2020-06-01 14:30:00", "value": 0.15539}, {"timestamp": "2020-06-01 14:45:00", "value": 0.12332}, {"timestamp": "2020-06-01 15:00:00", "value": 0.18217}, {"timestamp": "2020-06-01 15:15:00", "value": 0.29136999999999996}, {"timestamp": "2020-06-01 15:30:00", "value": 0.43349}, {"timestamp": "2020-06-01 15:45:00", "value": 0.46237}, {"timestamp": "2020-06-01 16:00:00", "value": 0.3767}, {"timestamp": "2020-06-01 16:15:00", "value": 0.26729}, {"timestamp": "2020-06-01 16:30:00", "value": 0.18149}, {"timestamp": "2020-06-01 16:45:00", "value": 0.13474}, {"timestamp": "2020-06-01 17:00:00", "value": 0.1007}, {"timestamp": "2020-06-01 17:15:00", "value": 0.17298}, {"timestamp": "2020-06-01 17:30:00", "value": 0.26533}, {"timestamp": "2020-06-01 17:45:00", "value": 0.37067}, {"timestamp": "2020-06-01 18:00:00", "value": 0.6321399999999999}, {"timestamp": "2020-06-01 18:15:00", "value": 0.5836100000000001}, {"timestamp": "2020-06-01 18:30:00", "value": 0.27412}, {"timestamp": "2020-06-01 18:45:00", "value": 0.02539}, {"timestamp": "2020-06-01 19:00:00", "value": 0.013040000000000001}, {"timestamp": "2020-06-01 19:15:00", "value": 0.02925}, {"timestamp": "2020-06-01 19:30:00", "value": 0.29562}, {"timestamp": "2020-06-01 19:45:00", "value": 0.5758800000000001}, {"timestamp": "2020-06-01 20:00:00", "value": 0.3158}, {"timestamp": "2020-06-01 20:15:00", "value": 0.01886}, {"timestamp": "2020-06-01 20:30:00", "value": 0.0062}, {"timestamp": "2020-06-01 20:45:00", "value": 0.0007}, {"timestamp": "2020-06-01 21:00:00", "value": 0.00047999999999999996}, {"timestamp": "2020-06-01 21:15:00", "value": 0.00049}, {"timestamp": "2020-06-01 21:30:00", "value": 0.00053}, {"timestamp": "2020-06-01 21:45:00", "value": 0.00095}, {"timestamp": "2020-06-01 22:00:00", "value": 0.00349}, {"timestamp": "2020-06-01 22:15:00", "value": 0.00847}, {"timestamp": "2020-06-01 22:30:00", "value": 0.012029999999999999}, {"timestamp": "2020-06-01 22:45:00", "value": 0.01769}, {"timestamp": "2020-06-01 23:00:00", "value": 0.07362}, {"timestamp": "2020-06-02 00:00:00", "value": 0.09007000000000001}, {"timestamp": "2020-06-02 00:15:00", "value": 0.061989999999999996}, {"timestamp": "2020-06-02 00:30:00", "value": 0.05441}, {"timestamp": "2020-06-02 00:45:00", "value": 0.061860000000000005}, {"timestamp": "2020-06-02 01:00:00", "value": 0.09007000000000001}, {"timestamp": "2020-06-02 01:15:00", "value": 0.18217}, {"timestamp": "2020-06-02 01:30:00", "value": 0.38449}, {"timestamp": "2020-06-02 01:45:00", "value": 0.6161300000000001}, {"timestamp": "2020-06-02 02:00:00", "value": 0.74857}, {"timestamp": "2020-06-02 02:15:00", "value": 0.81375}, {"timestamp": "2020-06-02 02:30:00", "value": 0.88116}, {"timestamp": "2020-06-02 02:45:00", "value": 0.93926}, {"timestamp": "2020-06-02 03:00:00", "value": 0.9449}, {"timestamp": "2020-06-02 03:15:00", "value": 0.71829}, {"timestamp": "2020-06-02 03:30:00", "value": 0.30282}, {"timestamp": "2020-06-02 03:45:00", "value": 0.0978}, {"timestamp": "2020-06-02 04:00:00", "value": 0.09731000000000001}, {"timestamp": "2020-06-02 04:15:00", "value": 0.24356}, {"timestamp": "2020-06-02 04:30:00", "value": 0.50632}, {"timestamp": "2020-06-02 04:45:00", "value": 1.18195}, {"timestamp": "2020-06-02 05:00:00", "value": 1.6446599999999998}, {"timestamp": "2020-06-02 05:15:00", "value": 1.9927}], "granularity": "minutely", "customInterval": 15, "sensitivity": 80, "boundaryVersion": "V3", "needDetectorId": true}, "response": {"isAnomaly": true, "expectedValue": 0.21744976744186048}, "type": "last"} -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case29_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2020-06-03 03:15:00", "value": 0.2231}, {"timestamp": "2020-06-03 03:30:00", "value": 0.39549}, {"timestamp": "2020-06-03 03:45:00", "value": 0.5805899999999999}, {"timestamp": "2020-06-03 04:00:00", "value": 0.74856}, {"timestamp": "2020-06-03 04:15:00", "value": 0.8627799999999999}, {"timestamp": "2020-06-03 04:30:00", "value": 0.85934}, {"timestamp": "2020-06-03 04:45:00", "value": 0.73375}, {"timestamp": "2020-06-03 05:00:00", "value": 0.60654}, {"timestamp": "2020-06-03 05:15:00", "value": 0.62047}, {"timestamp": "2020-06-03 05:30:00", "value": 0.72806}, {"timestamp": "2020-06-03 05:45:00", "value": 0.83549}, {"timestamp": "2020-06-03 06:00:00", "value": 0.9098799999999999}, {"timestamp": "2020-06-03 06:15:00", "value": 1.65842}, {"timestamp": "2020-06-03 06:30:00", "value": 1.6986599999999998}, {"timestamp": "2020-06-03 06:45:00", "value": 0.45968000000000003}, {"timestamp": "2020-06-03 07:00:00", "value": 0.44566999999999996}, {"timestamp": "2020-06-03 07:15:00", "value": 0.4448}, {"timestamp": "2020-06-03 07:30:00", "value": 0.46742}, {"timestamp": "2020-06-03 07:45:00", "value": 0.49307}, {"timestamp": "2020-06-03 08:00:00", "value": 0.49991}, {"timestamp": "2020-06-03 08:15:00", "value": 0.36912}, {"timestamp": "2020-06-03 08:30:00", "value": 0.11682999999999999}, {"timestamp": "2020-06-03 08:45:00", "value": 0.02139}, {"timestamp": "2020-06-03 09:00:00", "value": 0.01463}, {"timestamp": "2020-06-03 09:15:00", "value": 0.01546}, {"timestamp": "2020-06-03 09:30:00", "value": 0.021509999999999998}, {"timestamp": "2020-06-03 09:45:00", "value": 0.09881000000000001}, {"timestamp": "2020-06-03 10:00:00", "value": 0.16751}, {"timestamp": "2020-06-03 10:15:00", "value": 0.30156}, {"timestamp": "2020-06-03 10:30:00", "value": 0.5623}, {"timestamp": "2020-06-03 10:45:00", "value": 0.72665}, {"timestamp": "2020-06-03 11:00:00", "value": 0.92096}, {"timestamp": "2020-06-03 11:15:00", "value": 0.80633}, {"timestamp": "2020-06-03 11:30:00", "value": 0.83905}, {"timestamp": "2020-06-03 11:45:00", "value": 0.64593}, {"timestamp": "2020-06-03 12:00:00", "value": 0.53394}, {"timestamp": "2020-06-03 12:15:00", "value": 0.48118999999999995}, {"timestamp": "2020-06-03 12:30:00", "value": 0.46406000000000003}, {"timestamp": "2020-06-03 12:45:00", "value": 0.46321}, {"timestamp": "2020-06-03 13:00:00", "value": 0.45987}, {"timestamp": "2020-06-03 13:15:00", "value": 0.45261999999999997}, {"timestamp": "2020-06-03 13:30:00", "value": 0.42203999999999997}, {"timestamp": "2020-06-03 13:45:00", "value": 0.049760000000000006}, {"timestamp": "2020-06-03 14:00:00", "value": 0.00621}, {"timestamp": "2020-06-03 14:15:00", "value": 0.01384}, {"timestamp": "2020-06-03 14:30:00", "value": 0.1532}, {"timestamp": "2020-06-03 14:45:00", "value": 0.46388999999999997}, {"timestamp": "2020-06-03 15:00:00", "value": 0.57026}, {"timestamp": "2020-06-03 15:15:00", "value": 0.35225}, {"timestamp": "2020-06-03 15:30:00", "value": 0.18104}, {"timestamp": "2020-06-03 15:45:00", "value": 0.04949}, {"timestamp": "2020-06-03 16:00:00", "value": 0.038169999999999996}, {"timestamp": "2020-06-03 16:15:00", "value": 0.21641}, {"timestamp": "2020-06-03 16:30:00", "value": 0.42672}, {"timestamp": "2020-06-03 16:45:00", "value": 0.03367}, {"timestamp": "2020-06-03 17:00:00", "value": 0.0027300000000000002}, {"timestamp": "2020-06-03 17:15:00", "value": 0.00107}, {"timestamp": "2020-06-03 17:30:00", "value": 0.0006}, {"timestamp": "2020-06-03 17:45:00", "value": 0.00093}, {"timestamp": "2020-06-03 18:00:00", "value": 0.00277}, {"timestamp": "2020-06-03 18:15:00", "value": 0.005529999999999999}, {"timestamp": "2020-06-03 18:30:00", "value": 0.00458}, {"timestamp": "2020-06-03 18:45:00", "value": 0.0058}, {"timestamp": "2020-06-03 19:00:00", "value": 0.006840000000000001}, {"timestamp": "2020-06-03 19:15:00", "value": 0.00594}, {"timestamp": "2020-06-03 19:30:00", "value": 0.00838}, {"timestamp": "2020-06-03 19:45:00", "value": 0.00461}, {"timestamp": "2020-06-03 20:00:00", "value": 0.00232}, {"timestamp": "2020-06-03 20:15:00", "value": 0.00112}, {"timestamp": "2020-06-03 20:30:00", "value": 0.0006799999999999999}, {"timestamp": "2020-06-03 20:45:00", "value": 0.00212}, {"timestamp": "2020-06-03 21:00:00", "value": 0.044360000000000004}, {"timestamp": "2020-06-03 21:15:00", "value": 0.05173}, {"timestamp": "2020-06-03 21:30:00", "value": 0.00811}, {"timestamp": "2020-06-03 21:45:00", "value": 2.78274}, {"timestamp": "2020-06-03 22:00:00", "value": 2.3056900000000002}, {"timestamp": "2020-06-03 22:15:00", "value": 0.011290000000000001}, {"timestamp": "2020-06-03 22:30:00", "value": 0.22078000000000003}, {"timestamp": "2020-06-03 22:45:00", "value": 0.10907}, {"timestamp": "2020-06-03 23:00:00", "value": 0.04721}, {"timestamp": "2020-06-04 00:00:00", "value": 1.84307}, {"timestamp": "2020-06-04 00:15:00", "value": 1.90777}, {"timestamp": "2020-06-04 00:30:00", "value": 1.92413}, {"timestamp": "2020-06-04 00:45:00", "value": 1.8986599999999998}, {"timestamp": "2020-06-04 01:00:00", "value": 1.84307}, {"timestamp": "2020-06-04 01:15:00", "value": 1.6991}, {"timestamp": "2020-06-04 01:30:00", "value": 1.07237}, {"timestamp": "2020-06-04 01:45:00", "value": 0.9739700000000001}, {"timestamp": "2020-06-04 02:00:00", "value": 0.94486}, {"timestamp": "2020-06-04 02:15:00", "value": 0.9602200000000001}, {"timestamp": "2020-06-04 02:30:00", "value": 0.9862700000000001}, {"timestamp": "2020-06-04 02:45:00", "value": 1.4480600000000001}, {"timestamp": "2020-06-04 03:00:00", "value": 1.6445}, {"timestamp": "2020-06-04 03:15:00", "value": 2.0232900000000003}], "granularity": "minutely", "customInterval": 15, "sensitivity": 80, "boundaryVersion": "V3", "needDetectorId": true}, "response": {"isAnomaly": true, "expectedValue": 0.14069885714285713}, "type": "last"} -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case30_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"series": [{"timestamp": "2020-06-03 04:45:00", "value": 0.73375}, {"timestamp": "2020-06-03 05:00:00", "value": 0.60654}, {"timestamp": "2020-06-03 05:15:00", "value": 0.62047}, {"timestamp": "2020-06-03 05:30:00", "value": 0.72806}, {"timestamp": "2020-06-03 05:45:00", "value": 0.83549}, {"timestamp": "2020-06-03 06:00:00", "value": 0.9098799999999999}, {"timestamp": "2020-06-03 06:15:00", "value": 1.65842}, {"timestamp": "2020-06-03 06:30:00", "value": 1.6986599999999998}, {"timestamp": "2020-06-03 06:45:00", "value": 0.45968000000000003}, {"timestamp": "2020-06-03 07:00:00", "value": 0.44566999999999996}, {"timestamp": "2020-06-03 07:15:00", "value": 0.4448}, {"timestamp": "2020-06-03 07:30:00", "value": 0.46742}, {"timestamp": "2020-06-03 07:45:00", "value": 0.49307}, {"timestamp": "2020-06-03 08:00:00", "value": 0.49991}, {"timestamp": "2020-06-03 08:15:00", "value": 0.36912}, {"timestamp": "2020-06-03 08:30:00", "value": 0.11682999999999999}, {"timestamp": "2020-06-03 08:45:00", "value": 0.02139}, {"timestamp": "2020-06-03 09:00:00", "value": 0.01463}, {"timestamp": "2020-06-03 09:15:00", "value": 0.01546}, {"timestamp": "2020-06-03 09:30:00", "value": 0.021509999999999998}, {"timestamp": "2020-06-03 09:45:00", "value": 0.09881000000000001}, {"timestamp": "2020-06-03 10:00:00", "value": 0.16751}, {"timestamp": "2020-06-03 10:15:00", "value": 0.30156}, {"timestamp": "2020-06-03 10:30:00", "value": 0.5623}, {"timestamp": "2020-06-03 10:45:00", "value": 0.72665}, {"timestamp": "2020-06-03 11:00:00", "value": 0.92096}, {"timestamp": "2020-06-03 11:15:00", "value": 0.80633}, {"timestamp": "2020-06-03 11:30:00", "value": 0.83905}, {"timestamp": "2020-06-03 11:45:00", "value": 0.64593}, {"timestamp": "2020-06-03 12:00:00", "value": 0.53394}, {"timestamp": "2020-06-03 12:15:00", "value": 0.48118999999999995}, {"timestamp": "2020-06-03 12:30:00", "value": 0.46406000000000003}, {"timestamp": "2020-06-03 12:45:00", "value": 0.46321}, {"timestamp": "2020-06-03 13:00:00", "value": 0.45987}, {"timestamp": "2020-06-03 13:15:00", "value": 0.45261999999999997}, {"timestamp": "2020-06-03 13:30:00", "value": 0.42203999999999997}, {"timestamp": "2020-06-03 13:45:00", "value": 0.049760000000000006}, {"timestamp": "2020-06-03 14:00:00", "value": 0.00621}, {"timestamp": "2020-06-03 14:15:00", "value": 0.01384}, {"timestamp": "2020-06-03 14:30:00", "value": 0.1532}, {"timestamp": "2020-06-03 14:45:00", "value": 0.46388999999999997}, {"timestamp": "2020-06-03 15:00:00", "value": 0.57026}, {"timestamp": "2020-06-03 15:15:00", "value": 0.35225}, {"timestamp": "2020-06-03 15:30:00", "value": 0.18104}, {"timestamp": "2020-06-03 15:45:00", "value": 0.04949}, {"timestamp": "2020-06-03 16:00:00", "value": 0.038169999999999996}, {"timestamp": "2020-06-03 16:15:00", "value": 0.21641}, {"timestamp": "2020-06-03 16:30:00", "value": 0.42672}, {"timestamp": "2020-06-03 16:45:00", "value": 0.03367}, {"timestamp": "2020-06-03 17:00:00", "value": 0.0027300000000000002}, {"timestamp": "2020-06-03 17:15:00", "value": 0.00107}, {"timestamp": "2020-06-03 17:30:00", "value": 0.0006}, {"timestamp": "2020-06-03 17:45:00", "value": 0.00093}, {"timestamp": "2020-06-03 18:00:00", "value": 0.00277}, {"timestamp": "2020-06-03 18:15:00", "value": 0.005529999999999999}, {"timestamp": "2020-06-03 18:30:00", "value": 0.00458}, {"timestamp": "2020-06-03 18:45:00", "value": 0.0058}, {"timestamp": "2020-06-03 19:00:00", "value": 0.006840000000000001}, {"timestamp": "2020-06-03 19:15:00", "value": 0.00594}, {"timestamp": "2020-06-03 19:30:00", "value": 0.00838}, {"timestamp": "2020-06-03 19:45:00", "value": 0.00461}, {"timestamp": "2020-06-03 20:00:00", "value": 0.00232}, {"timestamp": "2020-06-03 20:15:00", "value": 0.00112}, {"timestamp": "2020-06-03 20:30:00", "value": 0.0006799999999999999}, {"timestamp": "2020-06-03 20:45:00", "value": 0.00212}, {"timestamp": "2020-06-03 21:00:00", "value": 0.044360000000000004}, {"timestamp": "2020-06-03 21:15:00", "value": 0.05173}, {"timestamp": "2020-06-03 21:30:00", "value": 0.00811}, {"timestamp": "2020-06-03 21:45:00", "value": 2.78274}, {"timestamp": "2020-06-03 22:00:00", "value": 2.3056900000000002}, {"timestamp": "2020-06-03 22:15:00", "value": 0.011290000000000001}, {"timestamp": "2020-06-03 22:30:00", "value": 0.22078000000000003}, {"timestamp": "2020-06-03 22:45:00", "value": 0.10907}, {"timestamp": "2020-06-03 23:00:00", "value": 0.04721}, {"timestamp": "2020-06-04 00:00:00", "value": 1.84307}, {"timestamp": "2020-06-04 00:15:00", "value": 1.90777}, {"timestamp": "2020-06-04 00:30:00", "value": 1.92413}, {"timestamp": "2020-06-04 00:45:00", "value": 1.8986599999999998}, {"timestamp": "2020-06-04 01:00:00", "value": 1.84307}, {"timestamp": "2020-06-04 01:15:00", "value": 1.6991}, {"timestamp": "2020-06-04 01:30:00", "value": 1.07237}, {"timestamp": "2020-06-04 01:45:00", "value": 0.9739700000000001}, {"timestamp": "2020-06-04 02:00:00", "value": 0.94486}, {"timestamp": "2020-06-04 02:15:00", "value": 0.9602200000000001}, {"timestamp": "2020-06-04 02:30:00", "value": 0.9862700000000001}, {"timestamp": "2020-06-04 02:45:00", "value": 1.4480600000000001}, {"timestamp": "2020-06-04 03:00:00", "value": 1.6445}, {"timestamp": "2020-06-04 03:15:00", "value": 2.0232900000000003}, {"timestamp": "2020-06-04 03:30:00", "value": 2.1697}, {"timestamp": "2020-06-04 03:45:00", "value": 1.37506}, {"timestamp": "2020-06-04 04:00:00", "value": 0.6976600000000001}, {"timestamp": "2020-06-04 04:15:00", "value": 0.8043600000000001}, {"timestamp": "2020-06-04 04:30:00", "value": 1.49682}, {"timestamp": "2020-06-04 04:45:00", "value": 2.03979}], "granularity": "minutely", "customInterval": 15, "sensitivity": 80, "boundaryVersion": "V3", "needDetectorId": true}, "response": {"isAnomaly": true, "expectedValue": 0.10347121212121213}, "type": "last"} -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/resource/error_message.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | InvalidCustomInterval = "The 'customInterval' field must be an integer > 0." 6 | InvalidJsonFormat = "Invalid Json Format, input data is Empty." 7 | # add this line 8 | InvalidInputFormat = "Invalid Input Format, input data is Empty." 9 | 10 | RequiredSeries = "The 'series' field is required in request." 11 | RequiredGranularity = "The 'granularity' field is required in request." 12 | InvalidGranularity = "The 'granularity' field can only be one of the following: {0}." 13 | InvalidPeriod = "The 'period' field must be an integer >= 0." 14 | InvalidImputeMode = "The 'imputeMode' field is required and should be one of the following: {0}." 15 | InvalidImputeValue = "The 'imputeFixedValue' field must be a number." 16 | InsufficientPoints = "The 'series' field must have more than 2 periods points" 17 | InvalidAlpha = "The 'alpha' field must > 0." 18 | InvalidAnomalyRatio = "The 'maxAnomalyRatio' field must be less than 50% of the series points (0 < " \ 19 | "maxAnomalyRatio < 0.5)." 20 | InvalidSensitivity = "The 'sensitivity' field must be an integer between 0 and 99." 21 | InvalidThreshold = "The 'threshold' field must be a float." 22 | InvalidSeries = "The 'series' field cannot be empty." 23 | InvalidSeriesValue = "The 'series' field cannot have none values." 24 | InvalidSeriesType = "The 'series' field must be array/list type." 25 | NotEnoughPoints = "The 'series' field must have at least {0} points." 26 | TooManyPoints = "The 'series' field cannot have more than 8640 points." 27 | NotEnoughPointsForSeasonalData = "Ratio of missing points should be less than 50% for seasonal timeseries. "\ 28 | "There should be at least {0} points, but got {1} points." 29 | InvalidSeriesFormat = "'timestamp' or 'value' is malformed in 'series' Field." 30 | InvalidSeriesOrder = "The 'series' field must be sorted by timestamp in ascending order." 31 | DuplicateSeriesTimestamp = "The 'series' field cannot have duplicated timestamp." 32 | InvalidSeriesTimestamp = "The 'timestamp' at index {0} is invalid in {1} granularity with {2} gran as interval." 33 | InvalidChangePointThreshold = "The 'threshold' field must be an float between 0.01 and 0.99." 34 | InvalidStableTrendWindow = "The 'stableTrendWindow' field must be an integer >= 3 and < half of series' length." 35 | RequiredSeriesValues = "The 'seriesValues' field is required in request." 36 | InvalidSeriesValues = "The 'seriesValues' field must have at least 12 values and at most 8640 values." 37 | InvalidTrendType = "The 'TrendType' field can only be one of the following: {0}." 38 | InvalidPeriodThresh = "The 'periodThresh' field must be an float between 0.01 and 0.99." 39 | InvalidMinVariance = "The 'minVariance' field must be an float between 0.01 and 0.99." 40 | CustomSupportRequired = "The series data can not be processed due to unexpected exception, " \ 41 | "please contact Anomaly Detector Team for further support." 42 | ValueOverflow = "The magnitude of data in 'value' in 'series' filed is too large." 43 | InvalidBoundaryVersion = "The 'boundaryVersion' field can only be one of the following: {0}." 44 | InvalidDetectorName = "The 'name' field is malformed in 'detector' in request." 45 | MissingDetectorName = "The 'name' field is missing in 'detector' in request." 46 | InvalidDetectorParameters = "The 'parameters' field is malformed in request." 47 | MissingDetectorParameters = "The 'parameters' field is missing in 'detector' in request." 48 | InvalidDetectorNameWithParameters = "Illegal detector name '{0}' and parameter combination." 49 | 50 | RequiredPatternDetectType = "The 'detectType' field is required in pattern detect request." 51 | RequiredPatternSeries = "The 'series' field is required in pattern detect request." 52 | RequiredPatternGranularity = "The 'granularity' field is required in pattern detect request" 53 | RequiredPatternCustomInterval = "The 'customInterval' field is required in pattern detect request" 54 | RequiredPatternSeriesId = "The 'seriesId' field is required in pattern detect request" 55 | RequiredPatternTimestamp = "The 'timestamp' field is required in pattern detect request" 56 | RequiredPatternAnomalyRatio = "The 'anomalyRatio' field is required in pattern detect request" 57 | RequiredPatternDeltaRatio = "The 'deltaRatio' field is required in pattern detect request" 58 | RequiredPatternValueRange = "The 'valueRange' field is required in pattern detect request" 59 | RequiredPatternOutOfExpected = "The 'outOfExpected' field is required in pattern detect request" 60 | InvalidPatternDetectType = "The 'detectType' field must be a string from {0}" 61 | InvalidPatternSeriesId = "The 'seriesId' field must be a string" 62 | InvalidPatternSeries = "The 'series' field should have at least 2 data points and at most 8640 data points." 63 | InvalidPatternTimestamp = "The 'timestamp' field must be a list of timestamps, with 1 element in last mode and more than 1 in entire mode, timestamp length: {0}, series length: {1}, detectType: {2}" 64 | InvalidPatternAnomalyRatio = "The 'anomalyRatio' field must be a float between 0 and 1" 65 | InvalidPatternDeltaRatio = "The 'deltaRatio' field must be a float >= 0" 66 | InvalidPatternValueRange = "The 'valueRange' field must be a float >= 0" 67 | InvalidPatternOutOfExpected = "The 'outOfExpected' field must be a list of bool, with 1 element in last mode and more than 1 in entire mode, outOfExpected length: {0}, series length: {1}, detectType: {2}" 68 | InvalidPatternSeriesFormat = "'timestamp', 'value', 'ExpectedValue', 'SpectrumPeriod' or 'IsAnomaly' is missing or malformed in 'series' Field." 69 | InvalidPatternGranularity = "The 'granularity' field is {0}, expected: {1}." 70 | InvalidPatternCustomInterval = "The 'customInterval' field must be None or an int > 0." 71 | NotfoundPatternTimestamp = "The pattern timestamp is not found in series" 72 | -------------------------------------------------------------------------------- /src/anomaly_detector/univariate/model/seasonal_series.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # --------------------------------------------------------- 4 | 5 | import math 6 | import numpy as np 7 | import pandas as pd 8 | 9 | from anomaly_detector.univariate.model.detect import detect 10 | from anomaly_detector.univariate.util import Value, Direction, ExpectedValue, IsAnomaly, IsNegativeAnomaly, IsPositiveAnomaly, Trend, \ 11 | get_verified_majority_value, ModelType, normalize 12 | from anomaly_detector.univariate.util import stl, stl_adjust_trend, interp, de_outlier_stl, MAPE_LB, MAPE_UB 13 | from anomaly_detector.univariate.detectors import ESD 14 | from anomaly_detector.univariate.detectors import ZScoreDetector 15 | 16 | 17 | def seasonal_series_detection(series, period, max_anomaly_ratio, alpha, adjust_trend=False, need_trend=False, last_value=None): 18 | return detect_ts(series, period, max_anomaly_ratio=max_anomaly_ratio, alpha=alpha, adjust_trend=adjust_trend, 19 | need_trend=need_trend, last_value=last_value) 20 | 21 | 22 | def detect_ts(series, num_obs_per_period, max_anomaly_ratio=0.10, alpha=0.05, adjust_trend=False, need_trend=False, last_value=None): 23 | if max_anomaly_ratio > 0.49: 24 | length = len(series) 25 | raise ValueError( 26 | ("max_anomaly_ratio must be less than 50% of " 27 | "the data points (max_anomaly_ratio =%f data_points =%s).") 28 | % (round(max_anomaly_ratio * length, 0), length)) 29 | 30 | num_obs = len(series) 31 | 32 | clamp = (1 / float(num_obs)) 33 | if max_anomaly_ratio < clamp: 34 | max_anomaly_ratio = clamp 35 | 36 | if num_obs_per_period is None: 37 | raise ValueError("must supply period length for time series decomposition") 38 | 39 | if num_obs < num_obs_per_period * 2 + 1: 40 | raise ValueError("Anomaly detection needs at least 2 periods worth of data") 41 | 42 | input_series = series.copy() 43 | stl_func = stl_adjust_trend if adjust_trend else stl 44 | decompose_result = de_outlier_stl(input_series, stl_func=stl_func, period=num_obs_per_period, log_transform=False) 45 | 46 | mape = np.mean(np.abs(decompose_result['remainder'] / input_series)) 47 | 48 | if mape > MAPE_UB: 49 | decompose_result_log = de_outlier_stl(input_series, stl_func=stl_func, period=num_obs_per_period, 50 | log_transform=True) 51 | mape_log = np.mean(np.abs(decompose_result_log['remainder'] / input_series)) 52 | if mape_log < MAPE_LB: 53 | decompose_result = decompose_result_log 54 | 55 | decompose_trend = decompose_result['trend'] 56 | decompose_season = decompose_result['seasonal'] 57 | 58 | de_seasoned_value = series - decompose_season 59 | remainder = de_seasoned_value - decompose_trend 60 | 61 | directions = [Direction.upper_tail, Direction.lower_tail] 62 | data = pd.Series(normalize(de_seasoned_value)) 63 | anomalies, model_id = detect_anomaly(data=data, alpha=alpha, ratio=max_anomaly_ratio, 64 | directions=directions, 65 | remainder_values=pd.Series(normalize(remainder)), 66 | last_value=last_value) 67 | 68 | if len(anomalies) != 0: 69 | decompose_trend[anomalies.index] = np.nan 70 | nan_window = num_obs_per_period // 2 71 | if np.sum(anomalies.index >= len(data) - nan_window) >= 0.5 * nan_window: 72 | decompose_trend[-nan_window:] = np.nan 73 | decompose_trend = interp(decompose_trend) 74 | 75 | p = { 76 | ExpectedValue: decompose_trend + decompose_season, 77 | Value: series 78 | } 79 | 80 | if need_trend: 81 | p[Trend] = decompose_trend 82 | 83 | expected_values = pd.DataFrame(p) 84 | if len(anomalies) != 0: 85 | anomalies = anomalies.join(expected_values, how='inner') 86 | anomalies[IsPositiveAnomaly], anomalies[IsNegativeAnomaly] = \ 87 | zip(*anomalies.apply(lambda x: modify_anomaly_status(x), axis=1)) 88 | 89 | anomalies = anomalies[[IsAnomaly, IsPositiveAnomaly, 90 | IsNegativeAnomaly]] 91 | 92 | merged_result = expected_values.join(anomalies, how='left') 93 | merged_result.fillna(False, inplace=True) 94 | 95 | return merged_result, model_id 96 | 97 | 98 | def detect_anomaly(data, alpha, ratio, directions, remainder_values, last_value=None): 99 | sorted_data = data.sort_values(ascending=True) 100 | sorted_reminder = remainder_values.sort_values(ascending=True) 101 | num_obs = len(sorted_data) 102 | max_outliers = min(max(math.ceil(num_obs * ratio), 1), len(data) // 2 - 1) 103 | majority_value = get_verified_majority_value(sorted_data, num_obs) 104 | reminder_majority_value = get_verified_majority_value(sorted_reminder, num_obs) 105 | 106 | detectors = [ESD(series=sorted_data, alpha=alpha, max_outliers=max_outliers, majority_value=majority_value), 107 | ZScoreDetector(series=sorted_data, max_outliers=max_outliers), 108 | ESD(series=sorted_reminder, alpha=alpha, max_outliers=max_outliers, 109 | majority_value=reminder_majority_value)] 110 | 111 | model_id = ModelType.AnomalyDetector 112 | 113 | if majority_value is not None or reminder_majority_value is not None: 114 | model_id = ModelType.AnomalyDetectorMad 115 | 116 | return detect(directions=directions, detectors=detectors, max_outliers=max_outliers, num_obs=num_obs, 117 | last_value=last_value), model_id 118 | 119 | 120 | def modify_anomaly_status(x): 121 | if not x[IsAnomaly]: 122 | return False, False 123 | if x[ExpectedValue] > x[Value]: 124 | return False, True 125 | else: 126 | return True, False 127 | -------------------------------------------------------------------------------- /tests/cases/json_dsat_test.cases.dsat_case31_0.json: -------------------------------------------------------------------------------- 1 | {"request": {"granularity": "daily", "maxAnomalyRatio": 0.25, "sensitivity": 95, "series": [{"timestamp": "2022-09-05T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-06T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-07T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-08T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-09T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-10T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-11T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-12T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-13T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-14T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-15T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-16T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-17T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-18T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-19T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-20T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-21T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-22T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-23T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-24T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-25T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-26T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-27T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-28T00:00:00.0000000Z", "value": 0}, {"timestamp": "2022-09-29T00:00:00.0000000Z", "value": 24}, {"timestamp": "2022-09-30T00:00:00.0000000Z", "value": 18}, {"timestamp": "2022-10-01T00:00:00.0000000Z", "value": 19}, {"timestamp": "2022-10-02T00:00:00.0000000Z", "value": 21}, {"timestamp": "2022-10-03T00:00:00.0000000Z", "value": 21}, {"timestamp": "2022-10-04T00:00:00.0000000Z", "value": 22}, {"timestamp": "2022-10-05T00:00:00.0000000Z", "value": 21}, {"timestamp": "2022-10-06T00:00:00.0000000Z", "value": 22}, {"timestamp": "2022-10-07T00:00:00.0000000Z", "value": 23}, {"timestamp": "2022-10-08T00:00:00.0000000Z", "value": 21}, {"timestamp": "2022-10-09T00:00:00.0000000Z", "value": 20}, {"timestamp": "2022-10-10T00:00:00.0000000Z", "value": 20}, {"timestamp": "2022-10-11T00:00:00.0000000Z", "value": 20}, {"timestamp": "2022-10-12T00:00:00.0000000Z", "value": 20}, {"timestamp": "2022-10-13T00:00:00.0000000Z", "value": 21}, {"timestamp": "2022-10-14T00:00:00.0000000Z", "value": 20}, {"timestamp": "2022-10-15T00:00:00.0000000Z", "value": 20}, {"timestamp": "2022-10-16T00:00:00.0000000Z", "value": 20}, {"timestamp": "2022-10-17T00:00:00.0000000Z", "value": 18}, {"timestamp": "2022-10-18T00:00:00.0000000Z", "value": 18}, {"timestamp": "2022-10-19T00:00:00.0000000Z", "value": 18}, {"timestamp": "2022-10-20T00:00:00.0000000Z", "value": 19}, {"timestamp": "2022-10-21T00:00:00.0000000Z", "value": 19}, {"timestamp": "2022-10-22T00:00:00.0000000Z", "value": 19}, {"timestamp": "2022-10-23T00:00:00.0000000Z", "value": 19}, {"timestamp": "2022-10-24T00:00:00.0000000Z", "value": 19}, {"timestamp": "2022-10-25T00:00:00.0000000Z", "value": 18}, {"timestamp": "2022-10-26T00:00:00.0000000Z", "value": 18}, {"timestamp": "2022-10-27T00:00:00.0000000Z", "value": 19}, {"timestamp": "2022-10-28T00:00:00.0000000Z", "value": 18}, {"timestamp": "2022-10-29T00:00:00.0000000Z", "value": 18}, {"timestamp": "2022-10-30T00:00:00.0000000Z", "value": 18}, {"timestamp": "2022-10-31T00:00:00.0000000Z", "value": 18}, {"timestamp": "2022-11-01T00:00:00.0000000Z", "value": 18}, {"timestamp": "2022-11-02T00:00:00.0000000Z", "value": 20}, {"timestamp": "2022-11-03T00:00:00.0000000Z", "value": 20}, {"timestamp": "2022-11-04T00:00:00.0000000Z", "value": 19}, {"timestamp": "2022-11-05T00:00:00.0000000Z", "value": 65}, {"timestamp": "2022-11-06T00:00:00.0000000Z", "value": 65}, {"timestamp": "2022-11-07T00:00:00.0000000Z", "value": 66}, {"timestamp": "2022-11-08T00:00:00.0000000Z", "value": 65}, {"timestamp": "2022-11-09T00:00:00.0000000Z", "value": 65}, {"timestamp": "2022-11-10T00:00:00.0000000Z", "value": 65}, {"timestamp": "2022-11-11T00:00:00.0000000Z", "value": 65}, {"timestamp": "2022-11-12T00:00:00.0000000Z", "value": 65}, {"timestamp": "2022-11-13T00:00:00.0000000Z", "value": 64}, {"timestamp": "2022-11-14T00:00:00.0000000Z", "value": 64}, {"timestamp": "2022-11-15T00:00:00.0000000Z", "value": 65}, {"timestamp": "2022-11-16T00:00:00.0000000Z", "value": 66}, {"timestamp": "2022-11-17T00:00:00.0000000Z", "value": 66}, {"timestamp": "2022-11-18T00:00:00.0000000Z", "value": 66}, {"timestamp": "2022-11-19T00:00:00.0000000Z", "value": 66}, {"timestamp": "2022-11-20T00:00:00.0000000Z", "value": 66}, {"timestamp": "2022-11-21T00:00:00.0000000Z", "value": 66}, {"timestamp": "2022-11-22T00:00:00.0000000Z", "value": 66}, {"timestamp": "2022-11-23T00:00:00.0000000Z", "value": 65}, {"timestamp": "2022-11-24T00:00:00.0000000Z", "value": 64}, {"timestamp": "2022-11-25T00:00:00.0000000Z", "value": 65}, {"timestamp": "2022-11-26T00:00:00.0000000Z", "value": 64}, {"timestamp": "2022-11-27T00:00:00.0000000Z", "value": 65}, {"timestamp": "2022-11-28T00:00:00.0000000Z", "value": 64}, {"timestamp": "2022-11-29T00:00:00.0000000Z", "value": 64}, {"timestamp": "2022-11-30T00:00:00.0000000Z", "value": 65}, {"timestamp": "2022-12-01T00:00:00.0000000Z", "value": 66}, {"timestamp": "2022-12-02T00:00:00.0000000Z", "value": 66}, {"timestamp": "2022-12-03T00:00:00.0000000Z", "value": 65}, {"timestamp": "2022-12-04T00:00:00.0000000Z", "value": 65}, {"timestamp": "2022-12-05T00:00:00.0000000Z", "value": 65}, {"timestamp": "2022-12-06T00:00:00.0000000Z", "value": 66}, {"timestamp": "2022-12-07T00:00:00.0000000Z", "value": 66}, {"timestamp": "2022-12-08T00:00:00.0000000Z", "value": 65}, {"timestamp": "2022-12-09T00:00:00.0000000Z", "value": 66}, {"timestamp": "2022-12-10T00:00:00.0000000Z", "value": 66}, {"timestamp": "2022-12-11T00:00:00.0000000Z", "value": 66}, {"timestamp": "2022-12-12T00:00:00.0000000Z", "value": 67}], "needDetectorId": true}, "response": {"expectedValue": 77.67313778989164, "isAnomaly": true, "isPositiveAnomaly": false, "isNegativeAnomaly": true}, "type": "last"} --------------------------------------------------------------------------------