├── tests ├── tools │ └── __init__.py ├── udfs │ ├── __init__.py │ ├── resources │ │ ├── _config4.yaml │ │ ├── _config2.yaml │ │ ├── rds_config2.yaml │ │ ├── rds_trainer_config_fetcher_conf1.yaml │ │ ├── _config3.yaml │ │ ├── _config.yaml │ │ ├── rds_trainer_config_fetcher_conf.yaml │ │ └── rds_config.yaml │ ├── test_factory.py │ ├── test_pipeline.py │ ├── utility.py │ ├── test_main.py │ ├── test_numaflow.py │ └── test_payloadtx.py ├── blocks │ ├── __init__.py │ └── test_blocks.py ├── config │ ├── __init__.py │ └── test_optdeps.py ├── connectors │ ├── __init__.py │ ├── rds │ │ ├── __init__.py │ │ ├── db │ │ │ ├── __init__.py │ │ │ └── test_factory.py │ │ └── test_rds.py │ ├── utils │ │ ├── __init__.py │ │ ├── aws │ │ │ ├── __init__.py │ │ │ ├── test_db_configurations.py │ │ │ ├── test_sts_client_manager.py │ │ │ └── test_config.py │ │ └── test_enum.py │ ├── test_config.py │ └── test_redis.py ├── models │ ├── __init__.py │ ├── vae │ │ └── __init__.py │ ├── forecast │ │ ├── __init__.py │ │ └── test_naive.py │ ├── autoencoder │ │ ├── __init__.py │ │ └── variants │ │ │ └── __init__.py │ └── threshold │ │ ├── __init__.py │ │ ├── test_std.py │ │ ├── test_median.py │ │ ├── test_static.py │ │ └── test_maha.py ├── registry │ ├── __init__.py │ └── test_serialize.py ├── synthetic │ ├── __init__.py │ ├── test_sparsity.py │ └── test_timeseries.py ├── transforms │ ├── __init__.py │ └── test_postprocess.py ├── __init__.py ├── resources │ ├── rds_db_config.yaml │ ├── config.yaml │ └── data │ │ └── prom_default.csv └── test_backtest.py ├── numalogic ├── models │ ├── __init__.py │ ├── forecast │ │ ├── __init__.py │ │ └── variants │ │ │ └── __init__.py │ ├── vae │ │ ├── __init__.py │ │ ├── variants │ │ │ └── __init__.py │ │ └── base.py │ ├── threshold │ │ ├── __init__.py │ │ ├── _median.py │ │ └── _std.py │ └── autoencoder │ │ ├── __init__.py │ │ ├── variants │ │ └── __init__.py │ │ └── base.py ├── tools │ ├── __init__.py │ ├── aggregators.py │ ├── loss.py │ ├── types.py │ ├── trainer.py │ └── exceptions.py ├── connectors │ ├── utils │ │ ├── __init__.py │ │ ├── aws │ │ │ ├── __init__.py │ │ │ ├── db_configurations.py │ │ │ └── exceptions.py │ │ └── enum.py │ ├── rds │ │ ├── db │ │ │ ├── __init__.py │ │ │ └── factory.py │ │ ├── __init__.py │ │ └── _rds.py │ ├── druid │ │ ├── __init__.py │ │ ├── postaggregator.py │ │ └── aggregators.py │ ├── exceptions.py │ ├── _base.py │ ├── __init__.py │ └── redis.py ├── backtest │ ├── _constants.py │ └── __init__.py ├── udfs │ ├── trainer │ │ └── __init__.py │ ├── _logger.py │ ├── __init__.py │ ├── __main__.py │ ├── payloadtx.py │ └── README.md ├── __init__.py ├── registry │ ├── _serialize.py │ └── __init__.py ├── synthetic │ ├── __init__.py │ └── sparsity.py ├── blocks │ ├── __init__.py │ └── _nn.py ├── _constants.py ├── config │ └── __init__.py ├── transforms │ ├── __init__.py │ └── _postprocess.py └── base.py ├── .github ├── pull_request_template.md ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md └── workflows │ ├── lint.yml │ ├── changelog.yml │ ├── gh-pages.yaml │ ├── pypi.yml │ ├── ci.yml │ ├── coverage.yml │ └── release.yml ├── pytest.ini ├── examples ├── multi_udf │ ├── entry.sh │ ├── .dockerignore │ ├── README.md │ ├── requirements.txt │ ├── starter.py │ ├── src │ │ ├── udf │ │ │ ├── __init__.py │ │ │ ├── postprocess.py │ │ │ ├── preprocess.py │ │ │ ├── threshold.py │ │ │ └── inference.py │ │ ├── __init__.py │ │ ├── factory.py │ │ └── utils.py │ └── Dockerfile └── block_pipeline │ ├── .dockerignore │ ├── requirements.txt │ ├── src │ ├── __init__.py │ ├── utils.py │ ├── inference.py │ └── train.py │ ├── server.py │ ├── Dockerfile │ └── numa-pl.yaml ├── docs ├── assets │ ├── logo.png │ ├── recon.png │ ├── outliers.png │ ├── example-ui.png │ ├── mlflow-ui.png │ ├── train_test.png │ ├── anomaly_score.png │ └── numaproj.svg ├── inference.md ├── threshold.md ├── forecasting.md ├── README.md ├── post-processing.md ├── data-generator.md ├── ml-flow.md └── pre-processing.md ├── .coveragerc ├── CONTRIBUTING.md ├── CODE_OF_CONDUCT.md ├── USERS.md ├── .codecov.yml ├── log.conf ├── .dockerignore ├── .hack └── changelog.sh ├── .pre-commit-config.yaml ├── mkdocs.yml ├── Makefile └── Dockerfile /tests/tools/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/udfs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /numalogic/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /numalogic/tools/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/blocks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/config/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/connectors/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/models/vae/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/registry/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/synthetic/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/transforms/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/connectors/rds/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/models/forecast/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /numalogic/connectors/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /numalogic/models/forecast/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/connectors/rds/db/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/connectors/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/connectors/utils/aws/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/models/autoencoder/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/models/threshold/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /numalogic/connectors/rds/db/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /numalogic/connectors/utils/aws/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/models/autoencoder/variants/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | Explain what this PR does. 2 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | log_cli = true 3 | log_cli_level = INFO 4 | -------------------------------------------------------------------------------- /examples/multi_udf/entry.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -eux 3 | 4 | python starter.py 5 | -------------------------------------------------------------------------------- /docs/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numaproj/numalogic/HEAD/docs/assets/logo.png -------------------------------------------------------------------------------- /docs/assets/recon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numaproj/numalogic/HEAD/docs/assets/recon.png -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | parallel = True 4 | source = numalogic 5 | omit = tests/* 6 | -------------------------------------------------------------------------------- /docs/assets/outliers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numaproj/numalogic/HEAD/docs/assets/outliers.png -------------------------------------------------------------------------------- /docs/assets/example-ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numaproj/numalogic/HEAD/docs/assets/example-ui.png -------------------------------------------------------------------------------- /docs/assets/mlflow-ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numaproj/numalogic/HEAD/docs/assets/mlflow-ui.png -------------------------------------------------------------------------------- /docs/assets/train_test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numaproj/numalogic/HEAD/docs/assets/train_test.png -------------------------------------------------------------------------------- /docs/assets/anomaly_score.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numaproj/numalogic/HEAD/docs/assets/anomaly_score.png -------------------------------------------------------------------------------- /examples/multi_udf/.dockerignore: -------------------------------------------------------------------------------- 1 | .git/ 2 | __pycache__/ 3 | **/__pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | .idea/ 7 | -------------------------------------------------------------------------------- /numalogic/models/vae/__init__.py: -------------------------------------------------------------------------------- 1 | from numalogic.models.vae.variants import Conv1dVAE 2 | 3 | __all__ = ["Conv1dVAE"] 4 | -------------------------------------------------------------------------------- /examples/block_pipeline/.dockerignore: -------------------------------------------------------------------------------- 1 | .git/ 2 | __pycache__/ 3 | **/__pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | .idea/ 7 | -------------------------------------------------------------------------------- /numalogic/connectors/rds/__init__.py: -------------------------------------------------------------------------------- 1 | from numalogic.connectors.rds._rds import RDSFetcher 2 | 3 | __all__ = ["RDSFetcher"] 4 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Please refer to [Contributing](https://github.com/numaproj/numaproj/blob/main/CONTRIBUTING.md) 4 | -------------------------------------------------------------------------------- /numalogic/models/vae/variants/__init__.py: -------------------------------------------------------------------------------- 1 | from numalogic.models.vae.variants.conv import Conv1dVAE 2 | 3 | __all__ = ["Conv1dVAE"] 4 | -------------------------------------------------------------------------------- /examples/multi_udf/README.md: -------------------------------------------------------------------------------- 1 | # Simple Numalogic Pipeline 2 | 3 | 4 | Refer to documentation: https://numalogic.numaproj.io/quick-start/ 5 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | Please refer to [Code of Conduct](https://github.com/numaproj/numaproj/blob/main/CODE_OF_CONDUCT.md) 4 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # These owners will be the default owners for everything in 2 | # the repo. Unless a later match takes precedence 3 | * @ab93 @qhuai @nkoppisetty @s0nicboOm 4 | -------------------------------------------------------------------------------- /USERS.md: -------------------------------------------------------------------------------- 1 | ## Who is using numalogic? 2 | 3 | Please add your company name and initial use case (optional) below. 4 | 5 | 1. [Intuit](https://www.intuit.com/) Anomaly detection using ML models. 6 | -------------------------------------------------------------------------------- /numalogic/connectors/druid/__init__.py: -------------------------------------------------------------------------------- 1 | from numalogic.connectors.druid._druid import DruidFetcher, make_filter_pairs, build_params 2 | 3 | __all__ = ["DruidFetcher", "make_filter_pairs", "build_params"] 4 | -------------------------------------------------------------------------------- /numalogic/models/forecast/variants/__init__.py: -------------------------------------------------------------------------------- 1 | from numalogic.models.forecast.variants.naive import BaselineForecaster, SeasonalNaiveForecaster 2 | 3 | __all__ = ["BaselineForecaster", "SeasonalNaiveForecaster"] 4 | -------------------------------------------------------------------------------- /.codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: 5 | target: auto 6 | threshold: 3% 7 | patch: 8 | default: 9 | target: auto 10 | threshold: 20% 11 | -------------------------------------------------------------------------------- /examples/block_pipeline/requirements.txt: -------------------------------------------------------------------------------- 1 | cachetools>5.2,<6.0 2 | numalogic[redis,numaflow] @ git+https://github.com/numaproj/numalogic.git@main 3 | # ../../../numalogic[redis,numaflow] # for local testing 4 | pytorch-lightning>2.0,< 3.0 5 | -------------------------------------------------------------------------------- /examples/multi_udf/requirements.txt: -------------------------------------------------------------------------------- 1 | cachetools>5.2,<6.0 2 | numalogic[mlflow,numaflow] @ git+https://github.com/numaproj/numalogic.git@main 3 | # ../../../numalogic[mlflow,numaflow] # for local testing 4 | pytorch-lightning>2.0,< 3.0 5 | pynumaflow>0.4,<0.5 6 | -------------------------------------------------------------------------------- /numalogic/backtest/_constants.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Final 3 | 4 | from numalogic._constants import BASE_DIR 5 | 6 | DEFAULT_OUTPUT_DIR: Final[str] = os.path.join(BASE_DIR, ".btoutput") 7 | DEFAULT_SEQUENCE_LEN: Final[int] = 12 8 | DEFAULT_PROM_LOCALHOST: Final[str] = "http://localhost:9090/" 9 | -------------------------------------------------------------------------------- /examples/multi_udf/starter.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from pynumaflow.function import Server 4 | 5 | from src.factory import UDFFactory 6 | 7 | if __name__ == "__main__": 8 | step_handler = UDFFactory.get_handler(sys.argv[1]) 9 | grpc_server = Server(map_handler=step_handler) 10 | grpc_server.start() 11 | -------------------------------------------------------------------------------- /examples/multi_udf/src/udf/__init__.py: -------------------------------------------------------------------------------- 1 | from src.udf.inference import Inference 2 | from src.udf.postprocess import Postprocess 3 | from src.udf.preprocess import Preprocess 4 | from src.udf.train import Trainer 5 | from src.udf.threshold import Threshold 6 | 7 | 8 | __all__ = ["Preprocess", "Inference", "Postprocess", "Trainer", "Threshold"] 9 | -------------------------------------------------------------------------------- /numalogic/udfs/trainer/__init__.py: -------------------------------------------------------------------------------- 1 | from numalogic.udfs.trainer._base import TrainerUDF 2 | from numalogic.udfs.trainer._prom import PromTrainerUDF 3 | from numalogic.udfs.trainer._druid import DruidTrainerUDF 4 | from numalogic.udfs.trainer._rds import RDSTrainerUDF 5 | 6 | __all__ = ["TrainerUDF", "PromTrainerUDF", "DruidTrainerUDF", "RDSTrainerUDF"] 7 | -------------------------------------------------------------------------------- /numalogic/connectors/exceptions.py: -------------------------------------------------------------------------------- 1 | class ConnectorFetcherException(Exception): 2 | """Custom exception class for grouping all Connector Exceptions together.""" 3 | 4 | pass 5 | 6 | 7 | class RDSFetcherConfValidationException(ConnectorFetcherException): 8 | """A custom exception class for handling validation errors in RDSFetcherConf.""" 9 | 10 | pass 11 | -------------------------------------------------------------------------------- /examples/multi_udf/src/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | LOGGER = logging.getLogger(__name__) 4 | LOGGER.setLevel(logging.INFO) 5 | 6 | stream_handler = logging.StreamHandler() 7 | stream_handler.setLevel(logging.INFO) 8 | 9 | formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") 10 | stream_handler.setFormatter(formatter) 11 | 12 | LOGGER.addHandler(stream_handler) 13 | -------------------------------------------------------------------------------- /tests/connectors/test_config.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from numalogic.connectors.exceptions import RDSFetcherConfValidationException 3 | from numalogic.connectors._config import RDSFetcherConf 4 | 5 | 6 | def test_RDSFetcherConf_post_init_exception(): 7 | with pytest.raises(RDSFetcherConfValidationException): 8 | RDSFetcherConf( 9 | dimensions=[], metrics=[], datasource="test", hash_query_type=True, hash_column_name="" 10 | ) 11 | -------------------------------------------------------------------------------- /numalogic/backtest/__init__.py: -------------------------------------------------------------------------------- 1 | from importlib.util import find_spec 2 | from numalogic.backtest._prom import PromBacktester, OutDataFrames 3 | 4 | 5 | def _validate_req_pkgs(): 6 | if (not find_spec("torch")) or (not find_spec("pytorch_lightning")): 7 | raise ModuleNotFoundError( 8 | "Pytorch and/or Pytorch lightning is not installed. Please install them first." 9 | ) 10 | 11 | 12 | _validate_req_pkgs() 13 | 14 | 15 | __all__ = ["PromBacktester", "OutDataFrames"] 16 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | import os 4 | import warnings 5 | 6 | if not sys.warnoptions: 7 | warnings.simplefilter("default", category=UserWarning) 8 | os.environ["PYTHONWARNINGS"] = "default" 9 | 10 | 11 | logger = logging.getLogger(__name__) 12 | stream_handler = logging.StreamHandler() 13 | stream_handler.setLevel(logging.INFO) 14 | 15 | formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") 16 | stream_handler.setFormatter(formatter) 17 | 18 | logger.addHandler(stream_handler) 19 | -------------------------------------------------------------------------------- /tests/resources/rds_db_config.yaml: -------------------------------------------------------------------------------- 1 | 2 | 3 | aws_assume_role_arn: "arn:aws:iam::123456789:role/ml_iam_role" 4 | aws_assume_role_session_name: "ml_pipeline_reader" 5 | endpoint: "ml-rds-aurora-mysql-cluster.us-west-2.rds.amazonaws.com" 6 | port: 3306 7 | database_name: "mldb" 8 | database_username: "ml_user" 9 | database_password: "" 10 | database_connection_timeout: 10 11 | database_type: "mysql" 12 | database_provider: "rds" 13 | aws_region: "us-west-2" 14 | aws_rds_use_iam: True 15 | ssl_enabled: true 16 | ssl: 17 | ca: "/usr/bin/ml_data/us-west-2-bundle.pem" 18 | -------------------------------------------------------------------------------- /examples/block_pipeline/src/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from src.inference import Inference 4 | from src.train import Train 5 | 6 | logging.basicConfig(level=logging.INFO) 7 | LOGGER = logging.getLogger(__name__) 8 | LOGGER.setLevel(logging.INFO) 9 | 10 | stream_handler = logging.StreamHandler() 11 | stream_handler.setLevel(logging.INFO) 12 | 13 | formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") 14 | stream_handler.setFormatter(formatter) 15 | 16 | LOGGER.addHandler(stream_handler) 17 | 18 | 19 | __all__ = ["Inference", "Train"] 20 | -------------------------------------------------------------------------------- /log.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root, pllogger 3 | 4 | [handlers] 5 | keys=consoleHandler 6 | 7 | [formatters] 8 | keys=consoleFormatter 9 | 10 | [logger_root] 11 | level=INFO 12 | handlers=consoleHandler 13 | 14 | [logger_pllogger] 15 | level=ERROR 16 | handlers=consoleHandler 17 | qualname=pytorch_lightning 18 | propagate=0 19 | 20 | [handler_consoleHandler] 21 | class=StreamHandler 22 | level=DEBUG 23 | formatter=consoleFormatter 24 | 25 | [formatter_consoleFormatter] 26 | format=%(asctime)s - %(thread)d - %(levelname)s - %(message)s 27 | class=logging.Formatter 28 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Propose an enhancement for this project 4 | labels: 'enhancement' 5 | --- 6 | # Summary 7 | 8 | What change needs making? 9 | 10 | # Use Cases 11 | 12 | When would you use this? 13 | 14 | --- 15 | 16 | **Message from the maintainers**: 17 | 18 | If you wish to see this enhancement implemented please add a 👍 reaction to this issue! We often sort issues this way to know what to prioritize. 19 | -------------------------------------------------------------------------------- /numalogic/models/threshold/__init__.py: -------------------------------------------------------------------------------- 1 | from numalogic.models.threshold._std import StdDevThreshold 2 | from numalogic.models.threshold._mahalanobis import MahalanobisThreshold, RobustMahalanobisThreshold 3 | from numalogic.models.threshold._static import StaticThreshold, SigmoidThreshold 4 | from numalogic.models.threshold._median import MaxPercentileThreshold 5 | 6 | __all__ = [ 7 | "StdDevThreshold", 8 | "StaticThreshold", 9 | "SigmoidThreshold", 10 | "MahalanobisThreshold", 11 | "RobustMahalanobisThreshold", 12 | "MaxPercentileThreshold", 13 | ] 14 | -------------------------------------------------------------------------------- /examples/block_pipeline/server.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from pynumaflow.function import Server 4 | from src import Inference, Train 5 | 6 | 7 | if __name__ == "__main__": 8 | if len(sys.argv) != 2: 9 | raise ValueError("Please provide a step name") 10 | 11 | step = sys.argv[1] 12 | if step == "inference": 13 | step_handler = Inference() 14 | elif step == "train": 15 | step_handler = Train() 16 | else: 17 | raise ValueError(f"Invalid step provided: {step}") 18 | 19 | grpc_server = Server(map_handler=step_handler) 20 | grpc_server.start() 21 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | tests 2 | .github 3 | .gitignore 4 | .hack 5 | .ruff_cache 6 | docs 7 | examples 8 | .coveragerc 9 | .pre-commit-config.yaml 10 | CHANGELOG.md 11 | CODE_OF_CONDUCT.md 12 | CONTRIBUTING.md 13 | LICENSE 14 | mkdocs.yml 15 | README.md 16 | USERS.md 17 | .idea 18 | .git 19 | .DS_Store 20 | .codecov.yml 21 | .pytest_cache 22 | **/__pycache__/ 23 | **/*.py[cod] 24 | .Python 25 | env/ 26 | build/ 27 | develop-eggs/ 28 | dist/ 29 | downloads/ 30 | eggs/ 31 | lib/ 32 | lib64/ 33 | parts/ 34 | sdist/ 35 | var/ 36 | *.egg-info/ 37 | .installed.cfg 38 | *.egg 39 | site 40 | 41 | # Virtual environment 42 | .env 43 | **/.venv/ 44 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | push: 5 | branches: [ "main", "release/*" ] 6 | pull_request: 7 | branches: [ "main", "release/*" ] 8 | 9 | jobs: 10 | black: 11 | name: Black format 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v4 15 | - uses: psf/black@stable 16 | with: 17 | options: "--check --verbose" 18 | version: "~= 23.3" 19 | 20 | ruff: 21 | runs-on: ubuntu-latest 22 | steps: 23 | - uses: actions/checkout@v3 24 | - uses: chartboost/ruff-action@v1 25 | with: 26 | version: "0.0.275" 27 | -------------------------------------------------------------------------------- /tests/connectors/test_redis.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest.mock import patch 3 | 4 | from redis.sentinel import Sentinel 5 | import fakeredis 6 | 7 | from numalogic.connectors.redis import get_redis_client 8 | 9 | server = fakeredis.FakeServer() 10 | fake_redis_client = fakeredis.FakeStrictRedis(server=server, decode_responses=True) 11 | 12 | 13 | class TestRedisClient(unittest.TestCase): 14 | def test_sentinel_redis_client(self): 15 | with patch.object(Sentinel, "master_for", return_value=fake_redis_client): 16 | r = get_redis_client("hostname", 6379, "pass", "mymaster") 17 | self.assertTrue(r.ping()) 18 | -------------------------------------------------------------------------------- /tests/resources/config.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | name: "SparseVanillaAE" 3 | conf: 4 | seq_len: 20 5 | n_features: 1 6 | encoder_layersizes: 7 | - 16 8 | - 8 9 | decoder_layersizes: 10 | - 8 11 | - 16 12 | dropout_p: 0.25 13 | trainer: 14 | pltrainer_conf: 15 | max_epochs: 40 16 | preprocess: 17 | - name: "LogTransformer" 18 | stateful: false 19 | conf: 20 | add_factor: 3 21 | - name: "StandardScaler" 22 | conf: 23 | with_mean: False 24 | threshold: 25 | name: "StdDevThreshold" 26 | postprocess: 27 | name: "TanhNorm" 28 | stateful: false 29 | conf: 30 | scale_factor: 5 31 | -------------------------------------------------------------------------------- /.hack/changelog.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | set -eu 3 | 4 | echo '# Changelog' 5 | echo 6 | 7 | tag= 8 | git tag -l 'v*' | sed 's/-rc/~/' | sort -rV | sed 's/~/-rc/' | while read last; do 9 | if [ "$tag" != "" ]; then 10 | echo "## $(git for-each-ref --format='%(refname:strip=2) (%(creatordate:short))' refs/tags/${tag})" 11 | echo 12 | git_log='git --no-pager log --no-merges --invert-grep --grep=^\(build\|chore\|ci\|docs\|test\):' 13 | $git_log --format=' * [%h](https://github.com/numaproj/numalogic/commit/%H) %s' $last..$tag 14 | echo 15 | echo "### Contributors" 16 | echo 17 | $git_log --format=' * %an' $last..$tag | sort -u 18 | echo 19 | fi 20 | tag=$last 21 | done 22 | -------------------------------------------------------------------------------- /docs/inference.md: -------------------------------------------------------------------------------- 1 | # Inference 2 | 3 | Now, once we have the model trained using one of the ML algorthims, we can predict the anomalies in the test data. 4 | 5 | This can be a streaming or a batched data. 6 | 7 | ```python 8 | X_test = scaler.transform(outlier_test_df.to_numpy()) 9 | 10 | # predict method returns the reconstruction produced by the AE 11 | test_recon = model.predict(X_test) 12 | 13 | # score method returns the anomaly score, calculated using thresholds. 14 | # A number less than 1 indicates an inlier, and greater than 1 indicates an outlier. 15 | test_anomaly_score = model.score(X_test) 16 | ``` 17 | 18 | ![Reconstruction](./assets/recon.png) 19 | ![Anomaly Score](./assets/anomaly_score.png) 20 | -------------------------------------------------------------------------------- /.github/workflows/changelog.yml: -------------------------------------------------------------------------------- 1 | name: Changelog 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | jobs: 8 | generate_changelog: 9 | if: github.repository == 'numaproj/numalogic' 10 | runs-on: ubuntu-latest 11 | name: Generate changelog 12 | steps: 13 | - uses: actions/checkout@v4 14 | with: 15 | ref: main 16 | fetch-depth: 0 17 | - run: git fetch --prune --prune-tags 18 | - run: git tag -l 'v*' 19 | - run: ./.hack/changelog.sh > CHANGELOG.md 20 | - uses: peter-evans/create-pull-request@v3 21 | with: 22 | title: 'docs: updated CHANGELOG.md' 23 | commit-message: 'docs: updated CHANGELOG.md' 24 | signoff: true 25 | -------------------------------------------------------------------------------- /examples/multi_udf/src/factory.py: -------------------------------------------------------------------------------- 1 | from typing import ClassVar 2 | 3 | from src.udf import Preprocess, Inference, Postprocess, Trainer, Threshold 4 | from numalogic.udfs import NumalogicUDF 5 | 6 | 7 | class UDFFactory: 8 | """Factory class to return the handler for the given step.""" 9 | 10 | _UDF_MAP: ClassVar[dict] = { 11 | "preprocess": Preprocess, 12 | "inference": Inference, 13 | "postprocess": Postprocess, 14 | "train": Trainer, 15 | "threshold": Threshold, 16 | } 17 | 18 | @classmethod 19 | def get_handler(cls, step: str) -> NumalogicUDF: 20 | """Return the handler for the given step.""" 21 | udf_cls = cls._UDF_MAP[step] 22 | return udf_cls() 23 | -------------------------------------------------------------------------------- /numalogic/tools/aggregators.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Callable 2 | 3 | import numpy as np 4 | import numpy.typing as npt 5 | 6 | from numalogic.transforms import expmov_avg_aggregator 7 | 8 | 9 | def aggregate_window( 10 | y: npt.NDArray[float], agg_func: Callable = expmov_avg_aggregator, **agg_func_kw 11 | ) -> npt.NDArray[float]: 12 | """Aggregate over window/sequence length.""" 13 | return np.apply_along_axis(func1d=agg_func, axis=0, arr=y, **agg_func_kw).reshape(-1) 14 | 15 | 16 | def aggregate_features( 17 | y: npt.NDArray[float], agg_func: Callable = np.mean, **agg_func_kw 18 | ) -> npt.NDArray[float]: 19 | """Aggregate over features.""" 20 | return agg_func(y, axis=1, keepdims=True, **agg_func_kw) 21 | -------------------------------------------------------------------------------- /numalogic/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | 13 | import logging 14 | 15 | 16 | LOGGER = logging.getLogger(__name__) 17 | LOGGER.addHandler(logging.NullHandler()) 18 | -------------------------------------------------------------------------------- /numalogic/models/autoencoder/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | 13 | from numalogic.tools.trainer import TimeseriesTrainer 14 | 15 | __all__ = ["TimeseriesTrainer"] 16 | -------------------------------------------------------------------------------- /tests/connectors/rds/db/test_factory.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from numalogic.connectors.rds.db.factory import RdsFactory 3 | from numalogic.connectors.rds.db.mysql_fetcher import MysqlFetcher 4 | from numalogic.connectors.utils.aws.exceptions import UnRecognizedDatabaseTypeException 5 | 6 | 7 | def test_get_db_handler_with_supported_db_type(): 8 | # Arrange 9 | db_type = "mysql" 10 | # Act 11 | result = RdsFactory.get_db_handler(db_type) 12 | 13 | # Assert 14 | assert result == MysqlFetcher 15 | 16 | 17 | def test_get_db_handler_with_unsupported_db_type(): 18 | # Arrange 19 | db_type = "not_supported" 20 | 21 | # Act and Assert 22 | with pytest.raises(UnRecognizedDatabaseTypeException): 23 | RdsFactory.get_db_handler(db_type) 24 | -------------------------------------------------------------------------------- /numalogic/registry/_serialize.py: -------------------------------------------------------------------------------- 1 | import io 2 | import pickle 3 | from typing import Union 4 | 5 | import torch 6 | 7 | from numalogic.tools.types import artifact_t, state_dict_t 8 | 9 | 10 | def dumps( 11 | deserialized_object: Union[artifact_t, state_dict_t], 12 | pickle_protocol: int = pickle.HIGHEST_PROTOCOL, 13 | ) -> bytes: 14 | buffer = io.BytesIO() 15 | torch.save(deserialized_object, buffer, pickle_protocol=pickle_protocol) 16 | serialized_obj = buffer.getvalue() 17 | buffer.close() 18 | return serialized_obj 19 | 20 | 21 | def loads(serialized_object: bytes) -> Union[artifact_t, state_dict_t]: 22 | buffer = io.BytesIO(serialized_object) 23 | deserialized_obj = torch.load(buffer) 24 | buffer.close() 25 | return deserialized_obj 26 | -------------------------------------------------------------------------------- /numalogic/connectors/druid/postaggregator.py: -------------------------------------------------------------------------------- 1 | from pydruid.utils.postaggregator import Field 2 | from pydruid.utils.postaggregator import Postaggregator 3 | 4 | 5 | class QuantilesDoublesSketchToQuantile(Postaggregator): 6 | """Class for building QuantilesDoublesSketchToQuantile post aggregator.""" 7 | 8 | def __init__(self, field: Field, fraction: float, output_name=None): 9 | name = output_name or "quantilesDoublesSketchToQuantile" 10 | 11 | super().__init__(None, None, name) 12 | self.field = field 13 | self.fraction = fraction 14 | self.post_aggregator = { 15 | "type": "quantilesDoublesSketchToQuantile", 16 | "name": name, 17 | "field": self.field.post_aggregator, 18 | "fraction": self.fraction, 19 | } 20 | -------------------------------------------------------------------------------- /numalogic/connectors/_base.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta, abstractmethod 2 | 3 | import pandas as pd 4 | 5 | 6 | class DataFetcher(metaclass=ABCMeta): 7 | __slots__ = ("url",) 8 | 9 | def __init__(self, url: str): 10 | self.url = url 11 | 12 | @abstractmethod 13 | def fetch(self, *args, **kwargs) -> pd.DataFrame: 14 | pass 15 | 16 | @abstractmethod 17 | def raw_fetch(self, *args, **kwargs) -> pd.DataFrame: 18 | pass 19 | 20 | 21 | class AsyncDataFetcher(metaclass=ABCMeta): 22 | __slots__ = ("url",) 23 | 24 | def __init__(self, url: str): 25 | self.url = url 26 | 27 | @abstractmethod 28 | async def fetch_data(self, *args, **kwargs) -> pd.DataFrame: 29 | pass 30 | 31 | @abstractmethod 32 | async def raw_fetch(self, *args, **kwargs) -> pd.DataFrame: 33 | pass 34 | -------------------------------------------------------------------------------- /.github/workflows/gh-pages.yaml: -------------------------------------------------------------------------------- 1 | name: docs 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - "release/*" 8 | pull_request: 9 | branches: 10 | - main 11 | - "release/*" 12 | 13 | jobs: 14 | docs: 15 | if: github.repository == 'numaproj/numalogic' 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v4 19 | - name: Setup Python 20 | uses: actions/setup-python@v5 21 | with: 22 | python-version: 3.9 23 | - name: build 24 | run: make docs 25 | - name: deploy 26 | uses: peaceiris/actions-gh-pages@v3 27 | if: github.repository == 'numaproj/numalogic' && github.ref == 'refs/heads/main' 28 | with: 29 | github_token: ${{ secrets.GITHUB_TOKEN }} 30 | publish_dir: ./site 31 | cname: numalogic.numaproj.io 32 | -------------------------------------------------------------------------------- /numalogic/models/autoencoder/variants/__init__.py: -------------------------------------------------------------------------------- 1 | from numalogic.models.autoencoder.variants.vanilla import ( 2 | VanillaAE, 3 | SparseVanillaAE, 4 | MultichannelAE, 5 | ) 6 | from numalogic.models.autoencoder.variants.icvanilla import VanillaICAE 7 | from numalogic.models.autoencoder.variants.conv import Conv1dAE, SparseConv1dAE 8 | from numalogic.models.autoencoder.variants.lstm import LSTMAE, SparseLSTMAE 9 | from numalogic.models.autoencoder.variants.transformer import TransformerAE, SparseTransformerAE 10 | from numalogic.models.autoencoder.base import BaseAE 11 | 12 | 13 | __all__ = [ 14 | "VanillaAE", 15 | "MultichannelAE", 16 | "SparseVanillaAE", 17 | "Conv1dAE", 18 | "SparseConv1dAE", 19 | "LSTMAE", 20 | "SparseLSTMAE", 21 | "TransformerAE", 22 | "SparseTransformerAE", 23 | "BaseAE", 24 | "VanillaICAE", 25 | ] 26 | -------------------------------------------------------------------------------- /numalogic/connectors/druid/aggregators.py: -------------------------------------------------------------------------------- 1 | def quantiles_doubles_sketch( 2 | raw_column: str, aggregator_name: str, k: int = 128, max_stream_length: int = 1000000000 3 | ) -> dict: 4 | """ 5 | 6 | Args: 7 | raw_column: Name of the column in druid 8 | aggregator_name: Arbitrary aggregator name 9 | k: Controls accuracy, higher the better. Must be a power of 2 from 2 to 32768 10 | max_stream_length: This parameter defines the number of items that can be presented to 11 | each sketch before it may need to move from off-heap to on-heap memory. 12 | 13 | Returns: quantilesDoublesSketch aggregator dict 14 | 15 | """ 16 | return { 17 | "type": "quantilesDoublesSketch", 18 | "name": aggregator_name, 19 | "fieldName": raw_column, 20 | "k": k, 21 | "maxStreamLength": max_stream_length, 22 | } 23 | -------------------------------------------------------------------------------- /numalogic/synthetic/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | 13 | from numalogic.synthetic.timeseries import SyntheticTSGenerator 14 | from numalogic.synthetic.anomalies import AnomalyGenerator 15 | from numalogic.synthetic.sparsity import SparsityGenerator 16 | 17 | 18 | __all__ = ["SyntheticTSGenerator", "AnomalyGenerator", "SparsityGenerator"] 19 | -------------------------------------------------------------------------------- /docs/threshold.md: -------------------------------------------------------------------------------- 1 | # Threshold Estimators 2 | 3 | Threshold Estimators are used for identifying the threshold limit above which we regard the datapoint as anomaly. 4 | It is a simple Estimator that extends BaseEstimator. 5 | 6 | Currently, the library supports `StdDevThreshold`. This takes in paramaters `min_thresh` and `std_factor`. This model 7 | defines threshold as `mean + 3 * std_factor`. 8 | 9 | ```python 10 | import numpy as np 11 | from numalogic.models.threshold import StdDevThreshold 12 | 13 | # Generate positive random data 14 | x_train = np.abs(np.random.randn(1000, 3)) 15 | x_test = np.abs(np.random.randn(30, 3)) 16 | 17 | # Here we want a threshold such that anything 18 | # outside 5 deviations from the mean will be anomalous. 19 | thresh_clf = StdDevThreshold(std_factor=5) 20 | thresh_clf.fit(x_train) 21 | 22 | # Let's get the predictions 23 | y_pred = thresh_clf.predict(x_test) 24 | 25 | # Anomaly scores can be given by, score_samples method 26 | y_score = thresh_clf.score_samples(x_test) 27 | ``` 28 | -------------------------------------------------------------------------------- /tests/connectors/utils/aws/test_db_configurations.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | import pytest 3 | import numalogic.connectors.utils.aws.db_configurations as db_configuration 4 | from numalogic._constants import TESTS_DIR 5 | import os 6 | 7 | 8 | def test_load_db_conf_file_exists(): 9 | result = db_configuration.load_db_conf( 10 | os.path.join(TESTS_DIR, "resources", "rds_db_config.yaml") 11 | ) 12 | assert result is not None 13 | 14 | 15 | @patch( 16 | "numalogic.connectors.utils.aws.db_configurations.OmegaConf.load", 17 | side_effect=FileNotFoundError(), 18 | ) 19 | def test_load_db_conf_file_not_exists(mock_load): 20 | path = "/path/doesnotexist/config.yaml" 21 | with pytest.raises(db_configuration.ConfigNotFoundError): 22 | db_configuration.load_db_conf(path) 23 | 24 | 25 | def test_RDSConnectionConfig_defaults(): 26 | config = db_configuration.RDSConnectionConfig() 27 | assert config.aws_region == "" 28 | assert config.aws_rds_use_iam is False 29 | -------------------------------------------------------------------------------- /tests/models/threshold/test_std.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import numpy as np 4 | from numpy.testing import assert_array_equal 5 | 6 | from numalogic.models.threshold import StdDevThreshold 7 | 8 | 9 | class TestStdDevThreshold(unittest.TestCase): 10 | def setUp(self) -> None: 11 | self.x_train = np.arange(100).reshape(50, 2) 12 | self.x_test = np.arange(100, 160, 6).reshape(5, 2) 13 | 14 | def test_estimator_predict(self): 15 | clf = StdDevThreshold() 16 | clf.fit(self.x_train) 17 | y = clf.predict(self.x_test) 18 | self.assertAlmostEqual(0.4, np.mean(y), places=1) 19 | 20 | def test_estimator_score(self): 21 | clf = StdDevThreshold() 22 | clf.fit(self.x_train) 23 | score = clf.score_samples(self.x_test) 24 | 25 | assert_array_equal(clf.mean + (clf.std_factor * clf.std), clf.threshold) 26 | self.assertAlmostEqual(0.93317, np.mean(score), places=2) 27 | 28 | 29 | if __name__ == "__main__": 30 | unittest.main() 31 | -------------------------------------------------------------------------------- /docs/forecasting.md: -------------------------------------------------------------------------------- 1 | # Forcasting 2 | 3 | Numalogic supports the following variants of forecasting based anomaly detection models. 4 | 5 | ## Naive Forecasters 6 | 7 | ### Baseline Forecaster 8 | 9 | This is a naive forecaster, that uses a combination of: 10 | 11 | 1. Log transformation 12 | 2. Z-Score normalization 13 | 14 | ```python 15 | from numalogic.models.forecast.variants import BaselineForecaster 16 | 17 | model = BaselineForecaster() 18 | model.fit(train_df) 19 | pred_df = model.predict(test_df) 20 | r2_score = model.r2_score(test_df) 21 | anomaly_score = model.score(test_df) 22 | ``` 23 | ### Seasonal Naive Forecaster 24 | 25 | A naive forecaster that takes seasonality into consideration and predicts the previous day/week values. 26 | 27 | ```python 28 | from numalogic.models.forecast.variants import SeasonalNaiveForecaster 29 | 30 | model = SeasonalNaiveForecaster() 31 | model.fit(train_df) 32 | pred_df = model.predict(test_df) 33 | r2_score = model.r2_score(test_df) 34 | anomaly_score = model.score(test_df) 35 | ``` 36 | -------------------------------------------------------------------------------- /numalogic/connectors/__init__.py: -------------------------------------------------------------------------------- 1 | from importlib.util import find_spec 2 | 3 | from numalogic.connectors._config import ( 4 | RedisConf, 5 | PrometheusConf, 6 | ConnectorConf, 7 | DruidConf, 8 | DruidFetcherConf, 9 | ConnectorType, 10 | RDSConf, 11 | RDSFetcherConf, 12 | ) 13 | from numalogic.connectors.prometheus import PrometheusFetcher 14 | from numalogic.connectors.wavefront import WavefrontFetcher 15 | 16 | __all__ = [ 17 | "RedisConf", 18 | "PrometheusConf", 19 | "ConnectorConf", 20 | "DruidConf", 21 | "DruidFetcherConf", 22 | "ConnectorType", 23 | "PrometheusFetcher", 24 | "RDSFetcher", 25 | "RDSConf", 26 | "RDSFetcherConf", 27 | "WavefrontFetcher", 28 | ] 29 | 30 | if find_spec("boto3"): 31 | from numalogic.connectors.rds import RDSFetcher # noqa: F401 32 | 33 | __all__.append("RDSFetcher") 34 | 35 | if find_spec("pydruid"): 36 | from numalogic.connectors.druid import DruidFetcher # noqa: F401 37 | 38 | __all__.append("DruidFetcher") 39 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | default_language_version: 2 | python: python3.9 3 | repos: 4 | - repo: https://github.com/psf/black 5 | rev: 23.3.0 6 | hooks: 7 | - id: black 8 | language_version: python3.9 9 | args: [--config=pyproject.toml, --diff, --color ] 10 | - repo: https://github.com/astral-sh/ruff-pre-commit 11 | rev: v0.1.14 12 | hooks: 13 | - id: ruff 14 | args: [ --fix ] 15 | - id: ruff-format 16 | - repo: https://github.com/adamchainz/blacken-docs 17 | rev: "1.13.0" 18 | hooks: 19 | - id: blacken-docs 20 | additional_dependencies: 21 | - black==22.12.0 22 | - repo: https://github.com/pre-commit/pre-commit-hooks 23 | rev: v4.4.0 24 | hooks: 25 | - id: end-of-file-fixer 26 | - id: trailing-whitespace 27 | - id: check-toml 28 | - id: check-added-large-files 29 | - id: check-ast 30 | - id: check-case-conflict 31 | - id: check-docstring-first 32 | - repo: https://github.com/python-poetry/poetry 33 | rev: "1.6" 34 | hooks: 35 | - id: poetry-check 36 | -------------------------------------------------------------------------------- /numalogic/tools/loss.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Callable 2 | 3 | import torch.nn.functional as F 4 | from torch import Tensor 5 | 6 | 7 | def l2_loss(input_: Tensor, target: Tensor, reduction: str = "mean") -> Tensor: 8 | """Compute the Torch MSE (L2) loss multiplied with a factor of 0.5.""" 9 | return 0.5 * F.mse_loss(input_, target, reduction=reduction) 10 | 11 | 12 | def get_loss_fn(loss_fn: str) -> Callable: 13 | """ 14 | Get the loss function based on the provided loss name. 15 | 16 | Args: 17 | ---- 18 | loss_fn: loss function name (huber, l1, mse) 19 | 20 | Returns 21 | ------- 22 | Callable: loss function 23 | 24 | Raises 25 | ------ 26 | NotImplementedError: If unsupported loss function provided 27 | """ 28 | if loss_fn == "huber": 29 | return F.huber_loss 30 | if loss_fn == "l1": 31 | return F.l1_loss 32 | if loss_fn == "mse": 33 | return l2_loss 34 | raise NotImplementedError(f"Unsupported loss function provided: {loss_fn}") 35 | -------------------------------------------------------------------------------- /.github/workflows/pypi.yml: -------------------------------------------------------------------------------- 1 | name: PyPi Publish 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | jobs: 8 | pypi_publish: 9 | if: github.repository == 'numaproj/numalogic' 10 | runs-on: ubuntu-latest 11 | environment: production 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | python-version: [ "3.11" ] 16 | 17 | name: Publish to PyPi 18 | steps: 19 | - uses: actions/checkout@v4 20 | 21 | - name: Install poetry 22 | run: pipx install poetry==1.6.1 23 | 24 | - name: Set up Python ${{ matrix.python-version }} 25 | uses: actions/setup-python@v5 26 | with: 27 | python-version: ${{ matrix.python-version }} 28 | cache: 'poetry' 29 | 30 | - name: Install dependencies 31 | run: | 32 | poetry env use ${{ matrix.python-version }} 33 | poetry install 34 | 35 | - name: Build dist 36 | run: poetry build 37 | 38 | - name: Publish 39 | run: poetry publish -u __token__ -p ${{ secrets.PYPI_PASSWORD }} 40 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | labels: "bug" 5 | --- 6 | 7 | **Describe the bug** 8 | A clear and concise description of what the bug is. 9 | 10 | **To Reproduce** 11 | Steps to reproduce the behavior: 12 | 13 | 1. .... 14 | 2. .... 15 | 3. .... 16 | 17 | **Expected behavior** 18 | A clear and concise description of what you expected to happen. 19 | 20 | **Screenshots** 21 | If applicable, add screenshots to help explain your problem. 22 | 23 | **Environment (please complete the following information):** 24 | 25 | - Kubernetes: [e.g. v1.18.6] 26 | - Numaflow: [e.g. v0.5.1] 27 | - Numalogic: [e.g. v0.2.0] 28 | - Numaflow-python: [e.g. v0.1.1] 29 | 30 | **Additional context** 31 | Add any other context about the problem here. 32 | 33 | --- 34 | 35 | 36 | 37 | **Message from the maintainers**: 38 | 39 | Impacted by this bug? Give it a 👍. We often sort issues this way to know what to prioritize. 40 | -------------------------------------------------------------------------------- /tests/udfs/resources/_config4.yaml: -------------------------------------------------------------------------------- 1 | stream_confs: 2 | mycustomconf: 3 | config_id: "mycustomconf" 4 | source: "prometheus" 5 | composite_keys: [ "namespace", "app" ] 6 | window_size: 12 7 | ml_pipelines: 8 | pipeline1: 9 | pipeline_id: "pipeline1" 10 | metrics: [ "namespace_app_rollouts_cpu_utilization", "namespace_app_rollouts_http_request_error_rate", "namespace_app_rollouts_memory_utilization" ] 11 | numalogic_conf: 12 | model: 13 | name: "Conv1dVAE" 14 | conf: 15 | seq_len: 12 16 | n_features: 3 17 | latent_dim: 1 18 | preprocess: 19 | - name: "StandardScaler" 20 | threshold: 21 | name: "MahalanobisThreshold" 22 | trainer: 23 | train_hours: 3 24 | min_train_size: 100 25 | pltrainer_conf: 26 | accelerator: cpu 27 | max_epochs: 5 28 | redis_conf: 29 | url: "http://localhost:6222" 30 | port: 26379 31 | expiry: 360 32 | master_name: "mymaster" 33 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: [ "main", "release/*" ] 6 | pull_request: 7 | branches: [ "main", "release/*" ] 8 | 9 | jobs: 10 | build: 11 | name: Python version 12 | runs-on: ubuntu-latest 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | python-version: ["3.9", "3.10", "3.11", "3.12"] 17 | 18 | steps: 19 | - uses: actions/checkout@v4 20 | 21 | - name: Install poetry 22 | run: pipx install poetry==1.6.1 23 | 24 | - name: Set up Python ${{ matrix.python-version }} 25 | uses: actions/setup-python@v5 26 | with: 27 | python-version: ${{ matrix.python-version }} 28 | cache: 'poetry' 29 | 30 | - name: Install dependencies 31 | run: | 32 | poetry env use ${{ matrix.python-version }} 33 | poetry install --all-extras --with dev 34 | poetry run pip install "torch<3.0" -i https://download.pytorch.org/whl/cpu 35 | poetry run pip install "pytorch-lightning<3.0" 36 | 37 | - name: Test with pytest 38 | run: make test 39 | -------------------------------------------------------------------------------- /tests/blocks/test_blocks.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import numpy as np 4 | 5 | from numalogic.blocks import Block 6 | from sklearn.ensemble import IsolationForest 7 | 8 | 9 | class DummyBlock(Block): 10 | def fit(self, input_: np.ndarray, **__) -> np.ndarray: 11 | return self._artifact.fit_predict(input_).reshape(-1, 1) 12 | 13 | def run(self, input_: np.ndarray, **__) -> np.ndarray: 14 | return self._artifact.predict(input_).reshape(-1, 1) 15 | 16 | 17 | class TestBlock(unittest.TestCase): 18 | def test_random_block(self): 19 | block = DummyBlock(IsolationForest(), name="isolation_forest") 20 | self.assertEqual(block.name, "isolation_forest") 21 | 22 | block.fit(np.arange(100).reshape(-1, 2)) 23 | out = block(np.arange(10).reshape(-1, 2)) 24 | self.assertEqual(out.shape, (5, 1)) 25 | 26 | self.assertIsInstance(block.artifact, IsolationForest) 27 | self.assertIsInstance(block.artifact_state, IsolationForest) 28 | self.assertTrue(block.stateful) 29 | 30 | 31 | if __name__ == "__main__": 32 | unittest.main() 33 | -------------------------------------------------------------------------------- /tests/connectors/utils/test_enum.py: -------------------------------------------------------------------------------- 1 | from numalogic.connectors.utils.enum import BaseEnum 2 | import logging 3 | 4 | logging.basicConfig(level=logging.DEBUG) 5 | 6 | 7 | class test_aws_init: 8 | def test_invalid_value_returns_false(self): 9 | # Arrange 10 | class MyEnum(BaseEnum): 11 | VALUE1 = 1 12 | VALUE2 = 2 13 | VALUE3 = 3 14 | 15 | # Act 16 | result = "INVALID" in MyEnum 17 | 18 | # Assert 19 | assert result is False 20 | 21 | def test_invalid_value_returns_true(self): 22 | # Arrange 23 | class MyEnum(BaseEnum): 24 | VALUE1 = 1 25 | VALUE2 = 2 26 | VALUE3 = 3 27 | 28 | # Act 29 | result = 1 in MyEnum 30 | 31 | # Assert 32 | assert result is True 33 | 34 | def test_list_method_returns_list_of_values(self): 35 | # Arrange 36 | class MyEnum(BaseEnum): 37 | VALUE1 = 1 38 | VALUE2 = 2 39 | VALUE3 = 3 40 | 41 | # Act 42 | result = MyEnum.list() 43 | 44 | # Assert 45 | assert result == [1, 2, 3] 46 | -------------------------------------------------------------------------------- /tests/models/threshold/test_median.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import numpy as np 4 | import numpy.typing as npt 5 | import pandas as pd 6 | import pytest 7 | 8 | from numalogic._constants import TESTS_DIR 9 | from numalogic.models.threshold import MaxPercentileThreshold 10 | 11 | 12 | @pytest.fixture 13 | def data() -> tuple[npt.NDArray[float], npt.NDArray[float]]: 14 | x = pd.read_csv( 15 | os.path.join(TESTS_DIR, "resources", "data", "prom_mv.csv"), index_col="timestamp" 16 | ).to_numpy(dtype=np.float32) 17 | return x[:-50], x[-50:] 18 | 19 | 20 | @pytest.fixture() 21 | def fitted(data): 22 | clf = MaxPercentileThreshold(max_inlier_percentile=75, min_threshold=1e-3) 23 | x_train, _ = data 24 | clf.fit(x_train) 25 | return clf 26 | 27 | 28 | def test_score_samples(data, fitted): 29 | _, x_test = data 30 | y_scores = fitted.score_samples(x_test) 31 | assert len(fitted.threshold) == 3 32 | assert fitted.threshold[1] == 1e-3 33 | assert y_scores.shape == (50, 3) 34 | 35 | 36 | def test_predict(data, fitted): 37 | _, x_test = data 38 | y_pred = fitted.predict(x_test) 39 | assert y_pred.shape == (50, 3) 40 | -------------------------------------------------------------------------------- /numalogic/udfs/_logger.py: -------------------------------------------------------------------------------- 1 | import structlog 2 | from structlog import processors, stdlib 3 | 4 | 5 | def configure_logger(): 6 | """Configure struct logger for the UDFs.""" 7 | structlog.configure( 8 | processors=[ 9 | stdlib.filter_by_level, 10 | stdlib.add_log_level, 11 | stdlib.PositionalArgumentsFormatter(), 12 | processors.TimeStamper(fmt="iso"), 13 | processors.StackInfoRenderer(), 14 | # processors.format_exc_info, 15 | processors.UnicodeDecoder(), 16 | processors.KeyValueRenderer(key_order=["uuid", "event"]), 17 | stdlib.ProcessorFormatter.wrap_for_formatter, 18 | ], 19 | logger_factory=stdlib.LoggerFactory(), 20 | wrapper_class=stdlib.BoundLogger, 21 | cache_logger_on_first_use=True, 22 | ) 23 | return structlog.getLogger(__name__) 24 | 25 | 26 | def log_data_payload_values(log, data_payload): 27 | return log.bind( 28 | uuid=data_payload["uuid"], 29 | config_id=data_payload["config_id"], 30 | pipeline_id=data_payload.get("pipeline_id", "default"), 31 | metadata=data_payload.get("metadata", {}), 32 | ) 33 | -------------------------------------------------------------------------------- /tests/models/threshold/test_static.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import numpy as np 3 | 4 | from numalogic.models.threshold import StaticThreshold, SigmoidThreshold 5 | 6 | 7 | @pytest.fixture 8 | def x(): 9 | return np.arange(20).reshape(10, 2).astype(float) 10 | 11 | 12 | def test_static_threshold_predict(x): 13 | clf = StaticThreshold(upper_limit=5) 14 | clf.fit(x) 15 | y = clf.predict(x) 16 | assert x.shape == y.shape 17 | assert np.max(y) == 1 18 | assert np.min(y) == 0 19 | 20 | 21 | def test_static_threshold_score_01(x): 22 | clf = StaticThreshold(upper_limit=5.0) 23 | y = clf.score_samples(x) 24 | assert x.shape == y.shape 25 | assert np.max(y) == clf.outlier_score 26 | assert np.min(y) == clf.inlier_score 27 | 28 | 29 | def test_sigmoid_threshold_score_02(x): 30 | clf = SigmoidThreshold(5, 10) 31 | y = clf.score_samples(x) 32 | assert x.shape == y.shape 33 | assert np.max(y) == clf.score_limit 34 | assert np.min(y) > 0.0 35 | 36 | 37 | def test_sigmoid_threshold_predict(x): 38 | clf = SigmoidThreshold(5) 39 | clf.fit(x) 40 | y = clf.predict(x) 41 | assert x.shape == y.shape 42 | assert np.max(y) == 1 43 | assert np.min(y) == 0 44 | -------------------------------------------------------------------------------- /numalogic/connectors/rds/db/factory.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from numalogic.connectors.utils.aws.config import DatabaseTypes 4 | from numalogic.connectors.utils.aws.exceptions import UnRecognizedDatabaseTypeException 5 | 6 | _LOGGER = logging.getLogger(__name__) 7 | 8 | 9 | class RdsFactory: 10 | """class represents a factory for creating database handlers for different database types.""" 11 | 12 | @classmethod 13 | def get_db_handler(cls, database_type: DatabaseTypes): 14 | """ 15 | Get the database handler for the specified database type. 16 | 17 | Args: 18 | - database_type (str): The type of the database. 19 | 20 | Returns 21 | ------- 22 | - The database handler for the specified database type. 23 | 24 | Raises 25 | ------ 26 | - UnRecognizedDatabaseTypeException: If the specified database type is not supported. 27 | 28 | """ 29 | if database_type == DatabaseTypes.MYSQL: 30 | from numalogic.connectors.rds.db.mysql_fetcher import MysqlFetcher 31 | 32 | return MysqlFetcher 33 | 34 | raise UnRecognizedDatabaseTypeException(f"database_type: {database_type} is not supported") 35 | -------------------------------------------------------------------------------- /numalogic/blocks/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | """ 13 | Module for numalogic blocks which are units of computation that can be 14 | chained together to form a pipeline if needed. A block can be stateful or stateless. 15 | """ 16 | 17 | from numalogic.blocks._base import Block 18 | from numalogic.blocks._nn import NNBlock 19 | from numalogic.blocks._transform import PreprocessBlock, PostprocessBlock, ThresholdBlock 20 | from numalogic.blocks.pipeline import BlockPipeline 21 | 22 | __all__ = [ 23 | "Block", 24 | "NNBlock", 25 | "PreprocessBlock", 26 | "PostprocessBlock", 27 | "ThresholdBlock", 28 | "BlockPipeline", 29 | ] 30 | -------------------------------------------------------------------------------- /numalogic/_constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | 13 | import os 14 | 15 | NUMALOGIC_DIR = os.path.dirname(__file__) 16 | BASE_DIR = os.path.split(NUMALOGIC_DIR)[0] 17 | TESTS_DIR = os.path.join(NUMALOGIC_DIR, "../tests") 18 | BASE_CONF_DIR = os.path.join(BASE_DIR, "config") 19 | 20 | DEFAULT_BASE_CONF_PATH = os.path.join(BASE_CONF_DIR, "default-configs", "config.yaml") 21 | DEFAULT_METRICS_CONF_PATH = os.path.join( 22 | BASE_CONF_DIR, "default-configs", "numalogic_udf_metrics.yaml" 23 | ) 24 | DEFAULT_APP_CONF_PATH = os.path.join(BASE_CONF_DIR, "app-configs", "config.yaml") 25 | DEFAULT_METRICS_PORT = 8490 26 | NUMALOGIC_METRICS = "numalogic_metrics" 27 | -------------------------------------------------------------------------------- /examples/multi_udf/src/utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | from dataclasses import dataclass, asdict 5 | 6 | import numpy as np 7 | import numpy.typing as npt 8 | from typing_extensions import Self 9 | 10 | DIR = os.path.dirname(__file__) 11 | ROOT_DIR = os.path.split(DIR)[0] 12 | TRAIN_DATA_PATH = os.path.join(ROOT_DIR, "src/resources/train_data.csv") 13 | TRACKING_URI = "http://mlflow-service.default.svc.cluster.local:5000" 14 | LOGGER = logging.getLogger(__name__) 15 | 16 | 17 | @dataclass(slots=True) 18 | class Payload: 19 | """Payload to be used for inter-vertex data transfer.""" 20 | 21 | uuid: str 22 | arr: list[float] 23 | anomaly_score: float = None 24 | is_artifact_valid: bool = True 25 | 26 | def get_array(self) -> npt.NDArray[float]: 27 | return np.asarray(self.arr) 28 | 29 | def set_array(self, arr: list[float]) -> None: 30 | self.arr = arr 31 | 32 | def to_json(self) -> bytes: 33 | """Converts the payload to json.""" 34 | return json.dumps(asdict(self)).encode("utf-8") 35 | 36 | @classmethod 37 | def from_json(cls, json_payload: bytes) -> Self: 38 | """Converts the json payload to Payload object.""" 39 | return cls(**json.loads(json_payload.decode("utf-8"))) 40 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: Coverage 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | 9 | jobs: 10 | build: 11 | name: Codecov 12 | runs-on: ubuntu-latest 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | python-version: ["3.9"] 17 | 18 | steps: 19 | - uses: actions/checkout@v4 20 | 21 | - name: Install poetry 22 | run: pipx install poetry==1.6.1 23 | 24 | - name: Set up Python ${{ matrix.python-version }} 25 | uses: actions/setup-python@v5 26 | with: 27 | python-version: ${{ matrix.python-version }} 28 | cache: 'poetry' 29 | 30 | - name: Install dependencies 31 | run: | 32 | poetry env use ${{ matrix.python-version }} 33 | poetry install --all-extras --with dev 34 | poetry run pip install "torch<3.0" -i https://download.pytorch.org/whl/cpu 35 | poetry run pip install "pytorch-lightning<3.0" 36 | 37 | - name: Run Coverage 38 | run: | 39 | poetry run pytest --cov-report=xml --cov=numalogic --cov-config .coveragerc tests/ -sq 40 | 41 | - name: Upload Coverage 42 | uses: codecov/codecov-action@v4 43 | with: 44 | token: ${{ secrets.CODECOV_TOKEN }} 45 | files: ./coverage.xml 46 | fail_ci_if_error: true 47 | verbose: true 48 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Numalogic 2 | 3 | 4 | Numalogic is a collection of ML models and algorithms for real-time data analytics and AIOps including anomaly detection. 5 | 6 | Numalogic can be installed as a library and used to build end-to-end ML pipelines. For streaming real time data processing, it can also be paired with our steaming data platform [Numaflow](https://numaflow.numaproj.io/). 7 | 8 | ## Key Features 9 | 10 | 1. Ease of use: simple and efficient tools for predictive data analytics 11 | 2. Reusability: all the functionalities can be re-used in various contexts 12 | 3. Model selection: easy to compare, validate, fine-tune and choose the model that works best with each data set 13 | 4. Data processing: readily available feature extraction, scaling, transforming and normalization tools 14 | 5. Extensibility: adding your own functions or extending over the existing capabilities 15 | 6. Model Storage: out-of-the-box support for MLFlow and support for other model ML lifecycle management tools 16 | 17 | ## Use Cases 18 | 1. Deployment failure detection 19 | 2. System failure detection for node failures or crashes 20 | 3. Fraud detection 21 | 4. Network intrusion detection 22 | 5. Forecasting on time series data 23 | 24 | ## Getting Started 25 | 26 | For set-up information and running your first pipeline using numalogic, please see our [getting started guide](./quick-start.md). 27 | -------------------------------------------------------------------------------- /examples/multi_udf/src/udf/postprocess.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import numpy as np 4 | from numalogic.udfs import NumalogicUDF 5 | from numalogic.transforms import TanhNorm 6 | from pynumaflow.function import Messages, Message, Datum 7 | 8 | from src.utils import Payload 9 | 10 | LOGGER = logging.getLogger(__name__) 11 | 12 | 13 | class Postprocess(NumalogicUDF): 14 | """UDF to postprocess the anomaly score, and scale it between [0,10].""" 15 | 16 | def __init__(self): 17 | super().__init__() 18 | 19 | def exec(self, _: list[str], datum: Datum) -> Messages: 20 | """The postprocess transforms the inferred data into anomaly score between [0,10] 21 | and sends it to log sink. 22 | 23 | For more information about the arguments, refer: 24 | https://github.com/numaproj/numaflow-python/blob/main/pynumaflow/function/_dtypes.py 25 | """ 26 | # Load json data 27 | payload = Payload.from_json(datum.value) 28 | 29 | # Postprocess step 30 | data = payload.get_array() 31 | 32 | # Taking mean of the anomaly scores 33 | normalizer = TanhNorm() 34 | payload.anomaly_score = normalizer.fit_transform(np.mean(data)) 35 | 36 | LOGGER.info("%s - The anomaly score is: %s", payload.uuid, payload.anomaly_score) 37 | 38 | # Convert Payload back to bytes 39 | return Messages(Message(value=payload.to_json())) 40 | -------------------------------------------------------------------------------- /examples/multi_udf/src/udf/preprocess.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import uuid 4 | 5 | from numalogic.transforms import LogTransformer 6 | from numalogic.udfs import NumalogicUDF 7 | from pynumaflow.function import Messages, Message, Datum 8 | 9 | from src.utils import Payload 10 | 11 | LOGGER = logging.getLogger(__name__) 12 | 13 | 14 | class Preprocess(NumalogicUDF): 15 | """UDF to preprocess the input data for ML inference.""" 16 | 17 | def __init__(self): 18 | super().__init__() 19 | 20 | def exec(self, _: list[str], datum: Datum) -> Messages: 21 | """The preprocess function here transforms the input data for ML inference and sends 22 | the payload to inference vertex. 23 | 24 | For more information about the arguments, refer: 25 | https://github.com/numaproj/numaflow-python/blob/main/pynumaflow/function/_dtypes.py 26 | """ 27 | # Load json data 28 | series = json.loads(datum.value)["data"] 29 | payload = Payload(uuid=str(uuid.uuid4()), arr=series) 30 | 31 | # preprocess step 32 | data = payload.get_array() 33 | clf = LogTransformer() 34 | out = clf.fit_transform(data) 35 | payload.set_array(out.tolist()) 36 | LOGGER.info("%s - Preprocess complete for data: %s", payload.uuid, payload.arr) 37 | 38 | # Return as a Messages object 39 | return Messages(Message(value=payload.to_json())) 40 | -------------------------------------------------------------------------------- /examples/block_pipeline/src/utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | from redis.sentinel import Sentinel 5 | 6 | from numalogic.tools.types import Singleton, redis_client_t 7 | 8 | _LOGGER = logging.getLogger(__name__) 9 | _DIR = os.path.dirname(__file__) 10 | _ROOT_DIR = os.path.split(_DIR)[0] 11 | TRAIN_DATA_PATH = os.path.join(_ROOT_DIR, "resources/train_data.csv") 12 | 13 | AUTH = os.getenv("REDIS_AUTH") 14 | HOST = os.getenv("REDIS_HOST", default="isbsvc-redis-isbs-redis-svc.default.svc") 15 | PORT = os.getenv("REDIS_PORT", default="26379") 16 | MASTERNAME = os.getenv("REDIS_MASTER_NAME", default="mymaster") 17 | 18 | 19 | class RedisClient(metaclass=Singleton): 20 | """Singleton class to manage redis client.""" 21 | 22 | _client: redis_client_t = None 23 | 24 | def __init__(self): 25 | if not self._client: 26 | self.set_client() 27 | 28 | def set_client(self) -> None: 29 | sentinel_args = { 30 | "sentinels": [(HOST, PORT)], 31 | "socket_timeout": 0.1, 32 | } 33 | _LOGGER.info("Connecting to redis sentinel: %s, %s, %s", sentinel_args, MASTERNAME, AUTH) 34 | sentinel = Sentinel( 35 | **sentinel_args, 36 | sentinel_kwargs=dict(password=AUTH), 37 | password=AUTH, 38 | ) 39 | self._client = sentinel.master_for(MASTERNAME) 40 | 41 | def get_client(self) -> redis_client_t: 42 | return self._client 43 | -------------------------------------------------------------------------------- /numalogic/registry/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | 13 | from numalogic.registry.artifact import ArtifactManager, ArtifactData, ArtifactCache 14 | from numalogic.registry.localcache import LocalLRUCache 15 | 16 | 17 | __all__ = ["ArtifactManager", "ArtifactData", "ArtifactCache", "LocalLRUCache"] 18 | 19 | 20 | try: 21 | from numalogic.registry.mlflow_registry import MLflowRegistry # noqa: F401 22 | except ImportError: 23 | pass 24 | else: 25 | __all__.append("MLflowRegistry") 26 | 27 | try: 28 | from numalogic.registry.redis_registry import RedisRegistry # noqa: F401 29 | except ImportError: 30 | pass 31 | else: 32 | __all__.append("RedisRegistry") 33 | 34 | try: 35 | from numalogic.registry.dynamodb_registry import DynamoDBRegistry # noqa: F401 36 | except ImportError: 37 | pass 38 | else: 39 | __all__.append("DynamoDBRegistry") 40 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: Numalogic - Collection of operational time series ML models and tools 2 | repo_url: https://github.com/numaproj/numalogic 3 | edit_uri: edit/main/docs/ 4 | strict: true 5 | theme: 6 | name: material 7 | favicon: assets/numaproj.svg 8 | font: 9 | text: Roboto 10 | code: Roboto Mono 11 | logo: assets/logo.png 12 | palette: 13 | - scheme: default 14 | primary: blue 15 | toggle: 16 | icon: material/toggle-switch-off-outline 17 | name: Switch to dark mode 18 | - scheme: slate 19 | toggle: 20 | icon: material/toggle-switch 21 | name: Switch to light mode 22 | features: 23 | - navigation.tabs 24 | - navigation.tabs.sticky 25 | - navigation.top 26 | extra: 27 | analytics: 28 | provider: google 29 | property: G-9D7BFBLNPB 30 | markdown_extensions: 31 | - codehilite 32 | - admonition 33 | - pymdownx.superfences 34 | - pymdownx.details 35 | - toc: 36 | permalink: true 37 | nav: 38 | - Home: README.md 39 | - Getting Started: 40 | - quick-start.md 41 | - User Guide: 42 | - data-generator.md 43 | - Core Concepts: 44 | - Training: 45 | - autoencoders.md 46 | - forecasting.md 47 | - inference.md 48 | - Addons: 49 | - pre-processing.md 50 | - post-processing.md 51 | - Model Registry: 52 | - ml-flow.md 53 | - Numaproj: https://numaproj.io 54 | -------------------------------------------------------------------------------- /numalogic/config/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | 13 | from numalogic.config._config import ( 14 | NumalogicConf, 15 | ModelInfo, 16 | LightningTrainerConf, 17 | RegistryInfo, 18 | TrainerConf, 19 | ScoreConf, 20 | AggregatorConf, 21 | ScoreAdjustConf, 22 | ) 23 | from numalogic.config.factory import ( 24 | ModelFactory, 25 | PreprocessFactory, 26 | PostprocessFactory, 27 | ThresholdFactory, 28 | RegistryFactory, 29 | AggregatorFactory, 30 | ) 31 | 32 | 33 | __all__ = [ 34 | "NumalogicConf", 35 | "ModelInfo", 36 | "LightningTrainerConf", 37 | "RegistryInfo", 38 | "ModelFactory", 39 | "PreprocessFactory", 40 | "PostprocessFactory", 41 | "ThresholdFactory", 42 | "RegistryFactory", 43 | "TrainerConf", 44 | "ScoreConf", 45 | "ScoreAdjustConf", 46 | "AggregatorConf", 47 | "AggregatorFactory", 48 | ] 49 | -------------------------------------------------------------------------------- /tests/synthetic/test_sparsity.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import unittest 3 | 4 | from numalogic.synthetic import SyntheticTSGenerator, SparsityGenerator 5 | 6 | 7 | class Testsparsity(unittest.TestCase): 8 | def test_sparsity_generator1(self): 9 | ts_generator = SyntheticTSGenerator(12000, 10) 10 | ts_df = ts_generator.gen_tseries() 11 | data = copy.deepcopy(ts_df) 12 | SparsityGN = SparsityGenerator(data, sparse_ratio=0) 13 | SparsityGN.generate_sparsity() 14 | transformed_data = SparsityGN.data 15 | self.assertEqual(transformed_data.equals(ts_df), True) 16 | 17 | def test_sparsity_generator2(self): 18 | ts_generator = SyntheticTSGenerator(12000, 10) 19 | ts_df = ts_generator.gen_tseries() 20 | data = copy.deepcopy(ts_df) 21 | SparsityGN = SparsityGenerator(data, sparse_ratio=1) 22 | SparsityGN.generate_sparsity() 23 | transformed_data = SparsityGN.data 24 | self.assertEqual(transformed_data.equals(ts_df), False) 25 | 26 | def test_sparsityreturn_series(self): 27 | ts_generator = SyntheticTSGenerator(12000, 10) 28 | ts_df = ts_generator.gen_tseries() 29 | data = copy.deepcopy(ts_df) 30 | SparsityGN = SparsityGenerator(data, sparse_ratio=0) 31 | SparsityGN.generate_sparsity() 32 | transformed_data = SparsityGN.data 33 | self.assertEqual(transformed_data.shape, ts_df.shape) 34 | 35 | 36 | if __name__ == "__main__": 37 | unittest.main() 38 | -------------------------------------------------------------------------------- /numalogic/connectors/utils/aws/db_configurations.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from numalogic.tools.exceptions import ConfigNotFoundError 4 | from omegaconf import OmegaConf 5 | from numalogic.connectors.utils.aws.config import RDSConnectionConfig 6 | 7 | _LOGGER = logging.getLogger(__name__) 8 | 9 | 10 | def load_db_conf(*paths: str) -> RDSConnectionConfig: 11 | """ 12 | Load database configuration from one or more YAML files. 13 | 14 | Args: 15 | - paths (str): One or more paths to YAML files containing the database configuration. 16 | 17 | Returns 18 | ------- 19 | - RDSConfig: An instance of the RDSConfig class representing the loaded database configuration. 20 | 21 | Raises 22 | ------ 23 | - ConfigNotFoundError: If none of the given configuration file paths exist. 24 | 25 | Example: 26 | load_db_conf("/path/to/config.yaml", "/path/to/another/config.yaml") 27 | """ 28 | confs = [] 29 | for _path in paths: 30 | try: 31 | conf = OmegaConf.load(_path) 32 | except FileNotFoundError: 33 | _LOGGER.warning("Config file path: %s not found. Skipping...", _path) 34 | continue 35 | confs.append(conf) 36 | 37 | if not confs: 38 | _err_msg = f"None of the given conf paths exist: {paths}" 39 | raise ConfigNotFoundError(_err_msg) 40 | 41 | schema = OmegaConf.structured(RDSConnectionConfig) 42 | conf = OmegaConf.merge(schema, *confs) 43 | return OmegaConf.to_object(conf) 44 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Check Python 2 | PYTHON:=$(shell command -v python 2> /dev/null) 3 | ifndef PYTHON 4 | PYTHON:=$(shell command -v python3 2> /dev/null) 5 | endif 6 | ifndef PYTHON 7 | $(error "Python is not available, please install.") 8 | endif 9 | 10 | clean: 11 | @rm -rf build dist .eggs *.egg-info 12 | @rm -rf .benchmarks .coverage coverage.xml htmlcov report.xml .tox 13 | @find . -type d -name '.mypy_cache' -exec rm -rf {} + 14 | @find . -type d -name '__pycache__' -exec rm -rf {} + 15 | @find . -type d -name '*pytest_cache*' -exec rm -rf {} + 16 | @find . -type f -name "*.py[co]" -exec rm -rf {} + 17 | 18 | format: clean 19 | poetry run black numalogic/ examples/ tests/ 20 | 21 | lint: format 22 | poetry run ruff check --fix . 23 | 24 | # install all dependencies 25 | setup: 26 | poetry install --with dev,torch,jupyter --all-extras 27 | 28 | # test your application (tests in the tests/ directory) 29 | test: 30 | poetry run pytest -v tests/ 31 | 32 | publish: 33 | @rm -rf dist 34 | poetry build 35 | poetry publish 36 | 37 | requirements: 38 | poetry export -f requirements.txt --output requirements.txt --without-hashes 39 | 40 | tag: 41 | VERSION=v$(shell poetry version -s) 42 | @echo "Tagging version $(VERSION)" 43 | git tag -s -a $(VERSION) -m "Release $(VERSION)" 44 | 45 | /usr/local/bin/mkdocs: 46 | $(PYTHON) -m pip install mkdocs==1.3.0 mkdocs_material==8.3.9 47 | 48 | # docs 49 | 50 | .PHONY: docs 51 | docs: /usr/local/bin/mkdocs 52 | mkdocs build 53 | 54 | .PHONY: docs-serve 55 | docs-serve: docs 56 | mkdocs serve 57 | -------------------------------------------------------------------------------- /tests/synthetic/test_timeseries.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from numalogic.synthetic import SyntheticTSGenerator 4 | 5 | 6 | class TestSyntheticTSGenerator(unittest.TestCase): 7 | def test_get_tseries(self): 8 | ts_generator = SyntheticTSGenerator(12000, 10) 9 | ts_df = ts_generator.gen_tseries() 10 | self.assertEqual((12000, 10), ts_df.shape) 11 | 12 | def test_baseline(self): 13 | ts_generator = SyntheticTSGenerator(12000, 10) 14 | baseline = ts_generator.baseline() 15 | self.assertTrue(baseline) 16 | 17 | def test_trend(self): 18 | ts_generator = SyntheticTSGenerator(12000, 10) 19 | trend = ts_generator.trend() 20 | self.assertEqual((12000,), trend.shape) 21 | 22 | def test_seasonality(self): 23 | ts_generator = SyntheticTSGenerator(1000, 10) 24 | seasonal = ts_generator.seasonality(ts_generator.primary_period) 25 | self.assertEqual((1000,), seasonal.shape) 26 | 27 | def test_noise(self): 28 | ts_generator = SyntheticTSGenerator(12000, 10) 29 | noise = ts_generator.noise() 30 | self.assertEqual((12000,), noise.shape) 31 | 32 | def test_train_test_split(self): 33 | ts_generator = SyntheticTSGenerator(10080, 10) 34 | df = ts_generator.gen_tseries() 35 | train_df, test_df = ts_generator.train_test_split(df, 1440) 36 | self.assertEqual((8640, 10), train_df.shape) 37 | self.assertEqual((1440, 10), test_df.shape) 38 | 39 | 40 | if __name__ == "__main__": 41 | unittest.main() 42 | -------------------------------------------------------------------------------- /examples/block_pipeline/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # builder: install needed dependencies 3 | #################################################################################################### 4 | 5 | FROM python:3.10-slim-bullseye AS builder 6 | 7 | ENV PYTHONFAULTHANDLER=1 \ 8 | PYTHONUNBUFFERED=1 \ 9 | PYTHONHASHSEED=random \ 10 | PIP_NO_CACHE_DIR=on \ 11 | PIP_DISABLE_PIP_VERSION_CHECK=on \ 12 | PIP_DEFAULT_TIMEOUT=100 \ 13 | PYSETUP_PATH="/opt/pysetup" \ 14 | VENV_PATH="/opt/pysetup/.venv" 15 | 16 | ENV PATH="$VENV_PATH/bin:$PATH" 17 | 18 | RUN apt-get update \ 19 | && apt-get install --no-install-recommends -y \ 20 | curl \ 21 | wget \ 22 | # deps for building python deps 23 | build-essential \ 24 | && apt-get install -y git \ 25 | && apt-get clean && rm -rf /var/lib/apt/lists/* \ 26 | \ 27 | # install dumb-init 28 | && wget -O /dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_x86_64 \ 29 | && chmod +x /dumb-init 30 | 31 | #################################################################################################### 32 | # udf: used for running the udf vertices 33 | #################################################################################################### 34 | FROM builder AS udf 35 | 36 | WORKDIR $PYSETUP_PATH 37 | COPY ./requirements.txt ./ 38 | RUN pip3 install -r requirements.txt 39 | 40 | ADD . /app 41 | WORKDIR /app 42 | 43 | ENTRYPOINT ["/dumb-init", "--"] 44 | 45 | EXPOSE 5000 46 | -------------------------------------------------------------------------------- /tests/udfs/resources/_config2.yaml: -------------------------------------------------------------------------------- 1 | stream_confs: 2 | druid-config: 3 | config_id: "druid-config" 4 | source: "druid" 5 | composite_keys: [ 'service-mesh', '1', '2' ] 6 | window_size: 10 7 | ml_pipelines: 8 | pipeline1: 9 | pipeline_id: "pipeline1" 10 | metrics: [ "failed" , "degraded" ] 11 | numalogic_conf: 12 | model: 13 | name: "VanillaAE" 14 | conf: 15 | seq_len: 10 16 | n_features: 2 17 | preprocess: 18 | - name: "LogTransformer" 19 | stateful: false 20 | conf: 21 | add_factor: 5 22 | - name: "StandardScaler" 23 | stateful: true 24 | threshold: 25 | name: "MahalanobisThreshold" 26 | conf: 27 | max_outlier_prob: 0.08 28 | trainer: 29 | train_hours: 3 30 | min_train_size: 100 31 | transforms: 32 | - name: "DataClipper" 33 | conf: 34 | lower: [0.0,0.0] 35 | pltrainer_conf: 36 | accelerator: cpu 37 | max_epochs: 1 38 | 39 | redis_conf: 40 | url: "isbsvc-redis-isbs-redis-svc.oss-analytics-numalogicosamfci-usw2-e2e.svc" 41 | port: 26379 42 | expiry: 360 43 | master_name: "mymaster" 44 | 45 | druid_conf: 46 | url: "druid-endpoint" 47 | endpoint: "endpoint" 48 | id_fetcher: 49 | druid-config-pipeline1: 50 | dimensions: [ "col1" ] 51 | datasource: "table-name" 52 | group_by: [ "timestamp", "col1" ] 53 | pivot: 54 | columns: [ "col2" ] 55 | -------------------------------------------------------------------------------- /numalogic/transforms/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | """ 13 | Module to provide timeseries transformations needed for preprocessing, 14 | feature engineering and postprocessing. 15 | """ 16 | 17 | from numalogic.transforms._scaler import TanhScaler, PercentileScaler 18 | from numalogic.transforms._stateless import ( 19 | LogTransformer, 20 | StaticPowerTransformer, 21 | DataClipper, 22 | GaussianNoiseAdder, 23 | DifferenceTransform, 24 | FlattenVector, 25 | FlattenVectorWithPadding, 26 | ) 27 | from numalogic.transforms._movavg import ExpMovingAverage, expmov_avg_aggregator 28 | from numalogic.transforms._postprocess import TanhNorm, tanh_norm, SigmoidNorm 29 | 30 | __all__ = [ 31 | "TanhScaler", 32 | "LogTransformer", 33 | "StaticPowerTransformer", 34 | "DataClipper", 35 | "ExpMovingAverage", 36 | "expmov_avg_aggregator", 37 | "TanhNorm", 38 | "tanh_norm", 39 | "GaussianNoiseAdder", 40 | "DifferenceTransform", 41 | "FlattenVector", 42 | "FlattenVectorWithPadding", 43 | "PercentileScaler", 44 | "SigmoidNorm", 45 | ] 46 | -------------------------------------------------------------------------------- /numalogic/udfs/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from logging import config as logconf 3 | import os 4 | 5 | from numalogic._constants import BASE_DIR 6 | from numalogic.udfs._base import NumalogicUDF 7 | from numalogic.udfs._config import StreamConf, PipelineConf, MLPipelineConf, load_pipeline_conf 8 | from numalogic.udfs._metrics_utility import MetricsLoader 9 | from numalogic.udfs.factory import UDFFactory, ServerFactory 10 | from numalogic.udfs.payloadtx import PayloadTransformer 11 | from numalogic.udfs.inference import InferenceUDF 12 | from numalogic.udfs.postprocess import PostprocessUDF 13 | from numalogic.udfs.preprocess import PreprocessUDF 14 | from numalogic.udfs.trainer import TrainerUDF, PromTrainerUDF, DruidTrainerUDF, RDSTrainerUDF 15 | 16 | 17 | def set_logger() -> None: 18 | """Sets the logger for the UDFs.""" 19 | logconf.fileConfig( 20 | fname=os.path.join(BASE_DIR, "log.conf"), 21 | disable_existing_loggers=False, 22 | ) 23 | if os.getenv("DEBUG", "false").lower() == "true": 24 | logging.getLogger("root").setLevel(logging.DEBUG) 25 | 26 | 27 | def set_metrics(conf_file: str) -> None: 28 | """Sets the metrics for the UDFs.""" 29 | MetricsLoader().load_metrics(config_file_path=conf_file) 30 | 31 | 32 | __all__ = [ 33 | "NumalogicUDF", 34 | "PayloadTransformer", 35 | "PreprocessUDF", 36 | "InferenceUDF", 37 | "TrainerUDF", 38 | "PromTrainerUDF", 39 | "DruidTrainerUDF", 40 | "RDSTrainerUDF", 41 | "PostprocessUDF", 42 | "UDFFactory", 43 | "StreamConf", 44 | "PipelineConf", 45 | "MLPipelineConf", 46 | "load_pipeline_conf", 47 | "ServerFactory", 48 | "set_logger", 49 | "set_metrics", 50 | "MetricsLoader", 51 | ] 52 | -------------------------------------------------------------------------------- /numalogic/synthetic/sparsity.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | 13 | import random 14 | 15 | 16 | class SparsityGenerator: 17 | """Introduces sparsity to data by reassigning certain rows and columns 18 | in the dataframe to value of 0 (based on sparsity ratio). 19 | """ 20 | 21 | def __init__(self, data, sparse_ratio=0.2): 22 | """@param data: Reference Multivariate time series DataFrame 23 | @param sparse_ratio: Ratio of sparsity to introduce wrt 24 | to number of samples. 25 | """ 26 | self.sparse_ratio = sparse_ratio 27 | self._data = data 28 | 29 | def generate_sparsity(self): 30 | shape = self._data.shape 31 | 32 | # based on sparsity ratio generating the rows 33 | # to which the data is going to be imputed with 0 34 | rows = random.sample(range(0, shape[0]), int(shape[0] * self.sparse_ratio)) 35 | 36 | for row in rows: 37 | # identifying the columns to which the data is going to be imputed with 0. 38 | columns = random.sample(range(0, shape[1]), int(shape[1] * self.sparse_ratio)) 39 | self._data.iloc[row, columns] = 0 40 | 41 | @property 42 | def data(self): 43 | return self._data 44 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # builder: install needed dependencies and setup virtual environment 3 | #################################################################################################### 4 | 5 | ARG PYTHON_VERSION=3.11 6 | FROM python:${PYTHON_VERSION}-bookworm AS builder 7 | 8 | ARG POETRY_VERSION=1.8 9 | ARG INSTALL_EXTRAS 10 | 11 | ENV POETRY_NO_INTERACTION=1 \ 12 | POETRY_VIRTUALENVS_IN_PROJECT=1 \ 13 | POETRY_VIRTUALENVS_CREATE=1 \ 14 | POETRY_CACHE_DIR=/tmp/poetry_cache \ 15 | POETRY_VERSION=${POETRY_VERSION} \ 16 | POETRY_HOME="/opt/poetry" \ 17 | PATH="$POETRY_HOME/bin:$PATH" 18 | 19 | RUN pip install --no-cache-dir poetry==$POETRY_VERSION 20 | 21 | WORKDIR /app 22 | COPY poetry.lock pyproject.toml ./ 23 | 24 | RUN poetry install --without dev --no-root --extras "${INSTALL_EXTRAS}" \ 25 | && poetry run pip install --no-cache-dir "torch>=2.0,<3.0" --index-url https://download.pytorch.org/whl/cpu \ 26 | && poetry run pip install --no-cache-dir "lightning[pytorch]<3.0" 27 | 28 | #################################################################################################### 29 | # runtime: used for running the udf vertices 30 | #################################################################################################### 31 | FROM python:${PYTHON_VERSION}-slim-bookworm AS runtime 32 | 33 | RUN apt-get update \ 34 | && apt-get install dumb-init \ 35 | && apt-get clean && rm -rf /var/lib/apt/lists/* \ 36 | && apt-get purge -y --auto-remove 37 | 38 | 39 | ENV VIRTUAL_ENV=/app/.venv 40 | COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} 41 | ENV PATH="$VIRTUAL_ENV/bin:$PATH" 42 | 43 | COPY . /app 44 | WORKDIR /app 45 | 46 | ENTRYPOINT ["/usr/bin/dumb-init", "--"] 47 | EXPOSE 5000 48 | -------------------------------------------------------------------------------- /tests/udfs/test_factory.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from fakeredis import FakeRedis, FakeServer 4 | 5 | from numalogic.udfs import UDFFactory, ServerFactory, PipelineConf 6 | 7 | 8 | class TestUDFFactory(unittest.TestCase): 9 | def test_get_cls_01(self): 10 | udf_cls = UDFFactory.get_udf_cls("inference") 11 | self.assertEqual(udf_cls.__name__, "InferenceUDF") 12 | 13 | def test_get_cls_02(self): 14 | udf_cls = UDFFactory.get_udf_cls("trainer") 15 | self.assertEqual(udf_cls.__name__, "DruidTrainerUDF") 16 | 17 | def test_get_cls_err(self): 18 | with self.assertRaises(ValueError): 19 | UDFFactory.get_udf_cls("some_udf") 20 | 21 | def test_get_instance_01(self): 22 | udf = UDFFactory.get_udf_instance("preprocess", r_client=FakeRedis(server=FakeServer())) 23 | self.assertIsInstance(udf, UDFFactory.get_udf_cls("preprocess")) 24 | 25 | def test_get_instance_02(self): 26 | udf = UDFFactory.get_udf_instance( 27 | "staticthresh", pl_conf=PipelineConf(), r_client=FakeRedis(server=FakeServer()) 28 | ) 29 | self.assertIsInstance(udf, UDFFactory.get_udf_cls("staticthresh")) 30 | 31 | 32 | class TestServerFactory(unittest.TestCase): 33 | def test_get_cls(self): 34 | server_cls = ServerFactory.get_server_cls("sync") 35 | self.assertEqual(server_cls.__name__, "MapServer") 36 | 37 | def test_get_cls_err(self): 38 | with self.assertRaises(ValueError): 39 | ServerFactory.get_server_cls("some_server") 40 | 41 | def test_get_instance(self): 42 | server = ServerFactory.get_server_instance("multiproc", mapper_instance=lambda x: x) 43 | self.assertIsInstance(server, ServerFactory.get_server_cls("multiproc")) 44 | 45 | 46 | if __name__ == "__main__": 47 | unittest.main() 48 | -------------------------------------------------------------------------------- /numalogic/base.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | """Base classes for all models and transforms.""" 13 | 14 | 15 | from abc import ABCMeta 16 | 17 | import numpy.typing as npt 18 | import pytorch_lightning as pl 19 | from sklearn.base import TransformerMixin, OutlierMixin 20 | 21 | 22 | class BaseTransformer(TransformerMixin): 23 | """Base class for all transformer classes.""" 24 | 25 | pass 26 | 27 | 28 | class StatelessTransformer(BaseTransformer): 29 | """Base class for stateless transforms.""" 30 | 31 | def transform(self, input_: npt.NDArray, **__): 32 | """Implement the transform method.""" 33 | raise NotImplementedError("transform method not implemented") 34 | 35 | def fit(self, _: npt.NDArray): 36 | """Fit method does nothing for stateless transforms.""" 37 | return self 38 | 39 | def fit_transform(self, input_: npt.NDArray, _=None, **__): 40 | """Return the result of the transform method.""" 41 | return self.transform(input_) 42 | 43 | 44 | class TorchModel(pl.LightningModule, metaclass=ABCMeta): 45 | """Base class for all Pytorch based models.""" 46 | 47 | pass 48 | 49 | 50 | class BaseThresholdModel(OutlierMixin): 51 | """Base class for all threshold models.""" 52 | 53 | pass 54 | -------------------------------------------------------------------------------- /docs/post-processing.md: -------------------------------------------------------------------------------- 1 | # Post Processing 2 | After the raw scores have been generated, we might need to do some additional postprocessing, 3 | for various reasons. 4 | 5 | ### Tanh Score Normalization 6 | Tanh normalization step is an optional step, where we normalize the anomalies between 0-10. This is mostly to make the scores more understandable. 7 | 8 | ```python 9 | import numpy as np 10 | from numalogic.transforms import tanh_norm 11 | 12 | raw_anomaly_score = np.random.randn(10, 2) 13 | test_anomaly_score_norm = tanh_norm(raw_anomaly_score) 14 | ``` 15 | 16 | A scikit-learn compatible API is also available. 17 | 18 | ```python 19 | import numpy as np 20 | from numalogic.transforms import TanhNorm 21 | 22 | raw_score = np.random.randn(10, 2) 23 | 24 | norm = TanhNorm(scale_factor=10, smooth_factor=10) 25 | norm_score = norm.fit_transform(raw_score) 26 | ``` 27 | 28 | ### Exponentially Weighted Moving Average 29 | The Exponentially Weighted Moving Average (EWMA) serves as an effective smoothing function, 30 | emphasizing the importance of more recent anomaly scores over those of previous elements within a sliding window. 31 | 32 | This approach proves particularly beneficial in streaming inference scenarios, as it allows for 33 | earlier increases in anomaly scores when a new outlier data point is encountered. 34 | Consequently, the EMA enables a more responsive and dynamic assessment of streaming data, 35 | facilitating timely detection and response to potential anomalies. 36 | 37 | ```python 38 | import numpy as np 39 | from numalogic.transforms import ExpMovingAverage 40 | 41 | raw_score = np.array([1.0, 1.5, 1.2, 3.5, 2.7, 5.6, 7.1, 6.9, 4.2, 1.1]).reshape(-1, 1) 42 | 43 | postproc_clf = ExpMovingAverage(beta=0.5) 44 | out = postproc_clf.transform(raw_score) 45 | 46 | # out: [[1.3], [1.433], [1.333], [2.473], [2.591], [4.119], [5.621], [6.263], [5.229], [3.163]] 47 | ``` 48 | -------------------------------------------------------------------------------- /tests/connectors/utils/aws/test_sts_client_manager.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch, MagicMock 2 | from numalogic.connectors.utils.aws.sts_client_manager import STSClientManager 3 | from datetime import datetime, timedelta, timezone 4 | 5 | 6 | @patch("numalogic.connectors.utils.aws.sts_client_manager.boto3.client") 7 | def test_STSClientManager(boto3_client_mock): 8 | # Prepare the mock methods 9 | mock_sts_client = MagicMock() 10 | mock_sts_client.assume_role.return_value = { 11 | "Credentials": { 12 | "AccessKeyId": "test_key", 13 | "SecretAccessKey": "test_access_key", 14 | "SessionToken": "test_token", 15 | "Expiration": (datetime.now(timezone.utc) + timedelta(hours=1)), 16 | } 17 | } 18 | boto3_client_mock.return_value = mock_sts_client 19 | 20 | manager = STSClientManager() 21 | 22 | # Test assume_role 23 | role_arn = "test_arn" 24 | role_session_name = "test_session" 25 | manager.assume_role(role_arn, role_session_name) 26 | mock_sts_client.assume_role.assert_called_once_with( 27 | RoleArn=role_arn, RoleSessionName=role_session_name, DurationSeconds=3600 28 | ) 29 | assert manager.credentials == mock_sts_client.assume_role.return_value["Credentials"] 30 | 31 | # Test is_token_about_to_expire 32 | assert manager.is_token_about_to_expire() is False 33 | 34 | # Test get_credentials 35 | credentials = manager.get_credentials(role_arn, role_session_name) 36 | assert manager.credentials == mock_sts_client.assume_role.return_value["Credentials"] 37 | assert credentials == mock_sts_client.assume_role.return_value["Credentials"] 38 | 39 | # Test renew of credentials 40 | manager.credentials["Expiration"] = datetime.now(timezone.utc) 41 | credentials = manager.get_credentials(role_arn, role_session_name) 42 | assert credentials == mock_sts_client.assume_role.return_value["Credentials"] 43 | -------------------------------------------------------------------------------- /numalogic/connectors/utils/aws/exceptions.py: -------------------------------------------------------------------------------- 1 | class AWSException(Exception): 2 | """Custom exception class for grouping all AWS Exceptions together.""" 3 | 4 | pass 5 | 6 | 7 | class UnRecognizedAWSClientException(AWSException): 8 | """ 9 | Custom exception class for handling unrecognized AWS clients. 10 | 11 | This exception is raised when an unrecognized AWS client is requested from the 12 | Boto3ClientManager class. 13 | 14 | Usage: This exception can be raised when attempting to retrieve an AWS client that is not 15 | recognized by the Boto3ClientManager. 16 | 17 | """ 18 | 19 | pass 20 | 21 | 22 | class UnRecognizedDatabaseTypeException(AWSException): 23 | """ 24 | Exception raised when an unrecognized database type is encountered. 25 | 26 | This exception is raised when a database type is encountered that is not recognized or 27 | supported by the application. It serves as a way to handle and communicate errors related to 28 | unrecognized database types. 29 | 30 | Usage: This exception can be raised when attempting to connect to a database with an 31 | unrecognized type, or when performing operations on a database with an unrecognized type. It 32 | can be caught and handled to provide appropriate error messages or take necessary actions. 33 | """ 34 | 35 | pass 36 | 37 | 38 | class UnRecognizedDatabaseServiceProviderException(Exception): 39 | """ 40 | Exception raised when an unrecognized database service provider is encountered. 41 | 42 | This exception is raised when a database service provider is not recognized or supported by 43 | the application. It can be used to handle cases where a specific database service provider 44 | is required, but the provided one is not supported. 45 | 46 | 47 | Usage: raise UnRecognizedDatabaseServiceProviderException("The provided database service 48 | provider is not recognized or supported.") 49 | """ 50 | 51 | pass 52 | -------------------------------------------------------------------------------- /tests/connectors/rds/test_rds.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | 3 | import pytest 4 | from pandas import DataFrame 5 | 6 | from numalogic.connectors.utils.aws.config import RDSConnectionConfig 7 | from numalogic.connectors.rds._rds import RDSFetcher 8 | import pandas as pd 9 | 10 | 11 | @pytest.fixture(autouse=True) 12 | def mock_db_config(): 13 | return RDSConnectionConfig() 14 | 15 | 16 | @pytest.fixture(autouse=True) 17 | def data_fetcher(mock_db_config): 18 | return RDSFetcher(db_config=mock_db_config) 19 | 20 | 21 | def test_init(data_fetcher, mock_db_config): 22 | assert data_fetcher.db_config == mock_db_config 23 | 24 | 25 | def test_fetch(mocker, data_fetcher): 26 | mock_data = pd.DataFrame(data={"col1": ["value1", "value2"], "col2": ["value3", "value4"]}) 27 | 28 | mocker.patch.object(RDSFetcher, "fetch", return_value=mock_data) 29 | 30 | result = data_fetcher.fetch("SELECT * FROM table") 31 | 32 | assert isinstance(result, pd.DataFrame) 33 | 34 | 35 | def test_execute_query(mocker): 36 | rds_config = RDSConnectionConfig(database_type="mysql") 37 | rds_fetcher = RDSFetcher(db_config=rds_config) 38 | mocker.patch.object(rds_fetcher.fetcher, "execute_query", return_value=DataFrame()) 39 | result = rds_fetcher.fetch("SELECT * FROM table", datetime_column_name="eventdatetime") 40 | assert result.empty == DataFrame().empty 41 | 42 | 43 | def test_rds_fetcher_fetch(): 44 | rds_config = RDSConnectionConfig(database_type="mysql") 45 | rds_fetcher = RDSFetcher(db_config=rds_config) 46 | with patch.object(rds_fetcher.fetcher, "execute_query") as mock_query: 47 | mock_query.return_value = pd.DataFrame({"test": [1, 2, 3]}) 48 | result = rds_fetcher.fetch("SELECT * FROM test", "test") 49 | mock_query.assert_called_once_with("SELECT * FROM test") 50 | assert not result.empty 51 | 52 | 53 | def test_raw_fetch(data_fetcher): 54 | with pytest.raises(NotImplementedError): 55 | data_fetcher.raw_fetch() 56 | -------------------------------------------------------------------------------- /tests/udfs/test_pipeline.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from datetime import datetime 4 | from omegaconf import OmegaConf 5 | from orjson import orjson 6 | import pytest 7 | 8 | from numalogic._constants import TESTS_DIR 9 | from numalogic.udfs import PipelineConf, MetricsLoader 10 | from numalogic.udfs.payloadtx import PayloadTransformer 11 | from tests.udfs.utility import input_json_from_file 12 | 13 | MetricsLoader().load_metrics( 14 | config_file_path=f"{TESTS_DIR}/udfs/resources/numalogic_udf_metrics.yaml" 15 | ) 16 | logging.basicConfig(level=logging.DEBUG) 17 | KEYS = ["service-mesh", "1", "2"] 18 | DATUM = input_json_from_file(os.path.join(TESTS_DIR, "udfs", "resources", "data", "stream.json")) 19 | 20 | DATUM_KW = { 21 | "event_time": datetime.now(), 22 | "watermark": datetime.now(), 23 | } 24 | 25 | 26 | @pytest.fixture 27 | def setup(): 28 | _given_conf = OmegaConf.load(os.path.join(TESTS_DIR, "udfs", "resources", "_config.yaml")) 29 | _given_conf_2 = OmegaConf.load(os.path.join(TESTS_DIR, "udfs", "resources", "_config2.yaml")) 30 | schema = OmegaConf.structured(PipelineConf) 31 | pl_conf = PipelineConf(**OmegaConf.merge(schema, _given_conf)) 32 | pl_conf_2 = PipelineConf(**OmegaConf.merge(schema, _given_conf_2)) 33 | udf1 = PayloadTransformer(pl_conf=pl_conf) 34 | udf2 = PayloadTransformer(pl_conf=pl_conf_2) 35 | udf1.register_conf("druid-config", pl_conf.stream_confs["druid-config"]) 36 | udf2.register_conf("druid-config", pl_conf_2.stream_confs["druid-config"]) 37 | return udf1, udf2 38 | 39 | 40 | def test_pipeline_1(setup): 41 | msgs = setup[0](KEYS, DATUM) 42 | assert 2 == len(msgs) 43 | for msg in msgs: 44 | data_payload = orjson.loads(msg.value) 45 | assert data_payload["pipeline_id"] 46 | 47 | 48 | def test_pipeline_2(setup): 49 | msgs = setup[1](KEYS, DATUM) 50 | assert 1 == len(msgs) 51 | for msg in msgs: 52 | data_payload = orjson.loads(msg.value) 53 | assert data_payload["pipeline_id"] 54 | -------------------------------------------------------------------------------- /tests/connectors/utils/aws/test_config.py: -------------------------------------------------------------------------------- 1 | from numalogic.connectors.utils.aws.config import ( 2 | DatabaseTypes, 3 | AWSConfig, 4 | SSLConfig, 5 | RDBMSConfig, 6 | RDSConnectionConfig, 7 | ) 8 | 9 | 10 | def test_aws_config(): 11 | config = AWSConfig( 12 | aws_assume_role_arn="arn:aws:iam::123456789012:role/roleName", 13 | aws_assume_role_session_name="Session", 14 | ) 15 | assert config.aws_assume_role_arn == "arn:aws:iam::123456789012:role/roleName" 16 | assert config.aws_assume_role_session_name == "Session" 17 | 18 | 19 | def test_ssl_config(): 20 | ssl = SSLConfig(ca="path_to_ca") 21 | assert ssl.ca == "path_to_ca" 22 | 23 | 24 | def test_rdbms_config(): 25 | rdbms = RDBMSConfig( 26 | endpoint="localhost", 27 | port=3306, 28 | database_name="testdb", 29 | database_username="user", 30 | database_password="password", 31 | database_connection_timeout=300, 32 | database_type=DatabaseTypes.MYSQL, 33 | ssl_enabled=True, 34 | ssl=SSLConfig(ca="path_to_ca"), 35 | ) 36 | assert rdbms.endpoint == "localhost" 37 | assert rdbms.database_name == "testdb" 38 | assert rdbms.ssl.ca == "path_to_ca" 39 | 40 | 41 | def test_rds_config(): 42 | rds = RDSConnectionConfig( 43 | aws_assume_role_arn="arn:aws:iam::123456789012:role/roleName", 44 | aws_assume_role_session_name="Session", 45 | aws_region="us-west-2", 46 | aws_rds_use_iam=True, 47 | endpoint="localhost", 48 | port=3306, 49 | database_name="testdb", 50 | database_username="user", 51 | database_password="password", 52 | database_connection_timeout=300, 53 | database_type=DatabaseTypes.MYSQL, 54 | ssl_enabled=True, 55 | ssl=SSLConfig(ca="path_to_ca"), 56 | ) 57 | assert rds.aws_assume_role_arn == "arn:aws:iam::123456789012:role/roleName" 58 | assert rds.aws_region == "us-west-2" 59 | assert rds.endpoint == "localhost" 60 | -------------------------------------------------------------------------------- /docs/assets/numaproj.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /examples/multi_udf/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # builder: install needed dependencies 3 | #################################################################################################### 4 | 5 | FROM python:3.10-slim-bullseye AS builder 6 | 7 | ENV PYTHONFAULTHANDLER=1 \ 8 | PYTHONUNBUFFERED=1 \ 9 | PYTHONHASHSEED=random \ 10 | PIP_NO_CACHE_DIR=on \ 11 | PIP_DISABLE_PIP_VERSION_CHECK=on \ 12 | PIP_DEFAULT_TIMEOUT=100 \ 13 | PYSETUP_PATH="/opt/pysetup" \ 14 | VENV_PATH="/opt/pysetup/.venv" 15 | 16 | ENV PATH="$VENV_PATH/bin:$PATH" 17 | 18 | RUN apt-get update \ 19 | && apt-get install --no-install-recommends -y \ 20 | curl \ 21 | wget \ 22 | # deps for building python deps 23 | build-essential \ 24 | && apt-get install -y git \ 25 | && apt-get clean && rm -rf /var/lib/apt/lists/* \ 26 | \ 27 | # install dumb-init 28 | && wget -O /dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_x86_64 \ 29 | && chmod +x /dumb-init 30 | 31 | #################################################################################################### 32 | # mlflow: used for running the mlflow server 33 | #################################################################################################### 34 | FROM builder AS mlflow 35 | 36 | WORKDIR $PYSETUP_PATH 37 | RUN pip3 install "mlflow==2.0.1" 38 | 39 | ADD . /app 40 | WORKDIR /app 41 | 42 | RUN chmod +x entry.sh 43 | 44 | ENTRYPOINT ["/dumb-init", "--"] 45 | CMD ["/app/entry.sh"] 46 | 47 | EXPOSE 5000 48 | 49 | #################################################################################################### 50 | # udf: used for running the udf vertices 51 | #################################################################################################### 52 | FROM builder AS udf 53 | 54 | WORKDIR $PYSETUP_PATH 55 | COPY ./requirements.txt ./ 56 | RUN pip3 install -r requirements.txt 57 | 58 | ADD . /app 59 | WORKDIR /app 60 | 61 | RUN chmod +x entry.sh 62 | 63 | ENTRYPOINT ["/dumb-init", "--"] 64 | CMD ["/app/entry.sh"] 65 | 66 | EXPOSE 5000 67 | -------------------------------------------------------------------------------- /docs/data-generator.md: -------------------------------------------------------------------------------- 1 | # Data Generator 2 | 3 | Numalogic provides a data generator to create some synthetic time series data, that can be used as train or test data sets. 4 | 5 | Using the synthetic data, we can: 6 | 7 | 1. Compare and evaluate different ML algorithms, since we have labeled anomalies 8 | 2. Understand different types of anomalies, and our models' performance on each of them 9 | 3. Recreate realtime scenarios 10 | 11 | ### Generate multivariate timeseries 12 | 13 | ```python 14 | from numalogic.synthetic import SyntheticTSGenerator 15 | 16 | ts_generator = SyntheticTSGenerator( 17 | seq_len=8000, 18 | num_series=3, 19 | freq="T", 20 | primary_period=720, 21 | secondary_period=6000, 22 | seasonal_ts_prob=0.8, 23 | baseline_range=(200.0, 350.0), 24 | slope_range=(-0.001, 0.01), 25 | amplitude_range=(10, 75), 26 | cosine_ratio_range=(0.5, 0.9), 27 | noise_range=(5, 15), 28 | ) 29 | 30 | # shape: (8000, 3) with column names [s1, s2, s3] 31 | ts_df = ts_generator.gen_tseries() 32 | 33 | # Split into test and train 34 | train_df, test_df = ts_generator.train_test_split(ts_df, test_size=1000) 35 | ``` 36 | ![Train and Test sets](./assets/train_test.png) 37 | 38 | ### Inject anomalies 39 | 40 | Now, once we generate the synthetic data like above, we can inject anomalies into the test data set using `AnomalyGenerator`. 41 | 42 | `AnomalyGenerator` supports the following types of anomalies: 43 | 44 | 1. global: Outliers in the global context 45 | 2. contextual: Outliers only in the seasonal context 46 | 3. causal: Outliers caused by a temporal causal effect 47 | 4. collective: Outliers present simultaneously in two or more time series 48 | 49 | You can also use `anomaly_ratio` to adjust the ratio of anomalous data points wrt number of samples. 50 | 51 | ```python 52 | from numalogic.synthetic import AnomalyGenerator 53 | 54 | # columns to inject anomalies 55 | injected_cols = ["s1", "s2"] 56 | anomaly_generator = AnomalyGenerator( 57 | train_df, anomaly_type="contextual", anomaly_ratio=0.3 58 | ) 59 | outlier_test_df = anomaly_generator.inject_anomalies( 60 | test_df, cols=injected_cols, impact=1.5 61 | ) 62 | ``` 63 | 64 | ![Outliers](./assets/outliers.png) 65 | -------------------------------------------------------------------------------- /tests/transforms/test_postprocess.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn.pipeline import make_pipeline 3 | import pytest 4 | 5 | from numalogic.transforms import ( 6 | tanh_norm, 7 | TanhNorm, 8 | ExpMovingAverage, 9 | SigmoidNorm, 10 | expmov_avg_aggregator, 11 | ) 12 | 13 | 14 | def test_tanh_norm_func(): 15 | arr = np.arange(10) 16 | scores = tanh_norm(arr) 17 | assert sum(scores) == pytest.approx(39.52, 0.01) # places=2 18 | 19 | 20 | def test_tanh_norm_clf(): 21 | arr = np.arange(10).reshape(5, 2) 22 | clf = TanhNorm() 23 | scores = clf.fit_transform(arr) 24 | 25 | assert arr.shape == scores.shape 26 | assert np.sum(scores) == pytest.approx(39.52, 0.01) # places=2 27 | 28 | 29 | def test_exp_mov_avg_estimator(): 30 | beta = 0.9 31 | arr = np.arange(1, 11).reshape(-1, 1) 32 | clf = ExpMovingAverage(beta) 33 | out = clf.fit_transform(arr) 34 | 35 | expected = expmov_avg_aggregator(arr, beta) 36 | 37 | assert arr.shape == out.shape 38 | assert expected == pytest.approx(out[-1].item(), 0.01) # places=2 39 | assert out.data.c_contiguous 40 | 41 | 42 | def test_exp_mov_avg_estimator_err(): 43 | with pytest.raises(ValueError): 44 | ExpMovingAverage(1.1) 45 | 46 | with pytest.raises(ValueError): 47 | ExpMovingAverage(0.0) 48 | 49 | with pytest.raises(ValueError): 50 | ExpMovingAverage(1.0) 51 | 52 | 53 | def test_exp_mov_avg_agg(): 54 | arr = np.arange(1, 11) 55 | val = expmov_avg_aggregator(arr, 0.9) 56 | assert isinstance(val, float) 57 | assert val < 10 58 | 59 | 60 | def test_exp_mov_avg_agg_err(): 61 | arr = np.arange(1, 11) 62 | with pytest.raises(ValueError): 63 | expmov_avg_aggregator(arr, 1.01) 64 | 65 | 66 | def test_postproc_pl(): 67 | x = np.arange(1, 11).reshape(-1, 1) 68 | pl = make_pipeline(TanhNorm(), ExpMovingAverage(0.9)) 69 | out = pl.transform(x) 70 | assert x.shape == out.shape 71 | 72 | 73 | def test_sig_norm(): 74 | x = np.arange(1, 11).reshape(-1, 1) 75 | clf = SigmoidNorm() 76 | out = clf.fit_transform(x) 77 | assert x.shape == out.shape 78 | assert out.data.c_contiguous 79 | assert np.all(out >= 0) 80 | assert np.all(out <= 10) 81 | -------------------------------------------------------------------------------- /numalogic/models/vae/base.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | 3 | import torch.nn.functional as F 4 | from torch import Tensor, optim 5 | 6 | from numalogic.base import TorchModel 7 | 8 | 9 | def _init_criterion(loss_fn: str) -> Callable: 10 | if loss_fn == "huber": 11 | return F.huber_loss 12 | if loss_fn == "l1": 13 | return F.l1_loss 14 | if loss_fn == "mse": 15 | return F.mse_loss 16 | raise ValueError(f"Unsupported loss function provided: {loss_fn}") 17 | 18 | 19 | class BaseVAE(TorchModel): 20 | """ 21 | Abstract Base class for all Pytorch based variational autoencoder models. 22 | 23 | Args: 24 | ---- 25 | lr: learning rate (default: 3e-4) 26 | weight_decay: weight decay factor weight for regularization (default: 0.0) 27 | loss_fn: loss function used to train the model 28 | supported values include: {mse, l1, huber} 29 | """ 30 | 31 | def __init__( 32 | self, 33 | lr: float = 3e-4, 34 | weight_decay: float = 0.0, 35 | loss_fn: str = "mse", 36 | ): 37 | super().__init__() 38 | self._lr = lr 39 | self.weight_decay = weight_decay 40 | self.criterion = _init_criterion(loss_fn) 41 | 42 | def configure_shape(self, x: Tensor) -> Tensor: 43 | """Method to configure the batch shape for each type of model architecture.""" 44 | return x 45 | 46 | def configure_optimizers(self) -> dict: 47 | optimizer = optim.Adam(self.parameters(), lr=self._lr, weight_decay=self.weight_decay) 48 | return {"optimizer": optimizer} 49 | 50 | def get_reconstruction_loss(self, batch: Tensor, reduction: str = "sum"): 51 | _, recon = self.forward(batch) 52 | return self.criterion(batch, recon, reduction=reduction) 53 | 54 | def validation_step(self, batch: Tensor, batch_idx: int) -> Tensor: 55 | """Validation step for the model.""" 56 | loss = self.get_reconstruction_loss(batch) 57 | self.log("val_loss", loss) 58 | return loss 59 | 60 | def predict_step(self, batch: Tensor, batch_idx: int, dataloader_idx: int = 0) -> Tensor: 61 | """Prediction step for the model.""" 62 | return self.get_reconstruction_loss(batch, reduction="none") 63 | -------------------------------------------------------------------------------- /tests/udfs/resources/rds_config2.yaml: -------------------------------------------------------------------------------- 1 | stream_confs: 2 | druid-config: 3 | config_id: "druid-config" 4 | source: "rds" 5 | composite_keys: [ 'service-mesh', '1', '2' ] 6 | window_size: 10 7 | ml_pipelines: 8 | pipeline1: 9 | pipeline_id: "pipeline1" 10 | metrics: [ "failed" , "degraded" ] 11 | numalogic_conf: 12 | model: 13 | name: "VanillaAE" 14 | conf: 15 | seq_len: 10 16 | n_features: 2 17 | preprocess: 18 | - name: "LogTransformer" 19 | stateful: false 20 | conf: 21 | add_factor: 5 22 | - name: "StandardScaler" 23 | stateful: true 24 | threshold: 25 | name: "MahalanobisThreshold" 26 | conf: 27 | max_outlier_prob: 0.08 28 | trainer: 29 | train_hours: 3 30 | min_train_size: 100 31 | transforms: 32 | - name: "DataClipper" 33 | conf: 34 | lower: [0.0,0.0] 35 | pltrainer_conf: 36 | accelerator: cpu 37 | max_epochs: 1 38 | 39 | redis_conf: 40 | url: "isbsvc-redis-isbs-redis-svc.oss-analytics-numalogicosamfci-usw2-e2e.svc" 41 | port: 26379 42 | expiry: 360 43 | master_name: "mymaster" 44 | 45 | rds_conf: 46 | connection_conf: 47 | aws_assume_role_arn: "arn:aws:iam::123456789:role/ml_iam_role" 48 | aws_assume_role_session_name: "ml_pipeline_reader" 49 | endpoint: "localhost1" 50 | port: 3306 51 | database_name: "ml_poc" 52 | database_username: "root" 53 | database_password: "admin123" 54 | database_connection_timeout: 10 55 | database_type: "mysql" 56 | database_provider: "rds" 57 | aws_region: "us-west-2" 58 | aws_rds_use_iam: False 59 | ssl_enabled: False 60 | ssl: 61 | ca: "/usr/bin/ml_data/us-west-2-bundle.pem" 62 | id_fetcher: 63 | rds-config-pipeline1: 64 | dimensions : [ "col1" ] 65 | metrics : ["count"] 66 | datasource: "table-name" 67 | group_by: [ "timestamp", "col1" ] 68 | pivot: 69 | columns: [ "col2" ] 70 | datetime_column_name: "timestamp" 71 | hash_query_type: True 72 | hash_column_name: model_md5_hash 73 | -------------------------------------------------------------------------------- /tests/udfs/utility.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import json 3 | 4 | import numpy as np 5 | from pynumaflow.mapper import Datum 6 | from sklearn.pipeline import make_pipeline 7 | 8 | from numalogic.config import PreprocessFactory 9 | from numalogic.models.autoencoder.variants import VanillaAE 10 | from numalogic.tools.types import KeyedArtifact 11 | 12 | 13 | def input_json_from_file(data_path: str) -> Datum: 14 | """ 15 | Read input json from file and return Datum object 16 | Args: 17 | data_path: file path. 18 | 19 | Returns 20 | ------- 21 | Datum object 22 | 23 | """ 24 | with open(data_path) as fp: 25 | data = json.load(fp) 26 | if not isinstance(data, bytes): 27 | data = json.dumps(data).encode("utf-8") 28 | 29 | return Datum( 30 | keys=["service-mesh", "1", "2"], 31 | value=data, 32 | event_time=datetime.datetime.now(), 33 | watermark=datetime.datetime.now(), 34 | ) 35 | 36 | 37 | def store_in_redis(pl_conf, registry): 38 | """Store preprocess artifacts in redis.""" 39 | for _pipeline_id, _ml_conf in pl_conf.stream_confs["druid-config"].ml_pipelines.items(): 40 | preproc_clfs = [] 41 | preproc_factory = PreprocessFactory() 42 | for _cfg in _ml_conf.numalogic_conf.preprocess: 43 | _clf = preproc_factory.get_instance(_cfg) 44 | preproc_clfs.append(_clf) 45 | if any(_conf.stateful for _conf in _ml_conf.numalogic_conf.preprocess): 46 | preproc_clf = make_pipeline(*preproc_clfs) 47 | preproc_clf.fit(np.asarray([[1, 3], [4, 6]])) 48 | registry.save_multiple( 49 | skeys=[*pl_conf.stream_confs["druid-config"].composite_keys], 50 | dict_artifacts={ 51 | "inference": KeyedArtifact( 52 | dkeys=[_pipeline_id, "AE"], artifact=VanillaAE(10), stateful=True 53 | ), 54 | "preproc": KeyedArtifact( 55 | dkeys=[ 56 | _pipeline_id, 57 | *[_conf.name for _conf in _ml_conf.numalogic_conf.preprocess], 58 | ], 59 | artifact=preproc_clf, 60 | stateful=True, 61 | ), 62 | }, 63 | ) 64 | -------------------------------------------------------------------------------- /tests/registry/test_serialize.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import pickle 3 | import timeit 4 | import unittest 5 | 6 | from sklearn.preprocessing import StandardScaler 7 | from torchinfo import summary 8 | 9 | from numalogic.models.autoencoder.variants import VanillaAE 10 | from numalogic.registry._serialize import loads, dumps 11 | 12 | 13 | LOGGER = logging.getLogger(__name__) 14 | logging.basicConfig(level=logging.INFO) 15 | 16 | 17 | class TestSerialize(unittest.TestCase): 18 | def test_dumps_loads1(self): 19 | model = VanillaAE(10) 20 | serialized_obj = dumps(model) 21 | deserialized_obj = loads(serialized_obj) 22 | self.assertEqual(str(summary(model)), str(summary(deserialized_obj))) 23 | 24 | def test_dumps_loads2(self): 25 | model = StandardScaler() 26 | model.mean_ = 1000 27 | serialized_obj = dumps(model) 28 | deserialized_obj = loads(serialized_obj) 29 | self.assertEqual(model.mean_, deserialized_obj.mean_) 30 | 31 | def test_benchmark_state_dict_vs_model(self): 32 | model = VanillaAE(10, 2) 33 | serialized_sd = dumps(model.state_dict()) 34 | serialized_obj = dumps(model) 35 | elapsed_obj = timeit.timeit(lambda: loads(serialized_obj), number=100) 36 | elapsed_sd = timeit.timeit(lambda: loads(serialized_sd), number=100) 37 | try: 38 | self.assertLess(elapsed_sd, elapsed_obj) 39 | except AssertionError: 40 | LOGGER.warning( 41 | "The state_dict time %.3f is more than the model time %.3f", 42 | elapsed_sd, 43 | elapsed_obj, 44 | ) 45 | 46 | def test_benchmark_protocol(self): 47 | model = VanillaAE(10, 2) 48 | serialized_default = dumps(model, pickle_protocol=1) 49 | serialized_highest = dumps(model, pickle_protocol=pickle.HIGHEST_PROTOCOL) 50 | elapsed_default = timeit.timeit(lambda: loads(serialized_default), number=1000) 51 | elapsed_highest = timeit.timeit(lambda: loads(serialized_highest), number=1000) 52 | try: 53 | self.assertLess(elapsed_highest, elapsed_default) 54 | except AssertionError: 55 | LOGGER.warning( 56 | "The default protocol time %.3f is less than the highest protocol time %.3f", 57 | elapsed_default, 58 | elapsed_highest, 59 | ) 60 | -------------------------------------------------------------------------------- /numalogic/models/threshold/_median.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import numpy.typing as npt 3 | from typing_extensions import Self, Final 4 | 5 | from numalogic.base import BaseThresholdModel 6 | from numalogic.tools.exceptions import InvalidDataShapeError, ModelInitializationError 7 | 8 | _INLIER: Final[int] = 0 9 | _OUTLIER: Final[int] = 1 10 | _INPUT_DIMS: Final[int] = 2 11 | 12 | 13 | class MaxPercentileThreshold(BaseThresholdModel): 14 | """ 15 | Percentile based Thresholding estimator. 16 | 17 | Args: 18 | max_inlier_percentile: Max percentile greater than which will be treated as outlier 19 | min_threshold: Value to be used if threshold is less than this 20 | """ 21 | 22 | __slots__ = ("_max_percentile", "_min_thresh", "_thresh", "_is_fitted") 23 | 24 | def __init__( 25 | self, 26 | max_inlier_percentile: float = 96.0, 27 | min_threshold: float = 1e-4, 28 | ): 29 | super().__init__() 30 | self._max_percentile = max_inlier_percentile 31 | self._min_thresh = min_threshold 32 | self._thresh = None 33 | self._is_fitted = False 34 | 35 | @property 36 | def threshold(self): 37 | return self._thresh 38 | 39 | @staticmethod 40 | def _validate_input(x: npt.NDArray[float]) -> None: 41 | """Validate the input matrix shape.""" 42 | if x.ndim != _INPUT_DIMS: 43 | raise InvalidDataShapeError(f"Input matrix should have 2 dims, given shape: {x.shape}.") 44 | 45 | def fit(self, x: npt.NDArray[float]) -> Self: 46 | self._validate_input(x) 47 | self._thresh = np.percentile(x, self._max_percentile, axis=0) 48 | self._thresh[self._thresh < self._min_thresh] = self._min_thresh 49 | self._is_fitted = True 50 | return self 51 | 52 | def predict(self, x: npt.NDArray[float]) -> npt.NDArray[int]: 53 | if not self._is_fitted: 54 | raise ModelInitializationError("Model not fitted yet.") 55 | self._validate_input(x) 56 | 57 | y_hat = np.zeros(x.shape, dtype=int) 58 | y_hat[x > self._thresh] = _OUTLIER 59 | return y_hat 60 | 61 | def score_samples(self, x: npt.NDArray[float]) -> npt.NDArray[float]: 62 | if not self._is_fitted: 63 | raise ModelInitializationError("Model not fitted yet.") 64 | 65 | self._validate_input(x) 66 | return x / self._thresh 67 | -------------------------------------------------------------------------------- /examples/block_pipeline/src/inference.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | 4 | import numpy as np 5 | from numalogic.blocks import ( 6 | BlockPipeline, 7 | PreprocessBlock, 8 | NNBlock, 9 | ThresholdBlock, 10 | PostprocessBlock, 11 | ) 12 | from numalogic.models.autoencoder.variants import SparseVanillaAE 13 | from numalogic.models.threshold import StdDevThreshold 14 | from numalogic.udfs import NumalogicUDF 15 | from numalogic.registry import RedisRegistry 16 | from numalogic.tools.exceptions import RedisRegistryError 17 | from numalogic.transforms import TanhNorm 18 | from pynumaflow.function import Messages, Datum, Message 19 | from sklearn.preprocessing import StandardScaler 20 | 21 | from src.utils import RedisClient 22 | 23 | _LOGGER = logging.getLogger(__name__) 24 | 25 | 26 | class Inference(NumalogicUDF): 27 | """UDF to preprocess the input data for ML inference.""" 28 | 29 | def __init__(self, seq_len: int = 12, num_series: int = 1): 30 | super().__init__() 31 | self.seq_len = seq_len 32 | self.n_features = num_series 33 | self.registry = RedisRegistry(client=RedisClient().get_client()) 34 | 35 | def exec(self, keys: list[str], datum: Datum) -> Messages: 36 | # Load json data 37 | series = json.loads(datum.value)["data"] 38 | 39 | block_pl = BlockPipeline( 40 | PreprocessBlock(StandardScaler()), 41 | NNBlock( 42 | SparseVanillaAE(seq_len=self.seq_len, n_features=self.n_features), self.seq_len 43 | ), 44 | ThresholdBlock(StdDevThreshold()), 45 | PostprocessBlock(TanhNorm()), 46 | registry=self.registry, 47 | ) 48 | 49 | # Load the model from the registry 50 | try: 51 | block_pl.load(skeys=["blockpl"], dkeys=["sparsevanillae"]) 52 | except RedisRegistryError as warn: 53 | _LOGGER.warning("Error loading block pipeline: %r", warn) 54 | return Messages(Message(value=b"", tags=["train"])) 55 | 56 | # Run inference 57 | try: 58 | output = block_pl(np.asarray(series).reshape(-1, self.n_features)) 59 | except Exception: 60 | _LOGGER.exception("Error running block pipeline") 61 | return Messages(Message.to_drop()) 62 | 63 | anomaly_score = np.mean(output) 64 | return Messages(Message(tags=["out"], value=json.dumps({"score": anomaly_score}).encode())) 65 | -------------------------------------------------------------------------------- /examples/block_pipeline/src/train.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import pandas as pd 4 | from cachetools import TTLCache 5 | from numalogic.blocks import ( 6 | BlockPipeline, 7 | PreprocessBlock, 8 | NNBlock, 9 | ThresholdBlock, 10 | PostprocessBlock, 11 | ) 12 | from numalogic.models.autoencoder.variants import SparseVanillaAE 13 | from numalogic.models.threshold import StdDevThreshold 14 | from numalogic.udfs import NumalogicUDF 15 | from numalogic.registry import RedisRegistry 16 | from numalogic.transforms import TanhNorm 17 | from pynumaflow.function import Datum, Messages, Message 18 | from sklearn.preprocessing import StandardScaler 19 | 20 | from src.utils import RedisClient, TRAIN_DATA_PATH 21 | 22 | _LOGGER = logging.getLogger(__name__) 23 | 24 | 25 | class Train(NumalogicUDF): 26 | """UDF to train the model and save it to the registry.""" 27 | 28 | ttl_cache = TTLCache(maxsize=16, ttl=60) 29 | 30 | def __init__(self, seq_len: int = 12, num_series: int = 1): 31 | super().__init__() 32 | self.seq_len = seq_len 33 | self.n_features = num_series 34 | self.registry = RedisRegistry(client=RedisClient().get_client()) 35 | self._model_key = "sparsevanillae" 36 | 37 | def exec(self, keys: list[str], datum: Datum) -> Messages: 38 | """The train function here trains the model and saves it to the registry.""" 39 | # Check if a training message has been received very recently 40 | if self._model_key in self.ttl_cache: 41 | return Messages(Message.to_drop()) 42 | self.ttl_cache[self._model_key] = self._model_key 43 | 44 | # Load Training data 45 | data = pd.read_csv(TRAIN_DATA_PATH, index_col=None) 46 | 47 | # Define the block pipeline 48 | block_pl = BlockPipeline( 49 | PreprocessBlock(StandardScaler()), 50 | NNBlock( 51 | SparseVanillaAE(seq_len=self.seq_len, n_features=self.n_features), self.seq_len 52 | ), 53 | ThresholdBlock(StdDevThreshold()), 54 | PostprocessBlock(TanhNorm()), 55 | registry=self.registry, 56 | ) 57 | block_pl.fit(data) 58 | 59 | # Save the model to the registry 60 | block_pl.save(skeys=["blockpl"], dkeys=["sparsevanillae"]) 61 | _LOGGER.info("Model saved to registry") 62 | 63 | # Train vertex is the last vertex in the pipeline 64 | return Messages(Message.to_drop()) 65 | -------------------------------------------------------------------------------- /examples/block_pipeline/numa-pl.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: numaflow.numaproj.io/v1alpha1 2 | kind: InterStepBufferService 3 | metadata: 4 | name: redis-isbs # change it 5 | spec: 6 | redis: 7 | native: 8 | version: 7.0.11 9 | affinity: 10 | podAntiAffinity: 11 | preferredDuringSchedulingIgnoredDuringExecution: 12 | - podAffinityTerm: 13 | labelSelector: 14 | matchLabels: 15 | app.kubernetes.io/component: isbsvc 16 | numaflow.numaproj.io/isbsvc-name: redis-isbs # Change it 17 | topologyKey: topology.kubernetes.io/zone 18 | weight: 100 19 | persistence: 20 | accessMode: ReadWriteOnce 21 | volumeSize: 1Gi 22 | settings: 23 | redis: | 24 | maxmemory 4096mb 25 | 26 | 27 | --- 28 | apiVersion: numaflow.numaproj.io/v1alpha1 29 | kind: Pipeline 30 | metadata: 31 | name: blocks 32 | spec: 33 | watermark: 34 | disabled: false 35 | limits: 36 | readBatchSize: 10 37 | bufferMaxLength: 500 38 | bufferUsageLimit: 100 39 | vertices: 40 | - name: in 41 | source: 42 | http: {} 43 | - name: inference 44 | scale: 45 | min: 1 46 | udf: 47 | container: 48 | image: blockpl:v0.0.8 49 | env: 50 | - name: REDIS_AUTH 51 | valueFrom: 52 | secretKeyRef: 53 | name: isbsvc-redis-isbs-redis-auth 54 | key: redis-password 55 | args: 56 | - python 57 | - server.py 58 | - inference 59 | - name: train 60 | scale: 61 | min: 1 62 | udf: 63 | container: 64 | image: blockpl:v0.0.8 65 | env: 66 | - name: REDIS_AUTH 67 | valueFrom: 68 | secretKeyRef: 69 | name: isbsvc-redis-isbs-redis-auth 70 | key: redis-password 71 | args: 72 | - python 73 | - server.py 74 | - train 75 | - name: out 76 | scale: 77 | min: 1 78 | sink: 79 | log: {} 80 | edges: 81 | - from: in 82 | to: inference 83 | - conditions: 84 | tags: 85 | operator: or 86 | values: 87 | - train 88 | from: inference 89 | to: train 90 | - from: inference 91 | to: out 92 | conditions: 93 | tags: 94 | operator: or 95 | values: 96 | - out 97 | -------------------------------------------------------------------------------- /numalogic/udfs/__main__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import sys 4 | from typing import Final 5 | 6 | from numaprom.monitoring import start_metrics_server 7 | 8 | from numalogic._constants import ( 9 | DEFAULT_BASE_CONF_PATH, 10 | DEFAULT_APP_CONF_PATH, 11 | DEFAULT_METRICS_PORT, 12 | DEFAULT_METRICS_CONF_PATH, 13 | ) 14 | from numalogic.connectors.redis import get_redis_client_from_conf 15 | from numalogic.udfs import load_pipeline_conf, UDFFactory, ServerFactory, set_logger, set_metrics 16 | 17 | LOGGER = logging.getLogger(__name__) 18 | 19 | BASE_CONF_FILE_PATH: Final[str] = os.getenv("BASE_CONF_PATH", default=DEFAULT_BASE_CONF_PATH) 20 | APP_CONF_FILE_PATH: Final[str] = os.getenv("APP_CONF_PATH", default=DEFAULT_APP_CONF_PATH) 21 | METRICS_PORT: Final[int] = int(os.getenv("METRICS_PORT", default=DEFAULT_METRICS_PORT)) 22 | METRICS_ENABLED: Final[bool] = bool(int(os.getenv("METRICS_ENABLED", default="0"))) 23 | METRICS_CONF_PATH: Final[str] = os.getenv("METRICS_CONF_PATH", default=DEFAULT_METRICS_CONF_PATH) 24 | 25 | 26 | def init_server(step: str, server_type: str): 27 | """Initializes and returns the server.""" 28 | LOGGER.info("Merging config with file paths: %s, %s", BASE_CONF_FILE_PATH, APP_CONF_FILE_PATH) 29 | pipeline_conf = load_pipeline_conf(BASE_CONF_FILE_PATH, APP_CONF_FILE_PATH) 30 | LOGGER.info("Pipeline config: %s", pipeline_conf) 31 | 32 | LOGGER.info("Starting vertex with step: %s, server_type %s", step, server_type) 33 | if step == "mlpipeline": 34 | udf = UDFFactory.get_udf_instance(step, pl_conf=pipeline_conf) 35 | else: 36 | redis_client = get_redis_client_from_conf(pipeline_conf.redis_conf) 37 | udf = UDFFactory.get_udf_instance(step, r_client=redis_client, pl_conf=pipeline_conf) 38 | 39 | return ServerFactory.get_server_instance(server_type, mapper_instance=udf) 40 | 41 | 42 | def start_server() -> None: 43 | """Starts the pynumaflow server.""" 44 | set_logger() 45 | step = sys.argv[1] 46 | 47 | try: 48 | server_type = sys.argv[2] 49 | except (IndexError, TypeError): 50 | server_type = "sync" 51 | 52 | LOGGER.info("Running %s on %s server", step, server_type) 53 | 54 | if METRICS_ENABLED: 55 | set_metrics(conf_file=METRICS_CONF_PATH) 56 | start_metrics_server(METRICS_PORT) 57 | 58 | server = init_server(step, server_type) 59 | server.start() 60 | 61 | 62 | if __name__ == "__main__": 63 | start_server() 64 | -------------------------------------------------------------------------------- /tests/udfs/test_main.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest.mock import patch 3 | 4 | from omegaconf import OmegaConf 5 | from pynumaflow.mapper import MapServer, MapMultiprocServer 6 | 7 | from numalogic._constants import TESTS_DIR 8 | from numalogic.tools.exceptions import ConfigNotFoundError 9 | from numalogic.udfs import MetricsLoader 10 | 11 | BASE_CONFIG_PATH = f"{TESTS_DIR}/udfs/resources/_config3.yaml" 12 | APP_CONFIG_PATH = f"{TESTS_DIR}/udfs/resources/_config4.yaml" 13 | REDIS_AUTH = "123" 14 | MetricsLoader().load_metrics( 15 | config_file_path=f"{TESTS_DIR}/udfs/resources/numalogic_udf_metrics.yaml" 16 | ) 17 | 18 | 19 | class TestMainScript(unittest.TestCase): 20 | @patch.dict("os.environ", {"BASE_CONF_PATH": BASE_CONFIG_PATH, "REDIS_AUTH": REDIS_AUTH}) 21 | def test_init_server_01(self): 22 | from numalogic.udfs.__main__ import init_server 23 | 24 | server = init_server("preprocess", "sync") 25 | self.assertIsInstance(server, MapServer) 26 | 27 | @patch.dict("os.environ", {"BASE_CONF_PATH": BASE_CONFIG_PATH, "REDIS_AUTH": REDIS_AUTH}) 28 | def test_init_server_02(self): 29 | from numalogic.udfs.__main__ import init_server 30 | 31 | server = init_server("inference", "multiproc") 32 | self.assertIsInstance(server, MapMultiprocServer) 33 | 34 | def test_conf_loader(self): 35 | from numalogic.udfs import load_pipeline_conf 36 | 37 | plconf = load_pipeline_conf(BASE_CONFIG_PATH, APP_CONFIG_PATH) 38 | base_conf = OmegaConf.load(BASE_CONFIG_PATH) 39 | app_conf = OmegaConf.load(APP_CONFIG_PATH) 40 | 41 | self.assertListEqual( 42 | list(plconf.stream_confs), 43 | list(base_conf["stream_confs"]) + list(app_conf["stream_confs"]), 44 | ) 45 | 46 | def test_conf_loader_appconf_not_exist(self): 47 | from numalogic.udfs import load_pipeline_conf 48 | 49 | app_conf_path = "_random.yaml" 50 | plconf = load_pipeline_conf(BASE_CONFIG_PATH, app_conf_path) 51 | base_conf = OmegaConf.load(BASE_CONFIG_PATH) 52 | 53 | self.assertListEqual(list(plconf.stream_confs), list(base_conf["stream_confs"])) 54 | 55 | def test_conf_loader_err(self): 56 | from numalogic.udfs import load_pipeline_conf 57 | 58 | with self.assertRaises(ConfigNotFoundError): 59 | load_pipeline_conf("_random1.yaml", "_random2.yaml") 60 | 61 | 62 | if __name__ == "__main__": 63 | unittest.main() 64 | -------------------------------------------------------------------------------- /examples/multi_udf/src/udf/threshold.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from numalogic.udfs import NumalogicUDF 4 | from numalogic.registry import MLflowRegistry 5 | from pynumaflow.function import Messages, Message, Datum 6 | 7 | from src.utils import Payload 8 | 9 | LOGGER = logging.getLogger(__name__) 10 | TRACKING_URI = "http://mlflow-service.default.svc.cluster.local:5000" 11 | 12 | 13 | class Threshold(NumalogicUDF): 14 | """UDF to apply thresholding to the reconstruction error returned by the autoencoder.""" 15 | 16 | def __init__(self): 17 | super().__init__() 18 | self.registry = MLflowRegistry(tracking_uri=TRACKING_URI) 19 | 20 | @staticmethod 21 | def _handle_not_found(payload: Payload) -> Messages: 22 | """ 23 | Handles the case when the model is not found. 24 | If model not found, send it to trainer for training. 25 | """ 26 | LOGGER.warning("%s - Model not found. Training the model.", payload.uuid) 27 | 28 | # Convert Payload back to bytes and conditional forward to train vertex 29 | payload.is_artifact_valid = False 30 | return Messages(Message(keys=["train"], value=payload.to_json())) 31 | 32 | def exec(self, _: list[str], datum: Datum) -> Messages: 33 | """ 34 | UDF that applies thresholding to the reconstruction error returned by the autoencoder. 35 | 36 | For more information about the arguments, refer: 37 | https://github.com/numaproj/numaflow-python/blob/main/pynumaflow/function/_dtypes.py 38 | """ 39 | # Load data and convert bytes to Payload 40 | payload = Payload.from_json(datum.value) 41 | 42 | # Load the threshold model from registry 43 | thresh_clf_artifact = self.registry.load( 44 | skeys=["thresh_clf"], dkeys=["model"], artifact_type="sklearn" 45 | ) 46 | recon_err = payload.get_array().reshape(-1, 1) 47 | 48 | # Check if model exists for inference 49 | if (not thresh_clf_artifact) or (not payload.is_artifact_valid): 50 | return self._handle_not_found(payload) 51 | 52 | thresh_clf = thresh_clf_artifact.artifact 53 | payload.set_array(thresh_clf.predict(recon_err).tolist()) 54 | 55 | LOGGER.info("%s - Thresholding complete", payload.uuid) 56 | 57 | # Convert Payload back to bytes and conditional forward to postprocess vertex 58 | return Messages(Message(keys=["postprocess"], value=payload.to_json())) 59 | -------------------------------------------------------------------------------- /numalogic/connectors/utils/enum.py: -------------------------------------------------------------------------------- 1 | from enum import Enum, EnumMeta 2 | 3 | 4 | class MetaEnum(EnumMeta): 5 | """ 6 | The 'MetaEnum' class is a metaclass for creating custom Enum classes. It extends the 7 | 'EnumMeta' metaclass provided by the 'enum' module. 8 | 9 | Methods: - __contains__(cls, item): This method is used to check if an item is a valid 10 | member of the Enum class. It tries to create an instance of the Enum class with the given 11 | item and if it raises a ValueError, it means the item is not a valid member. Returns True if 12 | the item is a valid member, otherwise False. 13 | 14 | Note: This class should not be used directly, but rather as a metaclass for creating custom 15 | Enum classes. 16 | """ 17 | 18 | def __contains__(cls, item): 19 | """ 20 | Check if an item is a valid member of the Enum class. 21 | 22 | Parameters 23 | ---------- 24 | - cls: The Enum class. 25 | - item: The item to check. 26 | 27 | Returns 28 | ------- 29 | - True if the item is a valid member of the Enum class, otherwise False. 30 | 31 | Note: This method tries to create an instance of the Enum class with the given item. If 32 | it raises a ValueError, it means the item is not a valid member. 33 | """ 34 | try: 35 | cls(item) 36 | except ValueError: 37 | return False 38 | return True 39 | 40 | 41 | class BaseEnum(Enum, metaclass=MetaEnum): 42 | """ 43 | The 'BaseEnum' class is a custom Enum class that extends the 'Enum' class provided by the 44 | 'enum' module. It uses the 'MetaEnum' metaclass to add additional functionality to the Enum 45 | class. 46 | 47 | Methods 48 | ------- 49 | - list(cls): This class method returns a list of all the values of the Enum class. 50 | 51 | Note: This class should be used as a base class for creating custom Enum classes. 52 | """ 53 | 54 | @classmethod 55 | def list(cls): 56 | """ 57 | Return a list of all the values of the Enum class. 58 | 59 | Parameters 60 | ---------- 61 | - cls: The Enum class. 62 | 63 | Returns 64 | ------- 65 | - A list of all the values of the Enum class. 66 | 67 | Note: 68 | This method should be called on the Enum class itself, not on an instance of the class. 69 | """ 70 | return list(map(lambda c: c.value, cls)) 71 | -------------------------------------------------------------------------------- /numalogic/tools/types.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | from collections.abc import Sequence 12 | from typing import Union, TypeVar, ClassVar, NamedTuple 13 | 14 | from sklearn.base import BaseEstimator 15 | from torch import Tensor 16 | 17 | from numalogic.base import TorchModel, BaseThresholdModel, BaseTransformer 18 | 19 | try: 20 | from redis.client import Redis 21 | except ImportError: 22 | redis_client_t = TypeVar("redis_client_t") 23 | else: 24 | redis_client_t = TypeVar("redis_client_t", bound=Redis, covariant=True) 25 | 26 | artifact_t = TypeVar( 27 | "artifact_t", 28 | bound=Union[TorchModel, BaseThresholdModel, BaseTransformer, BaseEstimator], 29 | covariant=True, 30 | ) 31 | nn_model_t = TypeVar("nn_model_t", bound=TorchModel) 32 | state_dict_t = TypeVar("state_dict_t", bound=dict[str, Tensor], covariant=True) 33 | transform_t = TypeVar("transform_t", bound=Union[BaseTransformer, BaseEstimator], covariant=True) 34 | thresh_t = TypeVar("thresh_t", bound=BaseThresholdModel, covariant=True) 35 | META_T = TypeVar("META_T", bound=dict[str, Union[str, float, int, list, dict]]) 36 | META_VT = TypeVar("META_VT", str, int, float, list, dict) 37 | EXTRA_T = TypeVar("EXTRA_T", bound=dict[str, Union[str, list, dict]]) 38 | KEYS = TypeVar("KEYS", bound=Sequence[str], covariant=False) 39 | 40 | 41 | class KeyedArtifact(NamedTuple): 42 | r"""namedtuple for artifacts.""" 43 | 44 | dkeys: KEYS 45 | artifact: artifact_t 46 | stateful: bool = True 47 | 48 | 49 | class Singleton(type): 50 | r"""Helper metaclass to use as a Singleton class.""" 51 | 52 | _instances: ClassVar[dict] = {} 53 | 54 | def __call__(cls, *args, **kwargs): 55 | if cls not in cls._instances: 56 | cls._instances[cls] = super().__call__(*args, **kwargs) 57 | return cls._instances[cls] 58 | 59 | @classmethod 60 | def clear_instances(cls): 61 | cls._instances = {} 62 | -------------------------------------------------------------------------------- /numalogic/transforms/_postprocess.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | import numpy as np 13 | import numpy.typing as npt 14 | 15 | from numalogic.base import StatelessTransformer 16 | 17 | 18 | def tanh_norm(scores: npt.NDArray[float], scale_factor=10, smooth_factor=10) -> npt.NDArray[float]: 19 | """ 20 | Applies column wise tanh normalization to the input data. 21 | 22 | This is most commonly used to normalize the anomaly scores to a desired range. 23 | 24 | Args: 25 | ---- 26 | scores: feature array 27 | scale_factor: scale the output by this factor (default: 10) 28 | smooth_factor: factor to broaden out the linear range of the graph (default: 10) 29 | """ 30 | return scale_factor * np.tanh(scores / smooth_factor) 31 | 32 | 33 | class TanhNorm(StatelessTransformer): 34 | """ 35 | Apply tanh normalization to the input data. 36 | 37 | Args: 38 | ---- 39 | scale_factor: scale the output by this factor (default: 10) 40 | smooth_factor: factor to broaden out the linear range of the graph (default: 10) 41 | """ 42 | 43 | __slots__ = ("scale_factor", "smooth_factor") 44 | 45 | def __init__(self, scale_factor=10, smooth_factor=10): 46 | self.scale_factor = scale_factor 47 | self.smooth_factor = smooth_factor 48 | 49 | def transform(self, input_: npt.NDArray[float], **__) -> npt.NDArray[float]: 50 | return tanh_norm(input_, scale_factor=self.scale_factor, smooth_factor=self.smooth_factor) 51 | 52 | 53 | class SigmoidNorm(StatelessTransformer): 54 | def __init__(self, scale_factor: float = 10.0, smooth_factor: float = 0.5): 55 | super().__init__() 56 | self.scale_factor = scale_factor 57 | self.smooth_factor = smooth_factor 58 | 59 | def transform(self, x: npt.NDArray[float], **__) -> npt.NDArray[float]: 60 | return self.scale_factor / (1.0 + np.exp(5 - (self.smooth_factor * x))) 61 | -------------------------------------------------------------------------------- /tests/udfs/resources/rds_trainer_config_fetcher_conf1.yaml: -------------------------------------------------------------------------------- 1 | stream_confs: 2 | rds-config: 3 | config_id: "rds-config" 4 | source: "rds" 5 | composite_keys: [ 'service-mesh', '1', '2' ] 6 | window_size: 10 7 | ml_pipelines: 8 | pipeline1: 9 | pipeline_id: "pipeline1" 10 | metrics: [ "failed" , "degraded" ] 11 | numalogic_conf: 12 | model: 13 | name: "VanillaAE" 14 | conf: 15 | seq_len: 10 16 | n_features: 2 17 | preprocess: 18 | - name: "LogTransformer" 19 | stateful: false 20 | conf: 21 | add_factor: 5 22 | - name: "StandardScaler" 23 | stateful: true 24 | threshold: 25 | name: "MahalanobisThreshold" 26 | conf: 27 | max_outlier_prob: 0.08 28 | trainer: 29 | train_hours: 3 30 | min_train_size: 100 31 | transforms: 32 | - name: "DataClipper" 33 | conf: 34 | lower: [0.0,0.0] 35 | pltrainer_conf: 36 | accelerator: cpu 37 | max_epochs: 1 38 | 39 | redis_conf: 40 | url: "isbsvc-fci-redis-isbs-redis-svc.oss-analytics-numalogicosamfci-usw2-prd.svc" 41 | port: 26379 42 | expiry: 360 43 | master_name: "mymaster" 44 | 45 | registry_conf: 46 | name: "RedisRegistry" 47 | model_expiry_sec: 172800 48 | jitter_conf: 49 | jitter_sec: 900 50 | jitter_steps_sec: 120 51 | 52 | rds_conf: 53 | connection_conf: 54 | aws_assume_role_arn: "arn:aws:iam::123456789:role/ml_iam_role" 55 | aws_assume_role_session_name: "ml_pipeline_reader" 56 | endpoint: "localhost1" 57 | port: 3306 58 | database_name: "ml_poc" 59 | database_username: "root" 60 | database_password: "admin123" 61 | database_connection_timeout: 10 62 | database_type: "mysql" 63 | database_provider: "rds" 64 | aws_region: "us-west-2" 65 | aws_rds_use_iam: False 66 | ssl_enabled: False 67 | ssl: 68 | ca: "/usr/bin/ml_data/us-west-2-bundle.pem" 69 | id_fetcher: 70 | rds-config-pipeline1: 71 | dimensions: [ "cistatus" ] 72 | metrics: [ "count" ] 73 | datasource: "ml_poc.o11y_fci_ml" 74 | group_by: [ "timestamp", "cistatus" ] 75 | pivot: 76 | columns: [ "cistatus" ] 77 | datetime_column_name: "eventdatetime" 78 | hash_query_type: True 79 | hash_column_name: model_md5_hash 80 | 81 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Docker Image Push 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | workflow_dispatch: 8 | inputs: 9 | extra_tag: 10 | description: 'Tag suffix' 11 | required: false 12 | type: string 13 | 14 | jobs: 15 | build: 16 | runs-on: ubuntu-latest 17 | strategy: 18 | matrix: 19 | python-version: ["3.11"] 20 | 21 | steps: 22 | - uses: actions/checkout@v4 23 | 24 | - name: Docker Login 25 | uses: docker/login-action@v2.1.0 26 | with: 27 | registry: quay.io 28 | username: ${{ secrets.QUAYIO_USERNAME }} 29 | password: ${{ secrets.QUAYIO_PASSWORD }} 30 | 31 | - name: Docker Build 32 | env: 33 | QUAYIO_ORG: quay.io/numaio 34 | PLATFORM: linux/x86_64 35 | TARGET: numalogic/udf 36 | TAG_SUFFIX: ${{ inputs.extra_tag }} 37 | run: | 38 | type=$(basename $(dirname $GITHUB_REF)) 39 | tag=$(basename $GITHUB_REF) 40 | 41 | if [[ $type == "heads" ]]; then 42 | tag="$(basename $GITHUB_REF)v${{ env.version }}" 43 | fi 44 | 45 | echo "Tag: $tag" 46 | echo "Type: $type" 47 | echo "Tag suffix: $TAG_SUFFIX" 48 | 49 | if [[ -n $TAG_SUFFIX ]]; then 50 | tag="$(basename $GITHUB_REF)-${TAG_SUFFIX}" 51 | fi 52 | 53 | image_name="${QUAYIO_ORG}/numalogic/udf:${tag}" 54 | 55 | docker buildx build --platform ${PLATFORM} --build-arg INSTALL_EXTRAS='redis druid' --output "type=image,push=true" . -t $image_name 56 | 57 | - name: Docker RDS Build 58 | env: 59 | QUAYIO_ORG: quay.io/numaio 60 | PLATFORM: linux/x86_64 61 | TARGET: numalogic/udf 62 | TAG_SUFFIX: ${{ inputs.extra_tag }} 63 | run: | 64 | type=$(basename $(dirname $GITHUB_REF)) 65 | tag=$(basename $GITHUB_REF) 66 | 67 | if [[ $type == "heads" ]]; then 68 | tag="$(basename $GITHUB_REF)v${{ env.version }}" 69 | fi 70 | 71 | echo "Tag: $tag" 72 | echo "Type: $type" 73 | echo "Tag suffix: $TAG_SUFFIX" 74 | 75 | if [[ -n $TAG_SUFFIX ]]; then 76 | tag="$(basename $GITHUB_REF)-rds-${TAG_SUFFIX}" 77 | fi 78 | 79 | image_name="${QUAYIO_ORG}/numalogic/udf:rds-${tag}" 80 | 81 | docker buildx build --platform ${PLATFORM} --build-arg INSTALL_EXTRAS='redis rds' --output "type=image,push=true" . -t $image_name 82 | 83 | -------------------------------------------------------------------------------- /tests/udfs/resources/_config3.yaml: -------------------------------------------------------------------------------- 1 | stream_confs: 2 | odl-graphql: 3 | config_id: "odl-graphql" 4 | source: "prometheus" 5 | composite_keys: [ "namespace", "app" ] 6 | window_size: 12 7 | ml_pipelines: 8 | pipeline1: 9 | pipeline_id: "pipeline1" 10 | metrics: [ "namespace_app_rollouts_cpu_utilization", "namespace_app_rollouts_http_request_error_rate", "namespace_app_rollouts_memory_utilization" ] 11 | numalogic_conf: 12 | model: 13 | name: "Conv1dVAE" 14 | conf: 15 | seq_len: 12 16 | n_features: 3 17 | latent_dim: 1 18 | preprocess: 19 | - name: "StandardScaler" 20 | threshold: 21 | name: "MahalanobisThreshold" 22 | trainer: 23 | train_hours: 3 24 | min_train_size: 100 25 | pltrainer_conf: 26 | accelerator: cpu 27 | max_epochs: 5 28 | default: 29 | config_id: "default" 30 | source: "prometheus" 31 | composite_keys: [ "namespace", "app" ] 32 | window_size: 12 33 | ml_pipelines: 34 | default: 35 | pipeline_id: "default" 36 | metrics: ["namespace_app_rollouts_http_request_error_rate"] 37 | numalogic_conf: 38 | model: 39 | name: "SparseVanillaAE" 40 | conf: 41 | seq_len: 12 42 | n_features: 1 43 | preprocess: 44 | - name: "StandardScaler" 45 | threshold: 46 | name: "StdDevThreshold" 47 | trainer: 48 | train_hours: 1 49 | min_train_size: 100 50 | pltrainer_conf: 51 | accelerator: cpu 52 | max_epochs: 5 53 | pipeline1: 54 | pipeline_id: "pipeline1" 55 | metrics: [ "namespace_app_rollouts_http_request_error_rate" ] 56 | numalogic_conf: 57 | model: 58 | name: "Conv1dVAE" 59 | conf: 60 | seq_len: 12 61 | n_features: 1 62 | latent_dim: 1 63 | preprocess: 64 | - name: "StandardScaler" 65 | threshold: 66 | name: "MahalanobisThreshold" 67 | trainer: 68 | train_hours: 3 69 | min_train_size: 100 70 | pltrainer_conf: 71 | accelerator: cpu 72 | max_epochs: 5 73 | 74 | redis_conf: 75 | url: "http://localhost:6379" 76 | port: 26379 77 | expiry: 360 78 | master_name: "mymaster" 79 | 80 | prometheus_conf: 81 | url: "http://localhost:9090" 82 | scrape_interval: 30 83 | -------------------------------------------------------------------------------- /tests/config/test_optdeps.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | import unittest 13 | from unittest.mock import patch 14 | 15 | import fakeredis 16 | 17 | from numalogic.config import RegistryInfo 18 | from numalogic.tools.exceptions import UnknownConfigArgsError 19 | 20 | 21 | class TestOptionalDependencies(unittest.TestCase): 22 | def setUp(self) -> None: 23 | self.regconf = RegistryInfo( 24 | name="RedisRegistry", model_expiry_sec=10, extra_param=dict(ttl=50) 25 | ) 26 | 27 | @patch("numalogic.config.factory.getattr", side_effect=AttributeError) 28 | def test_not_installed_dep_01(self, _): 29 | from numalogic.config.factory import RegistryFactory 30 | 31 | model_factory = RegistryFactory() 32 | server = fakeredis.FakeServer() 33 | redis_cli = fakeredis.FakeStrictRedis(server=server, decode_responses=False) 34 | with self.assertRaises(ImportError): 35 | model_factory.get_cls("RedisRegistry")(redis_cli, **self.regconf.extra_param) 36 | 37 | @patch("numalogic.config.factory.getattr", side_effect=AttributeError) 38 | def test_not_installed_dep_02(self, _): 39 | from numalogic.config.factory import RegistryFactory 40 | 41 | model_factory = RegistryFactory() 42 | server = fakeredis.FakeServer() 43 | redis_cli = fakeredis.FakeStrictRedis(server=server, decode_responses=False) 44 | with self.assertRaises(ImportError): 45 | model_factory.get_instance(self.regconf)(redis_cli, **self.regconf.extra_param) 46 | 47 | def test_unknown_registry(self): 48 | from numalogic.config.factory import RegistryFactory 49 | 50 | model_factory = RegistryFactory() 51 | reg_conf = RegistryInfo(name="UnknownRegistry", model_expiry_sec=1) 52 | with self.assertRaises(UnknownConfigArgsError): 53 | model_factory.get_cls("UnknownRegistry") 54 | with self.assertRaises(UnknownConfigArgsError): 55 | model_factory.get_instance(reg_conf) 56 | 57 | 58 | if __name__ == "__main__": 59 | unittest.main() 60 | -------------------------------------------------------------------------------- /tests/udfs/resources/_config.yaml: -------------------------------------------------------------------------------- 1 | stream_confs: 2 | druid-config: 3 | config_id: "druid-config" 4 | source: "druid" 5 | composite_keys: [ 'service-mesh', '1', '2' ] 6 | window_size: 20 7 | ml_pipelines: 8 | pipeline1: 9 | pipeline_id: "pipeline1" 10 | metrics: [ "col1" , "col2" ] 11 | numalogic_conf: 12 | model: 13 | name: "VanillaAE" 14 | conf: 15 | seq_len: 20 16 | n_features: 1 17 | preprocess: 18 | - name: "FlattenVector" 19 | stateful: false 20 | conf: 21 | n_features: 2 22 | - name: "LogTransformer" 23 | stateful: false 24 | conf: 25 | add_factor: 5 26 | threshold: 27 | name: "StdDevThreshold" 28 | conf: 29 | min_threshold: 0.1 30 | score: 31 | feature_agg: 32 | method: MAX 33 | adjust: 34 | upper_limits: 35 | "col1": 23 36 | trainer: 37 | train_hours: 3 38 | min_train_size: 100 39 | transforms: 40 | - name: DataClipper 41 | conf: 42 | lower: [0.0, -inf] 43 | upper: [0.0, inf] 44 | pltrainer_conf: 45 | accelerator: cpu 46 | max_epochs: 5 47 | pipeline2: 48 | pipeline_id: "pipeline2" 49 | metrics: [ "col1" , "col2" ] 50 | numalogic_conf: 51 | model: 52 | name: "VanillaAE" 53 | conf: 54 | seq_len: 10 55 | n_features: 2 56 | preprocess: 57 | - name: "LogTransformer" 58 | stateful: false 59 | conf: 60 | add_factor: 5 61 | threshold: 62 | name: "StdDevThreshold" 63 | conf: 64 | min_threshold: 0.1 65 | score: 66 | feature_agg: 67 | method: MEAN 68 | adjust: 69 | upper_limits: 70 | "col1": 20 71 | "col2": 18 72 | 73 | 74 | redis_conf: 75 | url: "isbsvc-redis-isbs-redis-svc.oss-analytics-numalogicosamfci-usw2-e2e.svc" 76 | port: 26379 77 | expiry: 360 78 | master_name: "mymaster" 79 | 80 | druid_conf: 81 | url: "druid-endpoint" 82 | endpoint: "endpoint" 83 | id_fetcher: 84 | druid-config-pipeline1: 85 | dimensions: [ "col1" ] 86 | datasource: "table-name" 87 | group_by: [ "timestamp", "col1" ] 88 | pivot: 89 | columns: [ "col2" ] 90 | -------------------------------------------------------------------------------- /tests/udfs/resources/rds_trainer_config_fetcher_conf.yaml: -------------------------------------------------------------------------------- 1 | stream_confs: 2 | fciPluginAppInteractions: 3 | config_id: "fciPluginAppInteractions" 4 | source: "druid" 5 | composite_keys: [ "pluginAssetId", "assetId", "interactionName" ] 6 | window_size: 10 7 | ml_pipelines: 8 | metrics: 9 | pipeline_id: "metrics" 10 | metrics: [ "failed" , "degraded" ] 11 | numalogic_conf: 12 | model: 13 | name: "SparseVanillaAE" 14 | conf: 15 | seq_len: 20 16 | n_features: 1 17 | loss_fn: "mse" 18 | preprocess: 19 | - name: "FlattenVector" 20 | stateful: false 21 | conf: 22 | n_features: 2 23 | - name: "LogTransformer" 24 | stateful: false 25 | conf: 26 | add_factor: 2 27 | threshold: 28 | name: "MaxPercentileThreshold" 29 | conf: 30 | min_threshold: 0.1 31 | trainer: 32 | min_train_size: 180 33 | retrain_freq_hr: 8 34 | batch_size: 256 35 | train_hours: 240 36 | retry_sec: 600 37 | pltrainer_conf: 38 | max_epochs: 40 39 | 40 | redis_conf: 41 | url: "isbsvc-fci-redis-isbs-redis-svc.oss-analytics-numalogicosamfci-usw2-prd.svc" 42 | port: 26379 43 | expiry: 360 44 | master_name: "mymaster" 45 | 46 | registry_conf: 47 | name: "RedisRegistry" 48 | model_expiry_sec: 172800 49 | jitter_conf: 50 | jitter_sec: 900 51 | jitter_steps_sec: 120 52 | 53 | rds_conf: 54 | connection_conf: 55 | aws_assume_role_arn: "arn:aws:iam::123456789:role/ml_iam_role" 56 | aws_assume_role_session_name: "ml_pipeline_reader" 57 | endpoint: "localhost1" 58 | port: 3306 59 | database_name: "ml_poc" 60 | database_username: "root" 61 | database_password: "admin123" 62 | database_connection_timeout: 10 63 | database_type: "mysql" 64 | database_provider: "rds" 65 | aws_region: "us-west-2" 66 | aws_rds_use_iam: False 67 | ssl_enabled: False 68 | ssl: 69 | ca: "/usr/bin/ml_data/us-west-2-bundle.pem" 70 | fetcher: 71 | dimensions : ["cistatus"] 72 | metrics : ["count"] 73 | datasource: "ml_poc.o11y_fci_ml" 74 | group_by: [ "timestamp", "cistatus" ] 75 | pivot: 76 | columns: [ "cistatus" ] 77 | datetime_column_name: "eventdatetime" 78 | hash_query_type: True 79 | hash_column_name: model_md5_hash 80 | 81 | -------------------------------------------------------------------------------- /docs/ml-flow.md: -------------------------------------------------------------------------------- 1 | # MLflow 2 | 3 | Numalogic has built in support for Mlflow's tracking and logging system. 4 | 5 | ### Starting MLflow 6 | 7 | To start the [mlflow server on localhost](https://www.mlflow.org/docs/latest/tracking.html#scenario-1-mlflow-on-localhost), 8 | which has already been installed optionally via `poetry` dependency, run the following command. 9 | 10 | Replace the `{directory}` with the path you want to save the models. 11 | 12 | ```shell 13 | mlflow server \ 14 | --default-artifact-root {directory}/mlruns --serve-artifacts \ 15 | --backend-store-uri sqlite:///mlflow.db --host 0.0.0.0 --port 5000 16 | ``` 17 | 18 | Once the mlflow server has been started, you can navigate to http://127.0.0.1:5000/ to explore mlflow UI. 19 | 20 | ### Model saving 21 | 22 | Numalogic provides `MLflowRegistry`, to save and load models to/from MLflow. 23 | 24 | Here, `tracking_uri` is the uri where mlflow server is running. The `static_keys` and `dynamic_keys` are used to form a unique key for the model. 25 | 26 | The `artifact` would be the model or transformer object that needs to be saved. Artifact saving also takes in 'artifact_type' which is the type of the artifact being saved. Currently, 'pytorch', 'sklearn' and 'pyfunc' is supported. 27 | A dictionary of metadata can also be saved along with the artifact. 28 | ```python 29 | from numalogic.registry import MLflowRegistry 30 | from numalogic.models.autoencoder.variants import VanillaAE 31 | 32 | model = VanillaAE(seq_len=10) 33 | 34 | # static and dynamic keys are used to look up a model 35 | static_keys = ["model", "autoencoder"] 36 | dynamic_keys = ["vanilla", "seq10"] 37 | 38 | registry = MLflowRegistry(tracking_uri="http://0.0.0.0:5000") 39 | registry.save( 40 | skeys=static_keys, dkeys=dynamic_keys, artifact_type='pytorch', artifact=model, seq_len=10, lr=0.001 41 | ) 42 | ``` 43 | 44 | ### Model loading 45 | 46 | Once, the models are save to MLflow, the `load` function of `MLflowRegistry` can be used to load the model. Like how the artifacts were saved with 'artifact_type', the same type shall be passed to the `load` function as well. 47 | 48 | ```python 49 | from numalogic.registry import MLflowRegistry 50 | 51 | static_keys = ["model", "autoencoder"] 52 | dynamic_keys = ["vanilla", "seq10"] 53 | 54 | registry = MLflowRegistry(tracking_uri="http://0.0.0.0:8080") 55 | artifact_data = registry.load( 56 | skeys=static_keys, dkeys=dynamic_keys, artifact_type="pytorch" 57 | ) 58 | 59 | # get the model and metadata 60 | model = artifact_data.artifact 61 | model_metadata = artifact_data.metadata 62 | ``` 63 | 64 | For more details, please refer to [MLflow Model Registry](https://www.mlflow.org/docs/latest/model-registry.html#) 65 | -------------------------------------------------------------------------------- /numalogic/models/threshold/_std.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | import numpy as np 13 | from numpy.typing import NDArray 14 | from typing_extensions import Self 15 | 16 | from numalogic.base import BaseThresholdModel 17 | 18 | 19 | class StdDevThreshold(BaseThresholdModel): 20 | r"""Threshold estimator that calculates based on the mean and the std deviation. 21 | 22 | Threshold = Mean + (std_factor * Std) 23 | 24 | Generates anomaly score as the ratio 25 | between the input data and threshold generated. 26 | 27 | Args: 28 | ---- 29 | std_factor: scaler factor for std to be added to mean 30 | min_threshold: clip the threshold value to be above this value 31 | """ 32 | 33 | def __init__(self, std_factor: float = 3.0, min_threshold: float = 0.0): 34 | self.std_factor = std_factor 35 | self.min_threshold = min_threshold 36 | 37 | self._std = None 38 | self._mean = None 39 | self._threshold = None 40 | 41 | @property 42 | def mean(self): 43 | return self._mean 44 | 45 | @property 46 | def std(self): 47 | return self._std 48 | 49 | @property 50 | def threshold(self): 51 | return self._threshold 52 | 53 | def fit(self, x_train: NDArray[float], _=None) -> Self: 54 | """Fit the estimator on the training set.""" 55 | self._std = np.std(x_train, axis=0) 56 | self._mean = np.mean(x_train, axis=0) 57 | self._threshold = self._mean + (self.std_factor * self._std) 58 | self._threshold[self._threshold < self.min_threshold] = self.min_threshold 59 | 60 | return self 61 | 62 | def predict(self, x_test: NDArray[float]) -> NDArray[int]: 63 | """Returns an integer array of same shape as input. 64 | 1 denotes outlier, 0 denotes inlier. 65 | """ 66 | y_pred = x_test.copy() 67 | y_pred[x_test < self._threshold] = 0 68 | y_pred[x_test >= self._threshold] = 1 69 | return y_pred 70 | 71 | def score_samples(self, x_test: NDArray[float]) -> NDArray[float]: 72 | """Returns an anomaly score array with the same shape as input.""" 73 | return x_test / self.threshold 74 | -------------------------------------------------------------------------------- /tests/test_backtest.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | 3 | import pandas as pd 4 | import pytest 5 | from omegaconf import OmegaConf 6 | 7 | from numalogic._constants import TESTS_DIR 8 | from numalogic.backtest import PromBacktester, OutDataFrames 9 | from numalogic.config import ( 10 | NumalogicConf, 11 | ModelInfo, 12 | TrainerConf, 13 | LightningTrainerConf, 14 | ScoreConf, 15 | ScoreAdjustConf, 16 | ) 17 | from numalogic.models.vae import Conv1dVAE 18 | 19 | URL = "http://localhost:9090" 20 | CONF = OmegaConf.structured( 21 | NumalogicConf( 22 | preprocess=[ModelInfo(name="LogTransformer")], 23 | model=ModelInfo(name="Conv1dVAE", conf=dict(seq_len=12, n_features=3, latent_dim=1)), 24 | threshold=ModelInfo(name="MaxPercentileThreshold"), 25 | trainer=TrainerConf(pltrainer_conf=LightningTrainerConf(accelerator="cpu", max_epochs=1)), 26 | score=ScoreConf( 27 | adjust=ScoreAdjustConf( 28 | upper_limits={ 29 | "namespace_app_rollouts_cpu_utilization": 80, 30 | "namespace_app_rollouts_memory_utilization": 80, 31 | } 32 | ) 33 | ), 34 | ) 35 | ) 36 | 37 | 38 | @pytest.fixture 39 | def backtester(tmp_path): 40 | return PromBacktester( 41 | url=URL, 42 | query="{namespace='sandbox-numalogic-demo'}", 43 | metrics=[ 44 | "namespace_app_rollouts_cpu_utilization", 45 | "namespace_app_rollouts_http_request_error_rate", 46 | "namespace_app_rollouts_memory_utilization", 47 | ], 48 | output_dir=tmp_path, 49 | numalogic_cfg=OmegaConf.to_container(CONF), 50 | ) 51 | 52 | 53 | @pytest.fixture 54 | def read_data(): 55 | return pd.read_csv( 56 | os.path.join(TESTS_DIR, "resources", "data", "prom_mv.csv"), index_col="timestamp" 57 | ) 58 | 59 | 60 | def test_train(backtester, read_data): 61 | artifacts = backtester.train_models(read_data) 62 | assert set(artifacts) == {"preproc_clf", "model", "threshold_clf"} 63 | assert isinstance(artifacts["model"], Conv1dVAE) 64 | 65 | 66 | def test_scores(backtester, read_data): 67 | out_dfs = backtester.generate_scores(read_data) 68 | assert isinstance(out_dfs, OutDataFrames) 69 | assert out_dfs.input.shape[0] == int(backtester.test_ratio * read_data.shape[0]) 70 | assert out_dfs.postproc_out.shape[0] == int(backtester.test_ratio * read_data.shape[0]) 71 | assert out_dfs.unified_out.shape[0] == int(backtester.test_ratio * read_data.shape[0]) 72 | 73 | 74 | def test_static_scores(backtester, read_data): 75 | out_df = backtester.generate_static_scores(read_data) 76 | assert isinstance(out_df, pd.DataFrame) 77 | assert out_df.shape[0] == read_data.shape[0] 78 | -------------------------------------------------------------------------------- /tests/models/forecast/test_naive.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from numalogic.synthetic.timeseries import SyntheticTSGenerator 4 | from numalogic.models.forecast.variants import BaselineForecaster, SeasonalNaiveForecaster 5 | 6 | 7 | class TestBaselineForecaster(unittest.TestCase): 8 | @classmethod 9 | def setUpClass(cls) -> None: 10 | ts_generator = SyntheticTSGenerator(seq_len=7200, num_series=3, freq="T") 11 | ts_df = ts_generator.gen_tseries() 12 | cls.train_df, cls.test_df = ts_generator.train_test_split(ts_df, test_size=1440) 13 | 14 | def test_predict(self): 15 | model = BaselineForecaster() 16 | model.fit(self.train_df) 17 | pred_df = model.predict(self.test_df) 18 | self.assertEqual(pred_df.shape, self.test_df.shape) 19 | 20 | def test_scores(self): 21 | model = BaselineForecaster() 22 | model.fit(self.train_df) 23 | pred_df = model.predict(self.test_df) 24 | r2_score = model.r2_score(self.test_df) 25 | anomaly_df = model.score(self.test_df) 26 | 27 | self.assertIsInstance(r2_score, float) 28 | self.assertEqual(pred_df.shape, self.test_df.shape) 29 | self.assertEqual(anomaly_df.shape, self.test_df.shape) 30 | 31 | 32 | class TestSeasonalNaiveForecaster(unittest.TestCase): 33 | @classmethod 34 | def setUpClass(cls) -> None: 35 | ts_generator = SyntheticTSGenerator(seq_len=7200, num_series=3, freq="T") 36 | ts_df = ts_generator.gen_tseries() 37 | cls.train_df, cls.test_df = ts_generator.train_test_split(ts_df, test_size=1440) 38 | 39 | def test_predict(self): 40 | model = SeasonalNaiveForecaster() 41 | model.fit(self.train_df) 42 | pred_df = model.predict(self.test_df) 43 | self.assertEqual(pred_df.shape, self.test_df.shape) 44 | 45 | def test_scores(self): 46 | model = SeasonalNaiveForecaster() 47 | model.fit(self.train_df) 48 | pred_df = model.predict(self.test_df) 49 | r2_score = model.r2_score(self.test_df) 50 | 51 | self.assertEqual(self.test_df.shape, pred_df.shape) 52 | self.assertIsInstance(r2_score, float) 53 | 54 | def test_period_err_01(self): 55 | model = SeasonalNaiveForecaster(season="weekly") 56 | with self.assertRaises(ValueError): 57 | model.fit(self.train_df) 58 | 59 | def test_period_err_02(self): 60 | with self.assertRaises(NotImplementedError): 61 | SeasonalNaiveForecaster(season="yearly") 62 | 63 | def test_evalset_err(self): 64 | model = SeasonalNaiveForecaster() 65 | model.fit(self.train_df) 66 | with self.assertRaises(RuntimeError): 67 | model.predict(self.train_df) 68 | 69 | 70 | if __name__ == "__main__": 71 | unittest.main() 72 | -------------------------------------------------------------------------------- /tests/udfs/test_numaflow.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from datetime import datetime 3 | 4 | import numpy.typing as npt 5 | from pynumaflow.mapper import Datum, Messages, Message 6 | 7 | from numalogic.tools.types import artifact_t 8 | from numalogic.udfs import NumalogicUDF 9 | 10 | 11 | class DummyUDF(NumalogicUDF): 12 | def __init__(self): 13 | super().__init__() 14 | 15 | def exec(self, keys: list[str], datum: Datum) -> Messages: 16 | val = datum.value 17 | return Messages(Message(value=val, keys=keys)) 18 | 19 | @classmethod 20 | def compute(cls, model: artifact_t, input_: npt.NDArray[float], **kwargs): 21 | pass 22 | 23 | 24 | class DummyAsyncUDF(NumalogicUDF): 25 | def __init__(self): 26 | super().__init__(is_async=True) 27 | 28 | async def aexec(self, keys: list[str], datum: Datum) -> Messages: 29 | val = datum.value 30 | return Messages(Message(value=val, keys=keys)) 31 | 32 | @classmethod 33 | def compute(cls, model: artifact_t, input_: npt.NDArray[float], **kwargs): 34 | pass 35 | 36 | 37 | class TestNumalogicUDF(unittest.TestCase): 38 | def setUp(self) -> None: 39 | self.datum = Datum( 40 | keys=["k1", "k2"], 41 | value=b"100", 42 | event_time=datetime.now(), 43 | watermark=datetime.now(), 44 | ) 45 | 46 | def test_exec(self): 47 | udf = DummyUDF() 48 | msgs = udf.exec(["key1", "key2"], self.datum) 49 | self.assertIsInstance(msgs, Messages) 50 | 51 | def test_call(self): 52 | udf = DummyUDF() 53 | msgs = udf(["key1", "key2"], self.datum) 54 | self.assertIsInstance(msgs, Messages) 55 | 56 | async def test_aexec(self): 57 | udf = DummyUDF() 58 | with self.assertRaises(NotImplementedError): 59 | await udf.aexec(["key1", "key2"], self.datum) 60 | 61 | 62 | class TestNumalogicAsyncUDF(unittest.IsolatedAsyncioTestCase): 63 | def setUp(self) -> None: 64 | self.datum = Datum( 65 | keys=["k1", "k2"], 66 | value=b"100", 67 | event_time=datetime.now(), 68 | watermark=datetime.now(), 69 | ) 70 | 71 | async def test_aexec(self): 72 | udf = DummyAsyncUDF() 73 | msgs = await udf.aexec(["key1", "key2"], self.datum) 74 | self.assertIsInstance(msgs, Messages) 75 | 76 | async def test_call(self): 77 | udf = DummyAsyncUDF() 78 | msgs = await udf(["key1", "key2"], self.datum) 79 | self.assertIsInstance(msgs, Messages) 80 | 81 | def test_exec(self): 82 | udf = DummyAsyncUDF() 83 | with self.assertRaises(NotImplementedError): 84 | udf.exec(["key1", "key2"], self.datum) 85 | 86 | 87 | if __name__ == "__main__": 88 | unittest.main() 89 | -------------------------------------------------------------------------------- /examples/multi_udf/src/udf/inference.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | import numpy.typing as npt 5 | from numalogic.models.autoencoder import TimeseriesTrainer 6 | from numalogic.udfs import NumalogicUDF 7 | from numalogic.registry import MLflowRegistry, ArtifactData 8 | from numalogic.tools.data import StreamingDataset 9 | from pynumaflow.function import Messages, Message, Datum 10 | from torch.utils.data import DataLoader 11 | 12 | from src.utils import Payload 13 | 14 | LOGGER = logging.getLogger(__name__) 15 | WIN_SIZE = int(os.getenv("WIN_SIZE")) 16 | TRACKING_URI = "http://mlflow-service.default.svc.cluster.local:5000" 17 | 18 | 19 | class Inference(NumalogicUDF): 20 | """ 21 | The inference function here performs inference on the streaming data and sends 22 | the payload to threshold vertex. 23 | """ 24 | 25 | def __init__(self): 26 | super().__init__() 27 | self.registry = MLflowRegistry(tracking_uri=TRACKING_URI) 28 | 29 | def load_model(self) -> ArtifactData: 30 | """Loads the model from the registry.""" 31 | return self.registry.load(skeys=["ae"], dkeys=["model"]) 32 | 33 | @staticmethod 34 | def _infer(artifact_data: ArtifactData, stream_data: npt.NDArray[float]) -> list[float]: 35 | """Performs inference on the streaming data.""" 36 | main_model = artifact_data.artifact 37 | streamloader = DataLoader(StreamingDataset(stream_data, WIN_SIZE)) 38 | 39 | trainer = TimeseriesTrainer() 40 | reconerr = trainer.predict(main_model, dataloaders=streamloader) 41 | return reconerr.tolist() 42 | 43 | def exec(self, keys: list[str], datum: Datum) -> Messages: 44 | """ 45 | Here inference is done on the data, given, the ML model is present 46 | in the registry. If a model does not exist, the payload is flagged for training. 47 | It then passes to the threshold vertex. 48 | 49 | For more information about the arguments, refer: 50 | https://github.com/numaproj/numaflow-python/blob/main/pynumaflow/function/_dtypes.py 51 | """ 52 | # Load data and convert bytes to Payload 53 | payload = Payload.from_json(datum.value) 54 | 55 | artifact_data = self.load_model() 56 | stream_data = payload.get_array().reshape(-1, 1) 57 | 58 | # Check if model exists for inference 59 | if artifact_data: 60 | payload.set_array(self._infer(artifact_data, stream_data)) 61 | LOGGER.info("%s - Inference complete", payload.uuid) 62 | else: 63 | # If model not found, set status as not found 64 | LOGGER.warning("%s - Model not found", payload.uuid) 65 | payload.is_artifact_valid = False 66 | 67 | # Convert Payload back to bytes and conditional forward to threshold vertex 68 | return Messages(Message(value=payload.to_json())) 69 | -------------------------------------------------------------------------------- /tests/udfs/test_payloadtx.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from datetime import datetime 4 | 5 | import pytest 6 | from fakeredis import FakeServer, FakeStrictRedis 7 | from omegaconf import OmegaConf 8 | from orjson import orjson 9 | 10 | from pynumaflow.mapper import Datum 11 | 12 | from numalogic._constants import TESTS_DIR 13 | from numalogic.udfs import MetricsLoader, PayloadTransformer 14 | from numalogic.udfs._config import PipelineConf 15 | from tests.udfs.utility import input_json_from_file 16 | 17 | logging.basicConfig(level=logging.DEBUG) 18 | REDIS_CLIENT = FakeStrictRedis(server=FakeServer()) 19 | KEYS = ["service-mesh", "1", "2"] 20 | DATUM = input_json_from_file(os.path.join(TESTS_DIR, "udfs", "resources", "data", "stream.json")) 21 | 22 | DATUM_KW = { 23 | "event_time": datetime.now(), 24 | "watermark": datetime.now(), 25 | } 26 | MetricsLoader().load_metrics( 27 | config_file_path=f"{TESTS_DIR}/udfs/resources/numalogic_udf_metrics.yaml" 28 | ) 29 | 30 | DATA = { 31 | "uuid": "dd7dfb43-532b-49a3-906e-f78f82ad9c4b", 32 | "config_id": "druid-config", 33 | "composite_keys": ["service-mesh", "1", "2"], 34 | "data": [], 35 | "raw_data": [ 36 | [17.0, 4.0], 37 | [22.0, 13.0], 38 | [17.0, 7.0], 39 | [23.0, 18.0], 40 | [15.0, 15.0], 41 | [16.0, 9.0], 42 | [10.0, 10.0], 43 | [3.0, 12.0], 44 | [6.0, 21.0], 45 | [5.0, 7.0], 46 | [10.0, 8.0], 47 | [0.0, 0.0], 48 | ], 49 | "metrics": ["failed", "degraded"], 50 | "timestamps": [ 51 | 1691623200000, 52 | 1691623260000, 53 | 1691623320000, 54 | 1691623380000, 55 | 1691623440000, 56 | 1691623500000, 57 | 1691623560000, 58 | 1691623620000, 59 | 1691623680000, 60 | 1691623740000, 61 | 1691623800000, 62 | 1691623860000, 63 | ], 64 | "metadata": { 65 | "tags": { 66 | "asset_alias": "some-alias", 67 | "asset_id": "362557362191815079", 68 | "env": "prd", 69 | }, 70 | }, 71 | } 72 | 73 | 74 | @pytest.fixture() 75 | def udf_args(): 76 | return KEYS, Datum( 77 | keys=KEYS, 78 | value=orjson.dumps(DATA), 79 | **DATUM_KW, 80 | ) 81 | 82 | 83 | @pytest.fixture 84 | def udf(): 85 | _given_conf = OmegaConf.load(os.path.join(TESTS_DIR, "udfs", "resources", "_config.yaml")) 86 | schema = OmegaConf.structured(PipelineConf) 87 | pl_conf = PipelineConf(**OmegaConf.merge(schema, _given_conf)) 88 | udf = PayloadTransformer(pl_conf=pl_conf) 89 | udf.register_conf("druid-config", pl_conf.stream_confs["druid-config"]) 90 | yield udf 91 | 92 | 93 | def test_payloadtx(udf, udf_args): 94 | msgs = udf(*udf_args) 95 | assert len(msgs) == 2 96 | assert orjson.loads(msgs[0].value)["pipeline_id"] == "pipeline1" 97 | assert orjson.loads(msgs[1].value)["pipeline_id"] == "pipeline2" 98 | -------------------------------------------------------------------------------- /numalogic/connectors/rds/_rds.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | from numalogic.connectors._base import DataFetcher 3 | from numalogic.connectors._config import Pivot 4 | from numalogic.connectors.rds._base import format_dataframe 5 | from numalogic.connectors.utils.aws.config import RDSConnectionConfig 6 | import logging 7 | import pandas as pd 8 | from numalogic.connectors.rds.db.factory import RdsFactory 9 | import time 10 | 11 | _LOGGER = logging.getLogger(__name__) 12 | 13 | 14 | class RDSFetcher(DataFetcher): 15 | """ 16 | class is a subclass of DataFetcher and ABC (Abstract Base Class). 17 | It is used to fetch data from an RDS (Relational Database Service) instance by executing 18 | a given SQL query. 19 | 20 | Attributes 21 | ---------- 22 | db_config (RDSConnectionConfig): The configuration object for the RDS instance. 23 | fetcher (db.CLASS_TYPE): The fetcher object for the specific database type. 24 | 25 | """ 26 | 27 | def __init__(self, db_config: RDSConnectionConfig): 28 | super().__init__(db_config.endpoint) 29 | self.db_config = db_config 30 | factory_object = RdsFactory() 31 | self.fetcher = factory_object.get_db_handler(db_config.database_type.lower())(db_config) 32 | _LOGGER.info("Executing for database type: %s", self.fetcher.database_type) 33 | 34 | def fetch( 35 | self, 36 | query, 37 | datetime_column_name: str, 38 | pivot: Optional[Pivot] = None, 39 | group_by: Optional[list[str]] = None, 40 | ) -> pd.DataFrame: 41 | """ 42 | Fetches data from the RDS instance by executing the given query. 43 | 44 | Args: 45 | query (str): The SQL query to be executed. 46 | datetime_column_name (str): The name of the datetime field in the fetched data. 47 | pivot (Optional[Pivot], optional): The pivot configuration for the fetched data. 48 | Defaults to None. 49 | group_by (Optional[list[str]], optional): The list of fields to group the 50 | fetched data by. Defaults to None. 51 | 52 | Returns 53 | ------- 54 | pd.DataFrame: A pandas DataFrame containing the fetched data. 55 | """ 56 | _start_time = time.perf_counter() 57 | df = self.fetcher.execute_query(query) 58 | if df.empty or df.shape[0] == 0: 59 | _LOGGER.warning("No data found for query : %s ", query) 60 | return pd.DataFrame() 61 | 62 | formatted_df = format_dataframe( 63 | df, 64 | query=query, 65 | datetime_column_name=datetime_column_name, 66 | pivot=pivot, 67 | group_by=group_by, 68 | ) 69 | _end_time = time.perf_counter() - _start_time 70 | _LOGGER.info("RDS Query: %s Fetch Time: %.4fs", query, _end_time) 71 | return formatted_df 72 | 73 | def raw_fetch(self, *args, **kwargs) -> pd.DataFrame: 74 | raise NotImplementedError 75 | -------------------------------------------------------------------------------- /docs/pre-processing.md: -------------------------------------------------------------------------------- 1 | # Pre Processing 2 | 3 | When creating a Machine Learning pipeline, data pre-processing plays a crucial role that takes in raw data and transforms it into a format that can be understood and analyzed by the ML Models. 4 | 5 | Generally, the majority of real-word datasets are incomplete, inconsistent or inaccurate (contains errors or outliers). Applying ML algorithms on this raw data would give inaccurate results, as they would fail to identify the underlying patterns effectively. 6 | 7 | Quality decisions must be based on quality data. Data Preprocessing is important to get this quality data, without which it would just be a Garbage In, Garbage Out scenario. 8 | 9 | Numalogic provides the following tranformers for pre-processing the training or testing data sets. You can also pair it with scalers like `MinMaxScaler` from [scikit-learn pre-processing](https://scikit-learn.org/stable/modules/preprocessing.html#preprocessing) tools. 10 | 11 | ### Log Transformer 12 | 13 | Log transformation is a data transformation method in which it replaces each data point x with a log(x). 14 | 15 | Now, with `add_factor`, each data point x is converted to log(x + add_factor) 16 | 17 | Log transformation reduces the variance in some distributions, especially with large outliers. 18 | 19 | ```python 20 | import numpy as np 21 | from sklearn.pipeline import make_pipeline 22 | from sklearn.preprocessing import MinMaxScaler 23 | from numalogic.transforms import LogTransformer 24 | 25 | # Generate some random train and test data 26 | x_train = np.random.randn(100, 3) 27 | x_test = np.random.randn(20, 3) 28 | 29 | transformer = LogTransformer(add_factor=4) 30 | scaler = MinMaxScaler() 31 | 32 | pipeline = make_pipeline(transformer, scaler) 33 | 34 | x_train_scaled = pipeline.fit_transform(x_train) 35 | X_test_scaled = pipeline.transform(x_test) 36 | ``` 37 | 38 | ### Static Power Transformer 39 | 40 | Static Power Transformer converts each data point x to xn. 41 | 42 | When `add_factor` is provided, each data point x is converted to (x + add_factor)n 43 | 44 | ```python 45 | import numpy as np 46 | from numalogic.transforms import StaticPowerTransformer 47 | 48 | # Generate some random train and test data 49 | x_train = np.random.randn(100, 3) 50 | x_test = np.random.randn(20, 3) 51 | 52 | transformer = StaticPowerTransformer(n=3, add_factor=2) 53 | 54 | # Since this transformer is stateless, we can just call transform() 55 | x_train_scaled = transformer.transform(x_train) 56 | X_test_scaled = transformer.transform(x_test) 57 | ``` 58 | 59 | ### Tanh Scaler 60 | 61 | Tanh Scaler is a stateful estimator that applies tanh normalization to the Z-score, 62 | and scales the values between 0 and 1. 63 | This scaler is seen to be more efficient as well as robust to the effect of outliers 64 | in the data. 65 | 66 | ```python 67 | import numpy as np 68 | from numalogic.transforms import TanhScaler 69 | 70 | # Generate some random train and test data 71 | x_train = np.random.randn(100, 3) 72 | x_test = np.random.randn(20, 3) 73 | 74 | scaler = TanhScaler() 75 | 76 | x_train_scaled = scaler.fit_transform(x_train) 77 | x_test_scaled = scaler.transform(x_test) 78 | ``` 79 | -------------------------------------------------------------------------------- /numalogic/udfs/payloadtx.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | from typing import Optional 4 | 5 | import orjson 6 | from numpy import typing as npt 7 | from pynumaflow.mapper import Datum, Messages, Message 8 | 9 | from numalogic.tools.types import artifact_t 10 | from numalogic.udfs import NumalogicUDF 11 | from numalogic.udfs._config import PipelineConf 12 | from numalogic.udfs._logger import configure_logger, log_data_payload_values 13 | from numalogic.udfs._metrics_utility import _increment_counter 14 | 15 | METRICS_ENABLED = bool(int(os.getenv("METRICS_ENABLED", default="0"))) 16 | 17 | _struct_log = configure_logger() 18 | 19 | 20 | class PayloadTransformer(NumalogicUDF): 21 | """ 22 | PayloadGenerator appends pipeline_id to the payload. 23 | 24 | Args: 25 | pl_conf: PipelineConf instance 26 | """ 27 | 28 | @classmethod 29 | def compute(cls, model: artifact_t, input_: npt.NDArray[float], **kwargs): 30 | pass 31 | 32 | def __init__(self, pl_conf: Optional[PipelineConf] = None): 33 | super().__init__(pl_conf=pl_conf, _vtx="payload_adder") 34 | 35 | def exec(self, keys: list[str], datum: Datum) -> Messages: 36 | """ 37 | The pipeline function here receives data from the data source. 38 | 39 | Perform ML pipelining on the input data based on the ml_pipelines provided in config 40 | 41 | Args: 42 | ------- 43 | keys: List of keys 44 | datum: Datum object 45 | 46 | Returns 47 | ------- 48 | Messages instance 49 | 50 | """ 51 | _start_time = time.perf_counter() 52 | logger = _struct_log.bind(udf_vertex=self._vtx) 53 | 54 | # check message sanity 55 | try: 56 | data_payload = orjson.loads(datum.value) 57 | except (orjson.JSONDecodeError, KeyError): # catch json decode error only 58 | logger.exception("Error while decoding input json") 59 | return Messages(Message.to_drop()) 60 | 61 | _stream_conf = self.get_stream_conf(data_payload["config_id"]) 62 | 63 | _metric_label_values = { 64 | "vertex": self._vtx, 65 | "composite_key": ":".join(keys), 66 | "config_id": data_payload["config_id"], 67 | } 68 | 69 | _increment_counter( 70 | counter="MSG_IN_COUNTER", 71 | labels=_metric_label_values, 72 | is_enabled=METRICS_ENABLED, 73 | ) 74 | # create a new message for each ML pipeline 75 | messages = Messages() 76 | for pipeline in _stream_conf.ml_pipelines: 77 | data_payload["pipeline_id"] = pipeline 78 | messages.append(Message(keys=keys, value=orjson.dumps(data_payload))) 79 | 80 | logger = log_data_payload_values(logger, data_payload) 81 | logger.info( 82 | "Appended pipeline id to the payload", 83 | keys=keys, 84 | execution_time_ms=round((time.perf_counter() - _start_time) * 1000, 4), 85 | ) 86 | _increment_counter( 87 | counter="MSG_PROCESSED_COUNTER", 88 | labels=_metric_label_values, 89 | is_enabled=METRICS_ENABLED, 90 | ) 91 | return messages 92 | -------------------------------------------------------------------------------- /numalogic/connectors/redis.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from typing import Optional 4 | 5 | from numalogic.connectors._config import RedisConf 6 | from numalogic.tools.exceptions import EnvVarNotFoundError, ConfigNotFoundError 7 | from numalogic.tools.types import redis_client_t 8 | from redis.backoff import ExponentialBackoff 9 | from redis.exceptions import RedisClusterException, RedisError 10 | from redis.retry import Retry 11 | from redis.sentinel import Sentinel, MasterNotFoundError 12 | 13 | _LOGGER = logging.getLogger(__name__) 14 | SENTINEL_CLIENT: Optional[redis_client_t] = None 15 | 16 | 17 | def get_redis_client( 18 | host: str, 19 | port: int, 20 | password: str, 21 | mastername: str, 22 | master_node: bool = True, 23 | ) -> redis_client_t: 24 | """ 25 | Return a master redis client for sentinel connections, with retry. 26 | 27 | Args: 28 | host: Redis host 29 | port: Redis port 30 | password: Redis password 31 | mastername: Redis sentinel master name 32 | master_node: Whether to use the master node or the slave nodes 33 | 34 | Returns 35 | ------- 36 | Redis client instance 37 | """ 38 | global SENTINEL_CLIENT 39 | 40 | if SENTINEL_CLIENT: 41 | return SENTINEL_CLIENT 42 | 43 | retry = Retry( 44 | ExponentialBackoff(), 45 | 3, 46 | supported_errors=( 47 | ConnectionError, 48 | TimeoutError, 49 | RedisClusterException, 50 | RedisError, 51 | MasterNotFoundError, 52 | ), 53 | ) 54 | 55 | conn_kwargs = { 56 | "socket_timeout": 1, 57 | "socket_connect_timeout": 1, 58 | "socket_keepalive": True, 59 | "health_check_interval": 10, 60 | } 61 | 62 | sentinel = Sentinel( 63 | [(host, port)], 64 | sentinel_kwargs=dict(password=password, **conn_kwargs), 65 | retry=retry, 66 | password=password, 67 | **conn_kwargs 68 | ) 69 | if master_node: 70 | SENTINEL_CLIENT = sentinel.master_for(mastername) 71 | else: 72 | SENTINEL_CLIENT = sentinel.slave_for(mastername) 73 | _LOGGER.info("Sentinel redis params: %s, master_node: %s", conn_kwargs, master_node) 74 | return SENTINEL_CLIENT 75 | 76 | 77 | def get_redis_client_from_conf(redis_conf: RedisConf, **kwargs) -> redis_client_t: 78 | """ 79 | Return a master redis client from config for sentinel connections, with retry. 80 | 81 | Args: 82 | redis_conf: RedisConf object with host, port, master_name, etc. 83 | **kwargs: Additional arguments to pass to get_redis_client. 84 | 85 | Returns 86 | ------- 87 | Redis client instance 88 | """ 89 | if not redis_conf: 90 | raise ConfigNotFoundError("RedisConf not found!") 91 | 92 | auth = os.getenv("REDIS_AUTH") 93 | if not auth: 94 | raise EnvVarNotFoundError("REDIS_AUTH not set!") 95 | 96 | return get_redis_client( 97 | redis_conf.url, 98 | redis_conf.port, 99 | password=os.getenv("REDIS_AUTH"), 100 | mastername=redis_conf.master_name, 101 | **kwargs 102 | ) 103 | -------------------------------------------------------------------------------- /tests/udfs/resources/rds_config.yaml: -------------------------------------------------------------------------------- 1 | stream_confs: 2 | rds-config: 3 | config_id: "rds-config" 4 | source: "rds" 5 | composite_keys: [ 'service-mesh', '1', '2' ] 6 | window_size: 20 7 | ml_pipelines: 8 | pipeline1: 9 | pipeline_id: "pipeline1" 10 | metrics: [ "col1" , "col2" ] 11 | numalogic_conf: 12 | model: 13 | name: "VanillaAE" 14 | conf: 15 | seq_len: 20 16 | n_features: 1 17 | preprocess: 18 | - name: "FlattenVector" 19 | stateful: false 20 | conf: 21 | n_features: 2 22 | - name: "LogTransformer" 23 | stateful: false 24 | conf: 25 | add_factor: 5 26 | threshold: 27 | name: "StdDevThreshold" 28 | conf: 29 | min_threshold: 0.1 30 | score: 31 | feature_agg: 32 | method: MAX 33 | adjust: 34 | upper_limits: 35 | "col1": 23 36 | trainer: 37 | train_hours: 3 38 | min_train_size: 100 39 | transforms: 40 | - name: DataClipper 41 | conf: 42 | lower: [0.0, -inf] 43 | upper: [0.0, inf] 44 | pltrainer_conf: 45 | accelerator: cpu 46 | max_epochs: 5 47 | pipeline2: 48 | pipeline_id: "pipeline2" 49 | metrics: [ "col1" , "col2" ] 50 | numalogic_conf: 51 | model: 52 | name: "VanillaAE" 53 | conf: 54 | seq_len: 10 55 | n_features: 2 56 | preprocess: 57 | - name: "LogTransformer" 58 | stateful: false 59 | conf: 60 | add_factor: 5 61 | threshold: 62 | name: "StdDevThreshold" 63 | conf: 64 | min_threshold: 0.1 65 | score: 66 | feature_agg: 67 | method: MEAN 68 | adjust: 69 | upper_limits: 70 | "col1": 20 71 | "col2": 18 72 | 73 | 74 | redis_conf: 75 | url: "isbsvc-redis-isbs-redis-svc.oss-analytics-numalogicosamfci-usw2-e2e.svc" 76 | port: 26379 77 | expiry: 360 78 | master_name: "mymaster" 79 | 80 | 81 | rds_conf: 82 | connection_conf: 83 | aws_assume_role_arn: "arn:aws:iam::123456789:role/ml_iam_role" 84 | aws_assume_role_session_name: "ml_pipeline_reader" 85 | endpoint: "localhost1" 86 | port: 3306 87 | database_name: "ml_poc" 88 | database_username: "root" 89 | database_password: "admin123" 90 | database_connection_timeout: 10 91 | database_type: "mysql" 92 | database_provider: "rds" 93 | aws_region: "us-west-2" 94 | aws_rds_use_iam: False 95 | ssl_enabled: False 96 | ssl: 97 | ca: "/usr/bin/ml_data/us-west-2-bundle.pem" 98 | id_fetcher: 99 | rds-config-pipeline1: 100 | dimensions : [ "col1" ] 101 | metrics : ["count"] 102 | datasource: "table-name" 103 | group_by: [ "timestamp", "cistatus" ] 104 | pivot: 105 | columns: [ "col2" ] 106 | datetime_column_name: "timestamp" 107 | hash_query_type: True 108 | hash_column_name: model_md5_hash 109 | -------------------------------------------------------------------------------- /numalogic/models/autoencoder/base.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | 13 | from typing import Any 14 | 15 | from torch import Tensor, optim 16 | 17 | from numalogic.base import TorchModel 18 | from numalogic.tools.loss import get_loss_fn 19 | 20 | 21 | class BaseAE(TorchModel): 22 | r"""Abstract Base class for all Pytorch based autoencoder models for time-series data. 23 | 24 | Args: 25 | ---- 26 | loss_fn: loss function used to train the model 27 | supported values include: {huber, l1, mae} 28 | optim_algo: optimizer algo to be used for training 29 | supported values include: {adam, adagrad, rmsprop} 30 | lr: learning rate (default: 1e-3) 31 | weight_decay: weight decay factor weight for regularization (default: 0.0) 32 | """ 33 | 34 | def __init__( 35 | self, 36 | loss_fn: str = "huber", 37 | optim_algo: str = "adam", 38 | lr: float = 1e-3, 39 | weight_decay: float = 0.0, 40 | ): 41 | super().__init__() 42 | self.lr = lr 43 | self.optim_algo = optim_algo 44 | self.criterion = get_loss_fn(loss_fn) 45 | self.weight_decay = weight_decay 46 | 47 | def init_optimizer(self, optim_algo: str): 48 | if optim_algo == "adam": 49 | return optim.Adam(self.parameters(), lr=self.lr, weight_decay=self.weight_decay) 50 | if optim_algo == "adagrad": 51 | return optim.Adagrad(self.parameters(), lr=self.lr, weight_decay=self.weight_decay) 52 | if optim_algo == "rmsprop": 53 | return optim.RMSprop(self.parameters(), lr=self.lr, weight_decay=self.weight_decay) 54 | raise NotImplementedError(f"Unsupported optimizer value provided: {optim_algo}") 55 | 56 | def configure_shape(self, x: Tensor) -> Tensor: 57 | """Method to configure the batch shape for each type of model architecture.""" 58 | return x 59 | 60 | def get_reconstruction_loss(self, batch: Tensor, reduction="mean") -> Tensor: 61 | _, recon = self.forward(batch) 62 | return self.criterion(batch, recon, reduction=reduction) 63 | 64 | def reconstruction(self, batch: Tensor) -> Tensor: 65 | _, recon = self.forward(batch) 66 | return recon 67 | 68 | def configure_optimizers(self) -> dict[str, Any]: 69 | optimizer = self.init_optimizer(self.optim_algo) 70 | return {"optimizer": optimizer} 71 | 72 | def training_step(self, batch: Tensor, batch_idx: int) -> Tensor: 73 | recon_loss = self.get_reconstruction_loss(batch) 74 | self.log("train_loss", recon_loss, on_epoch=True, on_step=False) 75 | return recon_loss 76 | 77 | def validation_step(self, batch: Tensor, batch_idx: int) -> Tensor: 78 | recon_loss = self.get_reconstruction_loss(batch) 79 | self.log("val_loss", recon_loss) 80 | return recon_loss 81 | -------------------------------------------------------------------------------- /tests/models/threshold/test_maha.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pytest 3 | 4 | from numalogic.models.threshold import MahalanobisThreshold, RobustMahalanobisThreshold 5 | from numalogic.tools.exceptions import ModelInitializationError, InvalidDataShapeError 6 | 7 | 8 | @pytest.fixture 9 | def rng_data(): 10 | rng = np.random.default_rng(42) 11 | x_train = rng.normal(size=(100, 15)) 12 | x_test = rng.normal(size=(30, 15)) 13 | return x_train, x_test 14 | 15 | 16 | class TestMahalanobisThreshold: 17 | def test_init(self, rng_data): 18 | x_train, x_test = rng_data 19 | clf = MahalanobisThreshold(max_outlier_prob=0.25) 20 | clf.fit(x_train) 21 | md = clf.mahalanobis(x_test) 22 | assert (x_test.shape[0],) == md.shape 23 | assert all(md) > 0.0 24 | assert clf.threshold > 0.0 25 | assert clf.std_factor == 2.0 26 | 27 | def test_init_err(self): 28 | with pytest.raises(ValueError): 29 | MahalanobisThreshold(max_outlier_prob=0.0) 30 | with pytest.raises(ValueError): 31 | MahalanobisThreshold(max_outlier_prob=1.0) 32 | 33 | def test_singular(self): 34 | clf = MahalanobisThreshold() 35 | clf.fit(np.ones((100, 15))) 36 | md = clf.mahalanobis(np.ones((30, 15))) 37 | assert (30,) == md.shape 38 | 39 | def test_predict(self, rng_data): 40 | x_train, x_test = rng_data 41 | clf = MahalanobisThreshold() 42 | clf.fit(x_train) 43 | y = clf.predict(x_test) 44 | assert (x_test.shape[0],) == y.shape 45 | assert np.max(y) == 1 46 | assert np.min(y) == 0 47 | 48 | def test_notfitted_err(self, rng_data): 49 | x_test = rng_data[1] 50 | clf = MahalanobisThreshold() 51 | with pytest.raises(ModelInitializationError): 52 | clf.predict(x_test) 53 | with pytest.raises(ModelInitializationError): 54 | clf.score_samples(x_test) 55 | 56 | def test_invalid_input_err(self, rng_data): 57 | x_train = rng_data[0] 58 | clf = MahalanobisThreshold() 59 | clf.fit(x_train) 60 | with pytest.raises(InvalidDataShapeError): 61 | clf.predict(np.ones((30, 15, 1))) 62 | with pytest.raises(InvalidDataShapeError): 63 | clf.score_samples(np.ones(30)) 64 | 65 | def test_score_samples(self, rng_data): 66 | x_train, x_test = rng_data 67 | clf = MahalanobisThreshold() 68 | clf.fit(x_train) 69 | y = clf.score_samples(x_test) 70 | assert (x_test.shape[0],) == y.shape 71 | 72 | def test_score_samples_err(self, rng_data): 73 | x_test = rng_data[1] 74 | clf = MahalanobisThreshold() 75 | with pytest.raises(ModelInitializationError): 76 | clf.score_samples(x_test) 77 | 78 | 79 | class TestRobustMahalanobisThreshold: 80 | def test_fit(self, rng_data): 81 | x_train, x_test = rng_data 82 | clf = RobustMahalanobisThreshold(max_outlier_prob=0.25) 83 | clf.fit(x_train) 84 | md = clf.mahalanobis(x_test) 85 | assert (x_test.shape[0],) == md.shape 86 | assert all(md) > 0.0 87 | assert clf.threshold > 0.0 88 | assert clf.std_factor == 2.0 89 | 90 | def test_score(self, rng_data): 91 | x_train, x_test = rng_data 92 | clf = MahalanobisThreshold() 93 | clf.fit(x_train) 94 | y = clf.score_samples(x_test) 95 | assert (x_test.shape[0],) == y.shape 96 | -------------------------------------------------------------------------------- /numalogic/tools/trainer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | 13 | import logging 14 | import sys 15 | import warnings 16 | 17 | import torch 18 | from pytorch_lightning import Trainer, LightningModule 19 | from torch import Tensor 20 | 21 | from numalogic.tools.callbacks import ConsoleLogger 22 | from numalogic.tools.data import inverse_window 23 | from typing import Optional 24 | 25 | _LOGGER = logging.getLogger(__name__) 26 | 27 | 28 | class TimeseriesTrainer(Trainer): 29 | """A PyTorch Lightning Trainer for timeseries NN models in numalogic. 30 | 31 | Args: 32 | ---- 33 | max_epochs: The maximum number of epochs to train for. (default: 100) 34 | logger: Whether to use a console logger to log metrics. (default: True) 35 | log_freq: The number of epochs between logging. (default: 5) 36 | check_val_every_n_epoch: The number of epochs between validation checks. (default: 5) 37 | enable_checkpointing: Whether to enable checkpointing. (default: False) 38 | enable_progress_bar: Whether to enable the progress bar. (default: False) 39 | enable_model_summary: Whether to enable the model summary. (default: False) 40 | **trainer_kw: Additional keyword arguments to pass to the Lightning Trainer. 41 | """ 42 | 43 | def __init__( 44 | self, 45 | max_epochs: int = 50, 46 | logger: bool = True, 47 | log_freq: int = 5, 48 | check_val_every_n_epoch: int = 5, 49 | enable_checkpointing: bool = False, 50 | enable_progress_bar: bool = False, 51 | enable_model_summary: bool = False, 52 | **trainer_kw 53 | ): 54 | if not sys.warnoptions: 55 | warnings.simplefilter("ignore", category=UserWarning) 56 | 57 | if logger: 58 | logger = ConsoleLogger(log_freq=log_freq) 59 | 60 | super().__init__( 61 | logger=logger, 62 | max_epochs=max_epochs, 63 | check_val_every_n_epoch=check_val_every_n_epoch, 64 | enable_checkpointing=enable_checkpointing, 65 | enable_progress_bar=enable_progress_bar, 66 | enable_model_summary=enable_model_summary, 67 | **trainer_kw 68 | ) 69 | 70 | def predict(self, model: Optional[LightningModule] = None, unbatch=True, **kwargs) -> Tensor: 71 | r"""Predicts the output of the model. 72 | 73 | Args: 74 | ---- 75 | model: The model to predict with. (default: None) 76 | unbatch: Whether to inverse window the output. (default: True) 77 | **kwargs: Additional keyword arguments to pass to the Lightning 78 | trainers predict method. 79 | """ 80 | recon_err = super().predict(model, **kwargs) 81 | recon_err = torch.vstack(recon_err) 82 | if unbatch: 83 | return inverse_window(recon_err, method="keep_first") 84 | return recon_err 85 | -------------------------------------------------------------------------------- /numalogic/udfs/README.md: -------------------------------------------------------------------------------- 1 | | Metric Name | Type | Labels | Description | 2 | |:-------------------------:|:---------:|:------------------------------------------------------------------------------:|:----------------------------------------------------:| 3 | | MSG_IN_COUNTER | Counter | vertex, composite_key, config_id, pipeline_id | Count msgs flowing in | 4 | | MSG_PROCESSED_COUNTER | Counter | vertex, composite_key, config_id, pipeline_id | Count msgs processed | 5 | | SOURCE_COUNTER | Counter | source, composite_key, config_id, pipeline_id | Count artifact source (registry or cache) calls | 6 | | INSUFFICIENT_DATA_COUNTER | Counter | composite_key, config_id, pipeline_id | Count insufficient data while Training | 7 | | MODEL_STATUS_COUNTER | Counter | status, vertex, composite_key, config_id, pipeline_id | Count status of the model | 8 | | DATASHAPE_ERROR_COUNTER | Counter | composite_key, config_id, pipeline_id | Count datashape errors in preprocess | 9 | | MSG_DROPPED_COUNTER | Counter | vertex, composite_key, config_id, pipeline_id | Count dropped msgs | 10 | | REDIS_ERROR_COUNTER | Counter | vertex, composite_key, config_id, pipeline_id | Count redis errors | 11 | | EXCEPTION_COUNTER | Counter | vertex, composite_key, config_id, pipeline_id | Count exceptions | 12 | | RUNTIME_ERROR_COUNTER | Counter | vertex, composite_key, config_id, pipeline_id | Count runtime errors | 13 | | FETCH_EXCEPTION_COUNTER | Counter | composite_key, config_id, pipeline_id | Count exceptions during train data fetch calls | 14 | | DATAFRAME_SHAPE_SUMMARY | Summary | composite_key, config_id, pipeline_id | len of dataframe for training | 15 | | NAN_SUMMARY | Summary | composite_key, config_id, pipeline_id | Count nan's in train data | 16 | | INF_SUMMARY | Summary | composite_key, config_id, pipeline_id | Count inf's in train data | 17 | | FETCH_TIME_SUMMARY | Summary | composite_key, config_id, pipeline_id | Train Data Fetch time | 18 | | MODEL_INFO | Info | composite_key, config_id, pipeline_id | Model info | 19 | | UDF_TIME | Histogram | composite_key, config_id, pipeline_id | Histogram for udf processing time | 20 | | RECORDED_DATA_GAUGE | Gauge | "source", "vertex", "composite_key", "config_id", "pipeline_id", "metric_name" | Gauge metric to observe the mean value of the window | 21 | -------------------------------------------------------------------------------- /numalogic/tools/exceptions.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | 13 | class ModelInitializationError(RuntimeError): 14 | """Raised when a model is not initialized properly.""" 15 | 16 | pass 17 | 18 | 19 | class InvalidRangeParameter(Exception): 20 | """Raised when the range parameter is not valid.""" 21 | 22 | pass 23 | 24 | 25 | class LayerSizeMismatchError(Exception): 26 | """Raised when the layer size is not valid.""" 27 | 28 | pass 29 | 30 | 31 | class DataModuleError(Exception): 32 | """Base class for all exceptions raised by the DataModule class.""" 33 | 34 | pass 35 | 36 | 37 | class InvalidDataShapeError(Exception): 38 | """Raised when the data shape is not valid.""" 39 | 40 | pass 41 | 42 | 43 | class UnknownConfigArgsError(Exception): 44 | """Raised when an unknown config argument is passed to a model.""" 45 | 46 | pass 47 | 48 | 49 | class ConfigNotFoundError(RuntimeError): 50 | """Raised when a config is not found.""" 51 | 52 | pass 53 | 54 | 55 | class ConfigError(RuntimeError): 56 | """Raised when a config value has a problem.""" 57 | 58 | pass 59 | 60 | 61 | class ModelVersionError(Exception): 62 | """Raised when a model version is not found in the registry.""" 63 | 64 | pass 65 | 66 | 67 | class RedisRegistryError(Exception): 68 | """Base class for all exceptions raised by the RedisRegistry class.""" 69 | 70 | pass 71 | 72 | 73 | class MetricConfigError(Exception): 74 | """Raised when a numalogic udf metric config is not valid.""" 75 | 76 | pass 77 | 78 | 79 | class DynamoDBRegistryError(Exception): 80 | """Base class for all exceptions raised by the DynamoDBRegistry class.""" 81 | 82 | pass 83 | 84 | 85 | class ModelKeyNotFound(RedisRegistryError): 86 | """Raised when a model key is not found in the registry.""" 87 | 88 | pass 89 | 90 | 91 | class EnvVarNotFoundError(LookupError): 92 | """Raised when an environment variable is not found.""" 93 | 94 | pass 95 | 96 | 97 | class PrometheusFetcherError(Exception): 98 | """Base class for all exceptions raised by the PrometheusFetcher class.""" 99 | 100 | pass 101 | 102 | 103 | class PrometheusInvalidResponseError(PrometheusFetcherError): 104 | """Raised when the Prometheus response is not a success.""" 105 | 106 | pass 107 | 108 | 109 | class DataFormatError(Exception): 110 | """Raised when the data format is not valid.""" 111 | 112 | pass 113 | 114 | 115 | class DruidFetcherError(Exception): 116 | """Base class for all exceptions raised by the DruidFetcher class.""" 117 | 118 | pass 119 | 120 | 121 | class RDSFetcherError(Exception): 122 | """Base class for all exceptions raised by the RDSFetcher class.""" 123 | 124 | pass 125 | 126 | 127 | class WavefrontFetcherError(Exception): 128 | """Base class for all exceptions raised by the WavefrontFetcher class.""" 129 | 130 | pass 131 | -------------------------------------------------------------------------------- /numalogic/blocks/_nn.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Numaproj Authors. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | import torch 13 | from torch.utils.data import DataLoader 14 | import numpy.typing as npt 15 | 16 | from numalogic.blocks import Block 17 | from numalogic.models.autoencoder import TimeseriesTrainer 18 | from numalogic.tools.data import StreamingDataset 19 | from numalogic.tools.types import nn_model_t, state_dict_t 20 | 21 | 22 | class NNBlock(Block): 23 | """ 24 | A block that uses a neural network model to operate on the artifact. 25 | 26 | Serialization is done by saving state dict of the model. 27 | 28 | Args: 29 | ---- 30 | model: The neural network model. 31 | seq_len: The sequence length of the input data. 32 | name: The name of the block. Defaults to "nn". 33 | """ 34 | 35 | __slots__ = ("seq_len",) 36 | 37 | def __init__(self, model: nn_model_t, seq_len: int, name: str = "nn"): 38 | super().__init__(artifact=model, name=name) 39 | self.seq_len = seq_len 40 | 41 | @property 42 | def artifact_state(self) -> state_dict_t: 43 | """The state dict of the model.""" 44 | return self._artifact.state_dict() 45 | 46 | @artifact_state.setter 47 | def artifact_state(self, artifact_state: state_dict_t) -> None: 48 | """Set the state dict of the model.""" 49 | self._artifact.load_state_dict(artifact_state) 50 | 51 | def fit( 52 | self, input_: npt.NDArray[float], batch_size: int = 64, **trainer_kwargs 53 | ) -> npt.NDArray[float]: 54 | """ 55 | Train the model on the input data. 56 | 57 | Args: 58 | ---- 59 | input_: The input data. 60 | batch_size: The batch size to use for training. 61 | trainer_kwargs: Keyword arguments to pass to the lightning trainer. 62 | 63 | Returns 64 | ------- 65 | The error of the model on the input data. 66 | """ 67 | trainer = TimeseriesTrainer(**trainer_kwargs) 68 | ds = StreamingDataset(input_, self.seq_len) 69 | trainer.fit(self._artifact, train_dataloaders=DataLoader(ds, batch_size=batch_size)) 70 | reconerr = trainer.predict( 71 | self._artifact, dataloaders=DataLoader(ds, batch_size=batch_size) 72 | ) 73 | return reconerr.numpy() 74 | 75 | def run(self, input_: npt.NDArray[float], **_) -> npt.NDArray[float]: 76 | """ 77 | Perform forward pass on the streaming input data. 78 | 79 | Args: 80 | ---- 81 | input_: The streaming input data. 82 | 83 | Returns 84 | ------- 85 | The error of the model on the input data. 86 | """ 87 | input_ = torch.from_numpy(input_).float() 88 | # Add a batch dimension 89 | input_ = torch.unsqueeze(input_, dim=0).contiguous() 90 | self._artifact.eval() 91 | with torch.no_grad(): 92 | reconerr = self._artifact.predict_step(input_, batch_idx=0) 93 | return torch.squeeze(reconerr, dim=0).numpy() 94 | -------------------------------------------------------------------------------- /tests/resources/data/prom_default.csv: -------------------------------------------------------------------------------- 1 | timestamp,namespace_app_rollouts_http_request_error_rate 2 | 2023-11-20 19:43:02,0.0 3 | 2023-11-20 19:43:32,0.0 4 | 2023-11-20 19:44:02,0.0 5 | 2023-11-20 19:44:32,0.0 6 | 2023-11-20 19:45:02,0.0 7 | 2023-11-20 19:45:32,0.0 8 | 2023-11-20 19:46:02,0.0 9 | 2023-11-20 19:46:32,0.0 10 | 2023-11-20 19:47:02,0.0 11 | 2023-11-20 19:47:32,0.0 12 | 2023-11-20 19:48:02,0.0 13 | 2023-11-20 19:48:32,0.0 14 | 2023-11-20 19:49:02,0.0 15 | 2023-11-20 19:49:32,0.0 16 | 2023-11-20 19:50:02,0.0 17 | 2023-11-20 19:50:32,0.0 18 | 2023-11-20 19:51:02,0.0 19 | 2023-11-20 19:51:32,0.0 20 | 2023-11-20 19:52:02,0.0 21 | 2023-11-20 19:52:32,0.0 22 | 2023-11-20 19:53:02,0.0 23 | 2023-11-20 19:53:32,0.0 24 | 2023-11-20 19:54:02,0.0 25 | 2023-11-20 19:54:32,0.0 26 | 2023-11-20 19:55:02,0.0 27 | 2023-11-20 19:55:32,0.0 28 | 2023-11-20 19:56:02,0.0 29 | 2023-11-20 19:56:32,0.0 30 | 2023-11-20 19:57:02,0.0 31 | 2023-11-20 19:57:32,0.0 32 | 2023-11-20 19:58:02,0.0 33 | 2023-11-20 19:58:32,0.0 34 | 2023-11-20 19:59:02,0.0 35 | 2023-11-20 19:59:32,0.0 36 | 2023-11-20 20:00:02,0.0 37 | 2023-11-20 20:00:32,0.0 38 | 2023-11-20 20:01:02,0.0 39 | 2023-11-20 20:01:32,0.0 40 | 2023-11-20 20:02:02,0.0 41 | 2023-11-20 20:02:32,0.0 42 | 2023-11-20 20:03:02,0.0 43 | 2023-11-20 20:03:32,0.0 44 | 2023-11-20 20:04:02,0.0 45 | 2023-11-20 20:04:32,0.0 46 | 2023-11-20 20:05:02,0.0 47 | 2023-11-20 20:05:32,0.0 48 | 2023-11-20 20:06:02,0.0 49 | 2023-11-20 20:06:32,0.0 50 | 2023-11-20 20:07:02,0.0 51 | 2023-11-20 20:07:32,0.0 52 | 2023-11-20 20:08:02,0.0 53 | 2023-11-20 20:08:32,0.0 54 | 2023-11-20 20:09:02,0.0 55 | 2023-11-20 20:09:32,0.0 56 | 2023-11-20 20:10:02,0.0 57 | 2023-11-20 20:10:32,0.0 58 | 2023-11-20 20:11:02,0.0 59 | 2023-11-20 20:11:32,0.0 60 | 2023-11-20 20:12:02,0.0 61 | 2023-11-20 20:12:32,0.0 62 | 2023-11-20 20:13:02,0.0 63 | 2023-11-20 20:13:32,0.0 64 | 2023-11-20 20:14:02,0.0 65 | 2023-11-20 20:14:32,0.0 66 | 2023-11-20 20:15:02,0.0 67 | 2023-11-20 20:15:32,0.0 68 | 2023-11-20 20:16:02,0.0 69 | 2023-11-20 20:16:32,0.0 70 | 2023-11-20 20:17:02,0.0 71 | 2023-11-20 20:17:32,0.0 72 | 2023-11-20 20:18:02,0.0 73 | 2023-11-20 20:18:32,0.0 74 | 2023-11-20 20:19:02,0.0 75 | 2023-11-20 20:19:32,0.0 76 | 2023-11-20 20:20:02,0.0 77 | 2023-11-20 20:20:32,0.0 78 | 2023-11-20 20:21:02,0.0 79 | 2023-11-20 20:21:32,0.0 80 | 2023-11-20 20:22:02,0.0 81 | 2023-11-20 20:22:32,0.0 82 | 2023-11-20 20:23:02,0.0 83 | 2023-11-20 20:23:32,0.0 84 | 2023-11-20 20:24:02,0.0 85 | 2023-11-20 20:24:32,0.0 86 | 2023-11-20 20:25:02,0.0 87 | 2023-11-20 20:25:32,0.0 88 | 2023-11-20 20:26:02,0.0 89 | 2023-11-20 20:26:32,0.0 90 | 2023-11-20 20:27:02,0.0 91 | 2023-11-20 20:27:32,0.0 92 | 2023-11-20 20:28:02,0.0 93 | 2023-11-20 20:28:32,0.0 94 | 2023-11-20 20:29:02,0.0 95 | 2023-11-20 20:29:32,0.0 96 | 2023-11-20 20:30:02,0.0 97 | 2023-11-20 20:30:32,0.0 98 | 2023-11-20 20:31:02,0.0 99 | 2023-11-20 20:31:32,0.0 100 | 2023-11-20 20:32:02,0.0 101 | 2023-11-20 20:32:32,0.0 102 | 2023-11-20 20:33:02,0.0 103 | 2023-11-20 20:33:32,0.0 104 | 2023-11-20 20:34:02,0.0 105 | 2023-11-20 20:34:32,0.0 106 | 2023-11-20 20:35:02,0.0 107 | 2023-11-20 20:35:32,0.0 108 | 2023-11-20 20:36:02,0.0 109 | 2023-11-20 20:36:32,0.0 110 | 2023-11-20 20:37:02,0.0 111 | 2023-11-20 20:37:32,0.0 112 | 2023-11-20 20:38:02,0.0 113 | 2023-11-20 20:38:32,0.0 114 | 2023-11-20 20:39:02,0.0 115 | 2023-11-20 20:39:32,0.0 116 | 2023-11-20 20:40:02,0.0 117 | 2023-11-20 20:40:32,0.0 118 | 2023-11-20 20:41:02,0.0 119 | 2023-11-20 20:41:32,0.0 120 | 2023-11-20 20:42:02,0.0 121 | 2023-11-20 20:42:32,0.0 122 | 2023-11-20 20:43:02,0.0 123 | --------------------------------------------------------------------------------