├── .gitignore ├── README.md ├── __init__.py ├── donutx ├── __init__.py ├── missing_imputation.py ├── modified_elbo.py └── variational_autoencoder │ ├── __init__.py │ ├── basic_variational_autoencoder.py │ ├── conditional_variational_autoencoder.py │ ├── partial_conditional_variational_autoencoder.py │ └── variational_autoencoder.py ├── evaluation_metric.py ├── kpi_frame_dataloader.py ├── kpi_frame_dataset.py ├── kpi_series.py ├── main.py ├── model.py ├── network ├── __init__.py ├── fc_gaussian_statistic.py ├── loop.py └── mlp.py ├── requirements.txt ├── sample_data.csv ├── torch_util ├── __init__.py └── parallel_dataset.py ├── utility ├── __init__.py └── timer.py └── visual ├── __init__.py └── curve.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | .DS_Store 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Bagel 2 | The implementation of 'Robust and Unsupervised KPI Anomaly Detection Based on Conditional Variational Autoencoder' 3 | Models are in `model.py`. 4 | 5 | # Dependencies 6 | python >= 3.7 7 | 8 | ``` bash 9 | pip install -r requirements.txt 10 | ``` 11 | 12 | 13 | # Run 14 | ``` bash 15 | python main.py 16 | ``` 17 | 18 | # Citation 19 | Li, Zeyan, Wenxiao Chen, and Dan Pei. "Robust and Unsupervised KPI Anomaly Detection Based on Conditional Variational Autoencoder." 2018 IEEE 37th International Performance Computing and Communications Conference (IPCCC). IEEE, 2018. 20 | 21 | ``` bibtex 22 | @inproceedings{li2018robust, 23 | title={Robust and Unsupervised KPI Anomaly Detection Based on Conditional Variational Autoencoder}, 24 | author={Li, Zeyan and Chen, Wenxiao and Pei, Dan}, 25 | booktitle={2018 IEEE 37th International Performance Computing and Communications Conference (IPCCC)}, 26 | pages={1--9}, 27 | year={2018}, 28 | organization={IEEE} 29 | } 30 | ``` 31 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | from .kpi_frame_dataloader import KpiFrameDataLoader 2 | from .kpi_frame_dataset import KpiFrameDataset, TimestampDataset 3 | from .kpi_series import KPISeries 4 | from .donutx import * 5 | from .model import * 6 | from .evaluation_metric import * 7 | from .threshold_selection import * 8 | -------------------------------------------------------------------------------- /donutx/__init__.py: -------------------------------------------------------------------------------- 1 | from .variational_autoencoder import * 2 | from .missing_imputation import * 3 | from .modified_elbo import * 4 | -------------------------------------------------------------------------------- /donutx/missing_imputation.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.autograd import Variable 3 | from .variational_autoencoder import VAE 4 | 5 | 6 | def mcmc_missing_imputation(observe_normal, vae: VAE, n_iteration=10, **inputs): 7 | assert "observe_x" in inputs 8 | observe_x = inputs["observe_x"] 9 | 10 | if isinstance(observe_x, Variable): 11 | test_x = Variable(observe_x.data) 12 | else: 13 | test_x = Variable(observe_x) 14 | with torch.no_grad(): 15 | for mcmc_step in range(n_iteration): 16 | p_xz, _, _ = vae(**inputs, n_sample=1) 17 | test_x[observe_normal == 0.] = p_xz.sample()[0][observe_normal == 0.] 18 | return test_x 19 | 20 | -------------------------------------------------------------------------------- /donutx/modified_elbo.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.distributions as dist 3 | 4 | 5 | def m_elbo(observe_x, observe_z, normal, p_xz: dist.Distribution, q_zx: dist.Distribution, p_z: dist.Distribution): 6 | """ 7 | :param observe_x: (batch_size, x_dims) 8 | :param observe_z: (sample_size, batch_size, z_dims) or (batch_size, z_dims,) 9 | :param normal: (batch_size, x_dims) 10 | :param p_xz: samples in shape (sample_size, batch_size, x_dims) 11 | :param q_zx: samples in shape (sample_size, batch_size, z_dims) 12 | :param p_z: samples in shape (z_dims, ) 13 | :return: 14 | """ 15 | observe_x = torch.unsqueeze(observe_x, 0) # (1, batch_size, x_dims) 16 | normal = torch.unsqueeze(normal, 0) # (1, batch_size, x_dims) 17 | log_p_xz = p_xz.log_prob(observe_x) # (1, batch_size, x_dims) 18 | if observe_z.dim() == 2: 19 | torch.unsqueeze(observe_z, 0, observe_z) # (sample_size, batch_size, z_dims) 20 | # noinspection PyArgumentList 21 | log_q_zx = torch.sum(q_zx.log_prob(observe_z), -1) # (sample_size, batch_size) 22 | # noinspection PyArgumentList 23 | log_p_z = torch.sum(p_z.log_prob(observe_z), -1) # (sample_size, batch_size) 24 | # noinspection PyArgumentList 25 | radio = (torch.sum(normal, -1) / float(normal.size()[-1])) # (1, batch_size) 26 | # noinspection PyArgumentList 27 | return - torch.mean(torch.sum(log_p_xz * normal, -1) + log_p_z * radio - log_q_zx) 28 | -------------------------------------------------------------------------------- /donutx/variational_autoencoder/__init__.py: -------------------------------------------------------------------------------- 1 | from .basic_variational_autoencoder import * 2 | from .partial_conditional_variational_autoencoder import * 3 | from .variational_autoencoder import * 4 | from .conditional_variational_autoencoder import * 5 | 6 | -------------------------------------------------------------------------------- /donutx/variational_autoencoder/basic_variational_autoencoder.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.distributions as dist 3 | import torch.nn as nn 4 | from torch.autograd import Variable 5 | 6 | from .variational_autoencoder import VAE 7 | 8 | 9 | class BasicVariationalAutoencoder(VAE): 10 | def __init__(self, variational: nn.Module, generative: nn.Module, n_sample=1): 11 | """ 12 | :param variational: nn.Module; \mu, \sigma=variational(x), where z~N(\mu, \sigma^2) 13 | :param generative: nn.Module; \mu, \sigma=generative(z), where x~N(\mu, \sigma^2) 14 | """ 15 | super(BasicVariationalAutoencoder, self).__init__(variational, generative, n_sample) 16 | 17 | def forward(self, *, n_sample=1, **kwargs): 18 | """ 19 | In training phase, z = \sigma * z_std + z_mean, and sample \sigma from N(0, 1) 20 | In testing phase, sample z from Q(z|x) 21 | :param n_sample: positive integer 22 | :param kwargs: 23 | observe_x: in shape (batch_size, x_dims) 24 | :return: P(x|z), Q(z|x), z_samples 25 | """ 26 | x = kwargs["observe_x"] 27 | if n_sample is None: 28 | n_sample = self.n_sample 29 | z_mean, z_std = self.variational(x) 30 | 31 | assert z_mean.size() == z_std.size() 32 | 33 | z_given_x_dist = dist.Normal(z_mean, z_std) 34 | 35 | if self.training: # this attribute is set by model.train() and model.eval() 36 | # reparameterize 37 | zero = Variable(torch.zeros(z_mean.size())).type_as(z_mean) 38 | one = Variable(torch.ones(z_std.size())).type_as(z_std) 39 | z = dist.Normal(zero, one).sample((n_sample, )) * z_std.unsqueeze(0) + z_mean.unsqueeze(0) 40 | assert z.size() == torch.Size((n_sample, *z_mean.size())) 41 | else: 42 | z = z_given_x_dist.sample((n_sample, )) 43 | 44 | x_mean, x_std = self.generative(z) 45 | assert x_mean.size() == x_std.size() == torch.Size((n_sample, *(x.size()))) 46 | x_given_z_dist = dist.Normal(x_mean, x_std) 47 | 48 | return x_given_z_dist, z_given_x_dist, z 49 | 50 | 51 | BasicVAE = BasicVariationalAutoencoder 52 | -------------------------------------------------------------------------------- /donutx/variational_autoencoder/conditional_variational_autoencoder.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.distributions as dist 3 | import torch.nn as nn 4 | from torch.autograd import Variable 5 | 6 | from .variational_autoencoder import VAE 7 | 8 | 9 | class ConditionalVariationalAutoencoder(VAE): 10 | """ 11 | P(x|y) 12 | """ 13 | 14 | def __init__(self, variational: nn.Module, generative: nn.Module, n_sample=1): 15 | """ 16 | :param variational: nn.Module; \mu, \sigma=variational(x), where z~N(\mu, \sigma^2) 17 | :param generative: nn.Module; \mu, \sigma=generative(z), where x~N(\mu, \sigma^2) 18 | """ 19 | super().__init__(variational, generative, n_sample) 20 | 21 | def forward(self, *, n_sample=1, **kwargs): 22 | """ 23 | In training phase, z = \sigma * z_std + z_mean, and sample \sigma from N(0, 1) 24 | In testing phase, sample z from Q(z|x) 25 | :param n_sample: positive integer 26 | :param kwargs: 27 | observe_x: in shape (batch_size, x_dims) 28 | observe_y: in shape (batch_size, y_dims) 29 | :return: P(x|z), Q(z|x), z_samples 30 | """ 31 | x = kwargs.pop("observe_x") 32 | y = kwargs.pop("observe_y") 33 | if n_sample is None: 34 | n_sample = self.n_sample 35 | 36 | # observe_x_mean = torch.mean(x, -1, keepdim=True) 37 | # observer_x_std = torch.std(x, -1, keepdim=True) 38 | # cat = torch.cat([observe_x_mean, observer_x_std, (x - observe_x_mean) / observer_x_std, y], dim=-1) 39 | cat = torch.cat([x, y], dim=-1) 40 | 41 | z_mean, z_std = self.variational(cat) 42 | 43 | z_given_x_dist = dist.Normal(z_mean, z_std) 44 | if self.training: # this attribute is set by model.train() and model.eval() 45 | # reparameterize 46 | zero = Variable(torch.zeros(z_mean.size())).type_as(z_mean) 47 | one = Variable(torch.ones(z_std.size())).type_as(z_std) 48 | z = dist.Normal(zero, one).sample((n_sample, )) * z_std.unsqueeze(0) + z_mean.unsqueeze(0) 49 | assert z.size() == torch.Size((n_sample, *z_mean.size())) 50 | else: 51 | z = z_given_x_dist.sample((n_sample, )) 52 | 53 | y = y.expand(n_sample, -1, -1) 54 | # observer_x_std = observer_x_std.expand(n_sample, -1, -1) 55 | # observe_x_mean = observe_x_mean.expand(n_sample, -1, -1) 56 | cat = torch.cat([z, y], dim=-1) 57 | x_mean, x_std = self.generative(cat) 58 | x_given_z_dist = dist.Normal(x_mean, x_std) 59 | 60 | return x_given_z_dist, z_given_x_dist, z 61 | 62 | def penalty(self) -> Variable: 63 | return self.variational.penalty() + self.generative.penalty() 64 | 65 | 66 | CVAE = ConditionalVariationalAutoencoder 67 | -------------------------------------------------------------------------------- /donutx/variational_autoencoder/partial_conditional_variational_autoencoder.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.distributions as dist 3 | import torch.nn as nn 4 | from torch.autograd import Variable 5 | 6 | from .variational_autoencoder import VAE 7 | 8 | 9 | class PartialConditionalVariationalAutoencoder(VAE): 10 | """ 11 | Abstract gaussian VAE Model 12 | """ 13 | 14 | def __init__(self, variational: nn.Module, generative: nn.Module, additional_variational: nn.Module, n_sample=1): 15 | """ 16 | :param variational: nn.Module; \mu, \sigma=variational(x), where z~N(\mu, \sigma^2) 17 | :param generative: nn.Module; \mu, \sigma=generative(z), where x~N(\mu, \sigma^2) 18 | """ 19 | super().__init__(nn.Sequential(variational, additional_variational), generative, n_sample) 20 | self.original_variational = variational 21 | self.additional_variational = additional_variational 22 | 23 | def forward(self, *, n_sample=1, **kwargs): 24 | """ 25 | In training phase, z = \sigma * z_std + z_mean, and sample \sigma from N(0, 1) 26 | In testing phase, sample z from Q(z|x) 27 | :param n_sample: positive integer 28 | :param kwargs: 29 | observe_x: in shape (batch_size, x_dims) 30 | observe_y: in shape (batch_size, y_dims) 31 | :return: P(x|z), Q(z|x), z_samples 32 | """ 33 | x = kwargs.pop("observe_x") 34 | y = kwargs.pop("observe_y") 35 | if n_sample is None: 36 | n_sample = self.n_sample 37 | 38 | z_mean, z_std = self.original_variational(x) 39 | 40 | assert z_mean.size() == z_std.size() 41 | 42 | z_given_x_dist = dist.Normal(z_mean, z_std) 43 | 44 | if self.training: # this attribute is set by model.train() and model.eval() 45 | # reparameterize 46 | zero = Variable(torch.zeros(z_mean.size())).type_as(z_mean) 47 | one = Variable(torch.ones(z_std.size())).type_as(z_std) 48 | z = dist.Normal(zero, one).sample((n_sample, )) * z_std.unsqueeze(0) + z_mean.unsqueeze(0) 49 | assert z.size() == torch.Size((n_sample, *z_mean.size())) 50 | else: 51 | z = z_given_x_dist.sample((n_sample, )) 52 | 53 | y = y.expand(n_sample, -1, -1) 54 | z = self.additional_variational(torch.cat([z, y], dim=-1)) 55 | 56 | x_mean, x_std = self.generative(z) 57 | assert x_mean.size() == x_std.size() == torch.Size((n_sample, *(x.size()))) 58 | x_given_z_dist = dist.Normal(x_mean, x_std) 59 | 60 | return x_given_z_dist, z_given_x_dist, z 61 | 62 | def penalty(self) -> Variable: 63 | return self.original_variational.penalty() + self.generative.penalty() + self.additional_variational.penalty() 64 | 65 | 66 | PartialCVAE = PartialConditionalVariationalAutoencoder 67 | -------------------------------------------------------------------------------- /donutx/variational_autoencoder/variational_autoencoder.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | from torch.autograd.variable import Variable 3 | 4 | 5 | class VariationalAutoencoder(nn.Module): 6 | def forward(self, *, n_sample=1, **kwargs): 7 | raise NotImplementedError("Don't call base class") 8 | 9 | def __init__(self, variational: nn.Module, generative: nn.Module, n_sample=1): 10 | super().__init__() 11 | self.variational = variational 12 | self.generative = generative 13 | self.n_sample = n_sample 14 | 15 | def penalty(self) -> Variable: 16 | return self.variational.penalty() + self.generative.penalty() 17 | 18 | 19 | VAE = VariationalAutoencoder 20 | -------------------------------------------------------------------------------- /evaluation_metric.py: -------------------------------------------------------------------------------- 1 | from sklearn.metrics import * 2 | 3 | import numpy as np 4 | 5 | 6 | def range_lift_with_delay(array: np.ndarray, label: np.ndarray, delay=None, inplace=False) -> np.ndarray: 7 | """ 8 | :param delay: maximum acceptable delay 9 | :param array: 10 | :param label: 11 | :param inplace: 12 | :return: new_array 13 | """ 14 | assert np.shape(array) == np.shape(label) 15 | if delay is None: 16 | delay = len(array) 17 | splits = np.where(label[1:] != label[:-1])[0] + 1 18 | is_anomaly = label[0] == 1 19 | new_array = np.copy(array) if not inplace else array 20 | pos = 0 21 | for sp in splits: 22 | if is_anomaly: 23 | ptr = min(pos + delay + 1, sp) 24 | new_array[pos: ptr] = np.max(new_array[pos: ptr]) 25 | new_array[ptr: sp] = np.maximum(new_array[ptr: sp], new_array[pos]) 26 | is_anomaly = not is_anomaly 27 | pos = sp 28 | sp = len(label) 29 | if is_anomaly: 30 | ptr = min(pos + delay + 1, sp) 31 | new_array[pos: sp] = np.max(new_array[pos: ptr]) 32 | return new_array 33 | 34 | 35 | def ignore_missing(*args, missing): 36 | result = [] 37 | for arr in args: 38 | _arr = np.copy(arr) 39 | result.append(_arr[missing != 1]) 40 | return tuple(result) 41 | 42 | 43 | def best_f1score_threshold(indicators: np.ndarray, labels: np.ndarray, return_fscore: bool=False, return_candidates: bool=False): 44 | ps, rs, ts = precision_recall_curve(labels, indicators) 45 | fs = 2 * ps * rs / np.clip(ps + rs, a_min=1e-8, a_max=None) 46 | threshold_candidates = ts 47 | f1_scores = fs 48 | best_threshold = threshold_candidates[np.argmax(f1_scores)] 49 | ret = [best_threshold] 50 | if return_fscore: 51 | ret.append(np.max(f1_scores)) 52 | if return_candidates: 53 | ret.append((threshold_candidates, f1_scores)) 54 | return tuple(ret) 55 | -------------------------------------------------------------------------------- /kpi_frame_dataloader.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | from torch.utils.data import Dataset, DataLoader 4 | 5 | 6 | class _IndexSampler(object): 7 | def __init__(self, length, shuffle, drop_last, batch_size): 8 | self.idx = np.arange(length) 9 | if shuffle: 10 | np.random.shuffle(self.idx) 11 | self.pos = 0 12 | self.drop_last = drop_last 13 | self.batch_size = batch_size 14 | self.length = length 15 | 16 | def next(self): 17 | if self.pos + self.batch_size <= self.length: 18 | data = self.idx[self.pos: self.pos + self.batch_size] 19 | elif self.pos >= self.length: 20 | raise StopIteration() 21 | elif self.drop_last: 22 | raise StopIteration() 23 | else: 24 | data = self.idx[self.pos:] 25 | self.pos += self.batch_size 26 | return data 27 | 28 | 29 | class KpiFrameDataLoader(DataLoader): 30 | def __init__(self, dataset: Dataset, batch_size, shuffle=False, drop_last=False): 31 | super().__init__(dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last) 32 | self.dataset = dataset 33 | self.shuffle = shuffle 34 | self.index_sampler = None # type: _IndexSampler 35 | 36 | def __next__(self): 37 | return tuple(torch.from_numpy(_) for _ in self.dataset[self.index_sampler.next()]) 38 | 39 | def __iter__(self): 40 | self.index_sampler = _IndexSampler(length=len(self.dataset), shuffle=self.shuffle, drop_last=self.drop_last, 41 | batch_size=self.batch_size) 42 | return self 43 | 44 | def __len__(self): 45 | if self.drop_last: 46 | return len(self.dataset) // self.batch_size 47 | else: 48 | return (len(self.dataset) + self.batch_size - 1) // self.batch_size 49 | 50 | 51 | def _test_index_sampler(): 52 | sampler = _IndexSampler(100, True, True, 11) 53 | try: 54 | while True: 55 | print(sampler.next()) 56 | except StopIteration: 57 | pass 58 | -------------------------------------------------------------------------------- /kpi_frame_dataset.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence 2 | import numpy as np 3 | from torch.utils.data import Dataset 4 | from kpi_series import KPISeries 5 | 6 | 7 | class KpiFrameDataset(Dataset): 8 | def __init__(self, kpi: KPISeries, frame_size: int, missing_injection_rate: float = 0.0): 9 | self._kpi = kpi 10 | self._frame_size = frame_size 11 | 12 | # self._strided_value = self.to_frames(self.normalized(kpi.value), frame_size) 13 | self._strided_value = self.to_frames(kpi.value, frame_size) 14 | self._strided_abnormal = self.to_frames(kpi.abnormal, frame_size) 15 | self._strided_missing = self.to_frames(kpi.missing, frame_size) 16 | self._strided_label = self.to_frames(kpi.label, frame_size) 17 | self._missing_injection_rate = missing_injection_rate 18 | self._missing_value = kpi.missing_value 19 | 20 | def __len__(self): 21 | return np.size(self._strided_value, 0) 22 | 23 | def __getitem__(self, item): 24 | value = np.copy(self._strided_value[item]).astype(np.float32) 25 | normal = 1 - np.copy(self._strided_abnormal[item]).astype(np.int) 26 | label = np.copy(self._strided_label[item]).astype(np.int) 27 | 28 | _missing_injection(value, normal=normal, label=label, missing_value=self._missing_value, 29 | missing_injection_rate=self._missing_injection_rate) 30 | return value.astype(np.float32), normal.astype(np.float32) 31 | 32 | @staticmethod 33 | def to_frames(array, frame_size: int = 120): 34 | # noinspection PyProtectedMember 35 | from numpy.lib.stride_tricks import as_strided 36 | array = as_strided(array, shape=(np.size(array, 0) - frame_size + 1, frame_size), 37 | strides=(array.strides[-1], array.strides[-1])) 38 | return array 39 | 40 | 41 | def _missing_injection(value, normal, label, missing_value, missing_injection_rate): 42 | injected_missing = np.random.binomial(1, missing_injection_rate, np.shape(value[normal == 1])) 43 | normal[normal == 1] = 1 - injected_missing 44 | value[np.logical_and(normal == 0, label == 0)] = missing_value 45 | return value, normal 46 | 47 | 48 | class TimestampDataset(KpiFrameDataset): 49 | TS_OFFSET = 8 * 3600 50 | 51 | def __init__(self, kpi: KPISeries, frame_size: int): 52 | super().__init__(kpi, frame_size, missing_injection_rate=0.0) 53 | self._timestamp_feature = self.normalize(self._kpi.timestamp) 54 | self._timestamp_digits = self.digits(self._kpi.timestamp) 55 | self._minute_feature = self.normalize(self.ts2minute(self._kpi.timestamp)) 56 | self._hour_feature = self.normalize(self.ts2hour(self._kpi.timestamp)) 57 | self._day_of_week_feature = self.normalize(self.ts2day_of_week(self._kpi.timestamp)) 58 | self._day_in_year_feature = self.normalize(self.ts2day_in_year(self._kpi.timestamp)) 59 | 60 | self._one_hot_minute = self.one_hot(self.ts2minute(self._kpi.timestamp), width=60, loc=0) 61 | self._one_hot_hour = self.one_hot(self.ts2hour(self._kpi.timestamp), width=24, loc=0) 62 | self._one_hot_day_of_week = self.one_hot(self.ts2day_of_week(self._kpi.timestamp), width=7, loc=0) 63 | self._one_hot_month = self.one_hot(self.ts2month(self._kpi.timestamp), width=12, loc=0) 64 | self._one_hot_year = self.one_hot(self.ts2year(self._kpi.timestamp)) 65 | 66 | def __getitem__(self, item): 67 | index = np.asarray(item) + self._frame_size - 1 68 | timestamp = self._kpi.timestamp[index] 69 | # ret = np.concatenate([self.one_hot(self.ts2hour(timestamp), width=24), ], axis=-1).astype(np.float32) 70 | hourly_feature = self._one_hot_hour[index] 71 | day_of_week_feature = self._one_hot_day_of_week[index] 72 | month_feature = self._one_hot_month[index] 73 | year_feature = self._one_hot_year[index] 74 | timestamp_feature = np.expand_dims( 75 | (timestamp - np.min(self._kpi.timestamp)) / (np.max(self._kpi.timestamp) - np.min(self._kpi.timestamp)), -1) 76 | # ret = np.concatenate([, hourly_feature, day_of_week_feature], axis=-1).astype(np.float32) 77 | ret = np.concatenate([ 78 | self._one_hot_minute[index], 79 | self._one_hot_hour[index], 80 | self._one_hot_day_of_week[index], 81 | # self._timestamp_digits[index], 82 | # np.expand_dims(self._day_in_year_feature[index], -1), 83 | # self._one_hot_year[index], 84 | ], 85 | axis=-1).astype(np.float32) 86 | # ret = np.concatenate([timestamp_feature, ], axis=-1).astype(np.float32) 87 | return ret 88 | 89 | @staticmethod 90 | def ts2day_of_week(_ts): 91 | ts = np.asarray(_ts) + TimestampDataset.TS_OFFSET 92 | return ((ts // 86400) + 4) % 7 93 | 94 | @staticmethod 95 | def ts2day_in_year(_ts): 96 | ts = np.asarray(_ts) + TimestampDataset.TS_OFFSET 97 | return (ts // 86400) % 365 98 | 99 | @staticmethod 100 | def ts2hour(_ts): 101 | ts = np.asarray(_ts) + TimestampDataset.TS_OFFSET 102 | return (ts % 86400) // 3600 103 | 104 | @staticmethod 105 | def ts2minute(_ts): 106 | ts = np.asarray(_ts) + TimestampDataset.TS_OFFSET 107 | return ((ts % 86400) % 3600) // 60 108 | 109 | @staticmethod 110 | def ts2month(_ts): 111 | ts = np.asarray(_ts) + TimestampDataset.TS_OFFSET 112 | return np.minimum(11, ((ts // 86400) % 365) // 30) 113 | 114 | @staticmethod 115 | def ts2year(_ts): 116 | ts = np.asarray(_ts) + TimestampDataset.TS_OFFSET 117 | return (ts // 86400) // 365 + 1970 118 | 119 | @staticmethod 120 | def normalize(arr): 121 | scale = np.max(arr) - np.min(arr) 122 | if np.abs(scale - 0) < 1e-3: 123 | return np.ones_like(arr) 124 | else: 125 | return (arr - np.min(arr)) / scale 126 | 127 | @staticmethod 128 | def one_hot(_array, width=None, loc=0): 129 | array = np.copy(np.asarray(_array)).astype(np.int) # type: np.ndarray 130 | assert np.ndim(array) == 1 or np.ndim(array) == 0, "only 1d array or scalar is supported, shape is {}".format( 131 | np.shape(array)) 132 | if width is None: 133 | width = np.max(array) - np.min(array) + 1 134 | loc = np.min(array) 135 | output = np.zeros(np.shape(array) + (width,), dtype=np.int) 136 | if np.ndim(array) == 1: 137 | output[np.arange(0, np.size(array)), array - loc] = 1 138 | else: 139 | output[array - loc] = 1 140 | return output 141 | 142 | @staticmethod 143 | def digits(_array, width=None): 144 | if width is None: 145 | width = int(np.max(np.floor(np.log(_array) / np.log(10)) + 1)) 146 | _ = np.copy(_array).astype(np.int) 147 | arrays = [] 148 | for i in range(width): 149 | arrays.append(np.expand_dims(_ % 10, -1)) 150 | _ //= 10 151 | return np.concatenate(arrays, -1) 152 | -------------------------------------------------------------------------------- /kpi_series.py: -------------------------------------------------------------------------------- 1 | from functools import lru_cache 2 | from typing import Union, Tuple 3 | import pandas as pd 4 | import numpy as np 5 | 6 | 7 | class KPISeries(object): 8 | def __init__(self, value, timestamp, label=None, missing=None, name="", normalized: bool=False): 9 | self._value = np.asarray(value, np.float32) 10 | self._timestamp = np.asarray(timestamp, np.int64) 11 | self._label = np.asarray(label, np.int) if label is not None else np.zeros(np.shape(value), dtype=np.int) 12 | self._missing = np.asarray(missing, np.int) if missing is not None else np.zeros(np.shape(value), dtype=np.int) 13 | self._label[self._missing == 1] = 0 14 | 15 | self.normalized = normalized 16 | 17 | if name == "": 18 | import uuid 19 | self.name = uuid.uuid4() 20 | else: 21 | self.name = name 22 | 23 | self._check_shape() 24 | 25 | def __update_with_index(__index): 26 | self._timestamp = self.timestamp[__index] 27 | self._label = self.label[__index] 28 | self._missing = self.missing[__index] 29 | self._value = self.value[__index] 30 | 31 | # check interval and add missing 32 | __update_with_index(np.argsort(self.timestamp)) 33 | __update_with_index(np.unique(self.timestamp, return_index=True)[1]) 34 | intervals = np.diff(self.timestamp) 35 | interval = np.min(intervals) 36 | assert interval > 0, "interval must be positive:{}".format(interval) 37 | if not np.max(intervals) == interval: 38 | index = (self.timestamp - self.timestamp[0]) // interval 39 | new_timestamp = np.arange(self.timestamp[0], self.timestamp[-1] + 1, interval) 40 | assert new_timestamp[-1] == self.timestamp[-1] and new_timestamp[0] == self.timestamp[0] 41 | assert np.min(np.diff(new_timestamp)) == interval 42 | new_value = np.ones(new_timestamp.shape, dtype=np.float32) * self.missing_value 43 | new_value[index] = self.value 44 | new_label = np.zeros(new_timestamp.shape, dtype=np.int) 45 | new_label[index] = self.label 46 | new_missing = np.ones(new_timestamp.shape, dtype=np.int) 47 | new_missing[index] = self.missing 48 | self._timestamp, self._value, self._label, self._missing = new_timestamp, new_value, new_label, new_missing 49 | self._check_shape() 50 | 51 | def _check_shape(self): 52 | # check shape 53 | assert np.shape(self._value) == np.shape(self._timestamp) == np.shape(self._label) == np.shape( 54 | self._missing), "data shape mismatch, value:{}, timestamp:{}, label:{}, missing:{}".format(np.shape(self._value), np.shape(self._timestamp), 55 | np.shape(self._label), np.shape(self._missing)) 56 | # assert self.normalized or all(self._value >= 0), "value should be non-negative" 57 | # if np.count_nonzero(self._missing) > 0: 58 | # assert self.normalized or all(np.isclose(self._value[self._missing == 1], 0)), "Missing Value should be zero:{}".format(np.unique(self.value[self.missing == 1])) 59 | 60 | @property 61 | def value(self): 62 | return self._value 63 | 64 | @property 65 | def timestamp(self): 66 | return self._timestamp 67 | 68 | @property 69 | def label(self): 70 | return self._label 71 | 72 | @property 73 | def missing(self): 74 | return self._missing 75 | 76 | @property 77 | def time_range(self): 78 | from datetime import datetime 79 | return datetime.fromtimestamp(np.min(self.timestamp)), datetime.fromtimestamp(np.max(self.timestamp)) 80 | 81 | @property 82 | def length(self): 83 | return np.size(self.value, 0) 84 | 85 | @property 86 | def abnormal(self): 87 | return np.logical_or(self.missing, self.label).astype(np.int) 88 | 89 | @property 90 | def missing_rate(self): 91 | return float(np.count_nonzero(self.missing)) / float(self.length) 92 | 93 | @property 94 | def anormaly_rate(self): 95 | return float(np.count_nonzero(self.label)) / float(self.length) 96 | 97 | @lru_cache() 98 | def normalize(self, mean=None, std=None, return_statistic=False): 99 | """ 100 | :param return_statistic: return mean and std or not 101 | :param std: optional, normalize by given std 102 | :param mean: optional, normalize by given mean 103 | :param inplace: inplace normalize 104 | :return: data_set, mean, std 105 | """ 106 | mean = np.mean(self.value) if mean is None else mean 107 | std = np.std(self.value) if std is None else std 108 | normalized_value = (self.value - mean) / np.clip(std, 1e-4, None) 109 | target = KPISeries(normalized_value, self.timestamp, self.label, self.missing, normalized=True, name=self.name) 110 | if return_statistic: 111 | return target, mean, std 112 | else: 113 | return target 114 | 115 | def split(self, radios: Union[Tuple[Tuple[float, float], Tuple[float, float], Tuple[float, float]], Tuple[float, float, float]]) -> \ 116 | Tuple['KPISeries', 'KPISeries', 'KPISeries']: 117 | """ 118 | :param radios: radios of each part, eg. [0.2, 0.3, 0.5] or [(0, 0.1), (0, 0.2), (0.5, 1.0)] 119 | :return: tuple of DataSets 120 | """ 121 | if np.asarray(radios).ndim == 1: 122 | radios = radios # type: Tuple[float, float, float] 123 | assert abs(1.0 - sum(radios)) < 1e-4 124 | split = np.asarray(np.cumsum(np.asarray(radios, np.float64)) * self.length, np.int) 125 | split[-1] = self.length 126 | split = np.concatenate([[0], split]) 127 | result = [] 128 | for l, r in zip(split[:-1], split[1:]): 129 | result.append(KPISeries(self.value[l:r], self.timestamp[l:r], self.label[l:r], self.missing[l:r])) 130 | elif np.asarray(radios).ndim == 2: 131 | radios = radios # type: Tuple[Tuple[float, float], Tuple[float, float], Tuple[float, float]] 132 | result = [] 133 | for start, end in radios: 134 | si = int(self.length * start) 135 | ei = int(self.length * end) 136 | result.append(KPISeries(self.value[si:ei], self.timestamp[si:ei], self.label[si:ei], self.missing[si:ei])) 137 | else: 138 | raise ValueError("split radios in wrong format: {}".format(radios)) 139 | ret = tuple(result) # type: Tuple 140 | return ret 141 | 142 | @lru_cache() 143 | def label_sampling(self, sampling_rate: float = 1.): 144 | """ 145 | sampling label by segments 146 | :param sampling_rate: keep at most sampling_rate labels 147 | :return: 148 | """ 149 | sampling_rate = float(sampling_rate) 150 | assert 0. <= sampling_rate <= 1., "sample rate must be in [0, 1]: {}".format(sampling_rate) 151 | if sampling_rate == 1.: 152 | return self 153 | elif sampling_rate == 0.: 154 | return KPISeries(value=self.value, timestamp=self.timestamp, label=None, missing=self.missing, name=self.name, normalized=self.normalized) 155 | else: 156 | target = np.count_nonzero(self.label) * sampling_rate 157 | label = np.copy(self.label).astype(np.int8) 158 | anormaly_start = np.where(np.diff(label) == 1)[0] + 1 159 | if label[0] == 1: 160 | anormaly_start = np.concatenate([[0], anormaly_start]) 161 | anormaly_end = np.where(np.diff(label) == -1)[0] + 1 162 | if label[-1] == 1: 163 | anormaly_end = np.concatenate([anormaly_end, [len(label)]]) 164 | 165 | x = np.arange(len(anormaly_start)) 166 | np.random.shuffle(x) 167 | 168 | for i in range(len(anormaly_start)): 169 | idx = np.asscalar(np.where(x == i)[0]) 170 | label[anormaly_start[idx]:anormaly_end[idx]] = 0 171 | if np.count_nonzero(label) <= target: 172 | break 173 | return KPISeries(value=self.value, timestamp=self.timestamp, label=label, missing=self.missing, name=self.name, normalized=self.normalized) 174 | 175 | @property 176 | def missing_value(self): 177 | return self.value[self.missing == 1][0] if np.count_nonzero(self.missing) > 0 else 2 * np.min(self.value) - np.max(self.value) 178 | 179 | def __add__(self, other): 180 | """ 181 | :type other KPISeries 182 | :param other: 183 | :return: 184 | """ 185 | if not isinstance(other, type(self)): 186 | raise ValueError("Only KpiSeries can be added together") 187 | value = np.concatenate([self.value, other.value]) 188 | missing = np.concatenate([self.missing, other.missing]) 189 | if len(np.unique(value[missing == 1])) != 1: 190 | value[missing == 1] = 2 * np.min(value) - np.max(value) 191 | timestamp = np.concatenate([self.timestamp, other.timestamp]) 192 | label = np.concatenate([self.label, other.label]) 193 | return KPISeries(timestamp=timestamp, value=value, label=label, missing=missing, name=self.name, normalized=self.normalized) 194 | 195 | def __iadd__(self, other): 196 | if not isinstance(other, type(self)): 197 | raise ValueError("Only KPISeries can be added together") 198 | value = np.concatenate([self.value, other.value]) 199 | missing = np.concatenate([self.missing, other.missing]) 200 | if len(np.unique(value[missing == 1])) != 1: 201 | value[missing == 1] = 2 * np.min(value) - np.max(value) 202 | timestamp = np.concatenate([self.timestamp, other.timestamp]) 203 | label = np.concatenate([self.label, other.label]) 204 | self._timestamp, self._value, self._label, self._missing = timestamp, value, label, missing 205 | self._check_shape() 206 | return self 207 | 208 | def __len__(self): 209 | return len(self.timestamp) 210 | 211 | @staticmethod 212 | def dump(kpi, path: str, **kwargs): 213 | import pandas as pd 214 | df = pd.DataFrame() 215 | df["timestamp"] = kpi.timestamp 216 | df["value"] = kpi.value 217 | df["label"] = kpi.label 218 | df["missing"] = kpi.missing 219 | if path.endswith(".csv"): 220 | df.to_csv(path, **kwargs) 221 | elif path.endswith(".hdf"): 222 | df.to_csv(path, **kwargs) 223 | else: 224 | raise ValueError(f"Unknown format. Csv and hdf are supported, but given {path}") 225 | 226 | @staticmethod 227 | def load(path: str, **kwargs): 228 | import pandas as pd 229 | import os 230 | if path.endswith(".csv"): 231 | df = pd.read_csv(path, **kwargs) 232 | elif path.endswith(".hdf"): 233 | df = pd.read_hdf(path, **kwargs) 234 | else: 235 | raise ValueError(f"Unknown format. Csv and hdf are supported, but given {path}") 236 | return KPISeries(timestamp=df["timestamp"], 237 | value=df["value"], 238 | missing=df["missing"] if "missing" in df else None, 239 | label=df["label"] if "label" in df else None, 240 | name=os.path.basename(path)[:-4]) 241 | 242 | @staticmethod 243 | def from_dataframe(dataframe: pd.DataFrame, name): 244 | df = dataframe 245 | return KPISeries(timestamp=df["timestamp"], 246 | value=df["value"], 247 | missing=df["missing"] if "missing" in df.columns else None, 248 | label=df["label"] if "label" in df.columns else None, name=name) 249 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from model import DonutX 2 | import pandas as pd 3 | import numpy as np 4 | from kpi_series import KPISeries 5 | from sklearn.metrics import precision_recall_curve 6 | from evaluation_metric import range_lift_with_delay 7 | 8 | df = pd.read_csv('sample_data.csv', header=0, index_col=None) 9 | kpi = KPISeries( 10 | value = df.value, 11 | timestamp = df.timestamp, 12 | label = df.label, 13 | name = 'sample_data', 14 | ) 15 | 16 | train_kpi, valid_kpi, test_kpi = kpi.split((0.49, 0.21, 0.3)) 17 | train_kpi, train_kpi_mean, train_kpi_std = train_kpi.normalize(return_statistic=True) 18 | valid_kpi = valid_kpi.normalize(mean=train_kpi_mean, std=train_kpi_std) 19 | test_kpi = test_kpi.normalize(mean=train_kpi_mean, std=train_kpi_std) 20 | 21 | model = DonutX(cuda=False, max_epoch=50, latent_dims=8, network_size=[100, 100]) 22 | model.fit(train_kpi.label_sampling(0.), valid_kpi) 23 | y_prob = model.predict(test_kpi.label_sampling(0.)) 24 | 25 | y_prob = range_lift_with_delay(y_prob, test_kpi.label) 26 | precisions, recalls, thresholds = precision_recall_curve(test_kpi.label, y_prob) 27 | f1_scores = (2 * precisions * recalls) / (precisions + recalls) 28 | print(f'best F1-score: {np.max(f1_scores[np.isfinite(f1_scores)])}') -------------------------------------------------------------------------------- /model.py: -------------------------------------------------------------------------------- 1 | from typing import * 2 | from scipy.special import erf 3 | import numpy as np 4 | import torch 5 | import torch.distributions as dist 6 | from torch.autograd import Variable 7 | from torch.nn.utils import clip_grad_norm_ 8 | from torch.optim import Adam 9 | from torch.optim.lr_scheduler import StepLR 10 | 11 | from evaluation_metric import ignore_missing 12 | from network import MultiLinearGaussianStatistic 13 | from network.loop import Loop, TestLoop 14 | from torch_util import VstackDataset 15 | from donutx import CVAE, m_elbo, mcmc_missing_imputation, VAE, BasicVAE 16 | from kpi_frame_dataloader import KpiFrameDataLoader 17 | from kpi_frame_dataset import TimestampDataset, KpiFrameDataset 18 | from kpi_series import KPISeries 19 | from evaluation_metric import range_lift_with_delay 20 | import sklearn 21 | 22 | 23 | def threshold_ml(indicators: np.ndarray, labels: np.ndarray, delay=None, factor=100, prior_best=None, return_statistics=False, return_fscore=False): 24 | assert np.shape(indicators) == np.shape(labels), f"indicator and label's shape must be equal. indicator:{np.shape(indicators)}, label:{np.shape(labels)}" 25 | assert np.ndim(indicators) == 1, f"indicator and label must be 1-d array like object. indicator:{np.shape(indicators)}" 26 | _min, _max = np.min(indicators), np.max(indicators) 27 | indicators_ = (indicators - _min) / (_max - _min + 1e-8) 28 | 29 | labeled_idx = np.where(labels != -1)[0] 30 | unlabeled_idx = np.where(labels == -1)[0] 31 | 32 | alpha = len(labeled_idx) / len(labels) 33 | 34 | _indicators_labeled = range_lift_with_delay(indicators_[labeled_idx], labels[labeled_idx], delay=delay) 35 | _ps, _rs, _ts = sklearn.metrics.precision_recall_curve(labels[labeled_idx], _indicators_labeled) 36 | _fs = 2.0 * _ps * _rs / np.clip(_ps + _rs, a_min=1e-4, a_max=None) 37 | thresholds = np.concatenate([_ts, [2.0]]) 38 | f1_scores = _fs 39 | 40 | idx = np.argsort(thresholds) 41 | thresholds = thresholds[idx] 42 | f1_scores = f1_scores[idx] 43 | # print("\nbefore", np.max(f1_scores), thresholds[np.argmax(f1_scores)]) 44 | # _ts = np.linspace(np.min(thresholds), np.max(thresholds), 10000) 45 | _ts = np.linspace(0, 1, 10000) 46 | _fs = f1_scores[np.searchsorted(thresholds, _ts, side="left")] 47 | thresholds = np.concatenate([_ts, thresholds]) 48 | f1_scores = np.concatenate([_fs, f1_scores]) 49 | 50 | idx = np.argsort(thresholds)[::-1] 51 | thresholds = thresholds[idx] 52 | f1_scores = f1_scores[idx] 53 | # print("after", np.max(f1_scores), thresholds[np.argmax(f1_scores)]) 54 | 55 | if len(unlabeled_idx) > 0: 56 | if prior_best is None: 57 | unlabeled_prior_best = threshold_prior(indicators_[unlabeled_idx]) 58 | else: 59 | unlabeled_prior_best = (prior_best - _min) / (_max - _min) 60 | unlabeled_prior = np.abs(thresholds - unlabeled_prior_best) 61 | unlabeled_prior = 1. - unlabeled_prior 62 | likelihood = (f1_scores * alpha + unlabeled_prior * (1.-alpha)) 63 | else: 64 | likelihood = f1_scores 65 | idx = np.argmax(likelihood) 66 | # print("f1 score", sklearn.metrics.f1_score(labels[labeled_idx], indicators_[labeled_idx] >= thresholds[idx])) 67 | # if idx > 0: 68 | # threshold = (thresholds[idx - 1] * (factor - 1) + thresholds[idx]) / factor 69 | # else: 70 | threshold = thresholds[idx] 71 | # print("f1 score", sklearn.metrics.f1_score(labels[labeled_idx], indicators_[labeled_idx] >= threshold)) 72 | threshold = threshold * (_max - _min) + _min 73 | # print("fscore", sklearn.metrics.f1_score(labels, indicators >= threshold)) 74 | if return_statistics: 75 | return threshold, likelihood, thresholds 76 | elif return_fscore: 77 | return threshold, np.max(likelihood) 78 | else: 79 | return threshold 80 | 81 | 82 | class DonutX: 83 | def __init__(self, max_epoch: int = 150, batch_size: int = 128, network_size: List[int] = None, 84 | latent_dims: int = 8, window_size: int = 120, cuda: bool = True, 85 | condition_dropout_left_rate=0.9, print_fn=print): 86 | if network_size is None: 87 | network_size = [100, 100] 88 | 89 | self.print_fn = print_fn 90 | self.condition_size = 60 + 24 + 7 91 | self.window_size = window_size 92 | self.latent_dims = latent_dims 93 | self.network_size = network_size 94 | self.batch_size = batch_size 95 | self.max_epoch = max_epoch 96 | self.cuda = cuda 97 | self.condition_dropout_left_rate = condition_dropout_left_rate 98 | 99 | self._model = CVAE( 100 | MultiLinearGaussianStatistic( 101 | self.window_size + self.condition_size, self.latent_dims, self.network_size, eps=1e-4), 102 | MultiLinearGaussianStatistic( 103 | self.latent_dims + self.condition_size, self.window_size, self.network_size, eps=1e-4), 104 | ) 105 | if self.cuda: 106 | self._model = self._model.cuda() 107 | 108 | if cuda: 109 | self.z_prior_dist = dist.Normal( 110 | Variable(torch.from_numpy(np.zeros((self.latent_dims,), np.float32)).cuda()), 111 | Variable(torch.from_numpy(np.ones((self.latent_dims,), np.float32)).cuda()) 112 | ) 113 | else: 114 | self.z_prior_dist = dist.Normal( 115 | Variable(torch.from_numpy(np.zeros((self.latent_dims,), np.float32))), 116 | Variable(torch.from_numpy(np.ones((self.latent_dims,), np.float32))) 117 | ) 118 | 119 | def fit(self, kpi: KPISeries, valid_kpi: KPISeries = None): 120 | bernoulli = torch.distributions.Bernoulli(probs=self.condition_dropout_left_rate) 121 | self._model.train() 122 | with Loop(max_epochs=self.max_epoch, use_cuda=self.cuda, disp_epoch_freq=5, 123 | print_fn=self.print_fn).with_context() as loop: 124 | optimizer = Adam(self._model.parameters(), lr=1e-3) 125 | lr_scheduler = StepLR(optimizer, step_size=10, gamma=0.75) 126 | train_timestamp_dataset = TimestampDataset( 127 | kpi, frame_size=self.window_size) 128 | train_kpiframe_dataset = KpiFrameDataset(kpi, 129 | frame_size=self.window_size, missing_injection_rate=0.01) 130 | train_dataloader = KpiFrameDataLoader(VstackDataset( 131 | [train_kpiframe_dataset, train_timestamp_dataset]), batch_size=self.batch_size, shuffle=True, 132 | drop_last=True) 133 | if valid_kpi is not None: 134 | valid_timestamp_dataset = TimestampDataset( 135 | valid_kpi.label_sampling(0.), frame_size=self.window_size) 136 | valid_kpiframe_dataset = KpiFrameDataset(valid_kpi, 137 | frame_size=self.window_size, missing_injection_rate=0.) 138 | valid_dataloader = KpiFrameDataLoader(VstackDataset( 139 | [valid_kpiframe_dataset, valid_timestamp_dataset]), batch_size=256, shuffle=True) 140 | else: 141 | valid_dataloader = None 142 | 143 | for epoch in loop.iter_epochs(): 144 | lr_scheduler.step() 145 | for _, batch_data in loop.iter_steps(train_dataloader): 146 | optimizer.zero_grad() 147 | observe_x, observe_normal, observe_y = batch_data 148 | if self.cuda: 149 | mask = bernoulli.sample(sample_shape=observe_y.size()).cuda() 150 | else: 151 | mask = bernoulli.sample(sample_shape=observe_y.size()) 152 | 153 | observe_y = observe_y * mask 154 | p_xz, q_zx, observe_z = self._model( 155 | observe_x=observe_x, observe_y=observe_y) 156 | loss = m_elbo(observe_x, observe_z, observe_normal, p_xz, q_zx, 157 | self.z_prior_dist) + self._model.penalty() * 0.001 # type: Variable 158 | loss.backward() 159 | clip_grad_norm_(self._model.parameters(), max_norm=10.) 160 | optimizer.step() 161 | loop.submit_metric("train_loss", loss.data) 162 | if valid_kpi is not None: 163 | with torch.no_grad(): 164 | for _, batch_data in loop.iter_steps(valid_dataloader): 165 | observe_x, observe_normal, observe_y = batch_data # type: Variable, Variable 166 | p_xz, q_zx, observe_z = self._model( 167 | observe_x=observe_x, observe_y=observe_y) 168 | loss = m_elbo(observe_x, observe_z, observe_normal, p_xz, q_zx, 169 | self.z_prior_dist) + self._model.penalty() * 0.001 # type: Variable 170 | loop.submit_metric("valid_loss", loss.data) 171 | 172 | def predict(self, kpi: KPISeries, return_statistics=False, indicator_name="indicator"): 173 | """ 174 | :param kpi: 175 | :param return_statistics: 176 | :param indicator_name: 177 | default "indicator": Reconstructed probability 178 | "indicator_prior": E_q(z|x)[log p(x|z) * p(z) / q(z|x)] 179 | "indicator_erf": erf(abs(x - x_mean) / x_std * scale_factor) 180 | :return: 181 | """ 182 | with torch.no_grad(): 183 | with TestLoop(use_cuda=self.cuda, print_fn=self.print_fn).with_context() as loop: 184 | test_timestamp_dataset = TimestampDataset(kpi, frame_size=self.window_size) 185 | test_kpiframe_dataset = KpiFrameDataset(kpi, frame_size=self.window_size, missing_injection_rate=0.0) 186 | test_dataloader = KpiFrameDataLoader(VstackDataset( 187 | [test_kpiframe_dataset, test_timestamp_dataset]), batch_size=32, shuffle=False, drop_last=False) 188 | self._model.eval() 189 | for _, batch_data in loop.iter_steps(test_dataloader): 190 | observe_x, observe_normal, observe_y = batch_data # type: Variable, Variable 191 | observe_x = mcmc_missing_imputation(observe_normal=observe_normal, 192 | vae=self._model, 193 | n_iteration=10, 194 | observe_x=observe_x, 195 | observe_y=observe_y 196 | ) 197 | p_xz, q_zx, observe_z = self._model(observe_x=observe_x, 198 | n_sample=128, 199 | observe_y=observe_y) 200 | loss = m_elbo(observe_x, observe_z, observe_normal, 201 | p_xz, q_zx, self.z_prior_dist) # type: Variable 202 | loop.submit_metric("test_loss", loss.data.cpu()) 203 | 204 | log_p_xz = p_xz.log_prob(observe_x).data.cpu().numpy() 205 | 206 | log_p_x = log_p_xz * np.sum( 207 | torch.exp(self.z_prior_dist.log_prob(observe_z) - q_zx.log_prob(observe_z)).cpu().numpy(), 208 | axis=-1, keepdims=True) 209 | 210 | indicator_erf = erf((torch.abs(observe_x - p_xz.mean) / p_xz.stddev).cpu().numpy() * 0.1589967) 211 | 212 | loop.submit_data("indicator", -np.mean(log_p_xz[:, :, -1], axis=0)) 213 | loop.submit_data("indicator_prior", -np.mean(log_p_x[:, :, -1], axis=0)) 214 | loop.submit_data("indicator_erf", np.mean(indicator_erf[:, :, -1], axis=0)) 215 | 216 | loop.submit_data("x_mean", np.mean( 217 | p_xz.mean.data.cpu().numpy()[:, :, -1], axis=0)) 218 | loop.submit_data("x_std", np.mean( 219 | p_xz.stddev.data.cpu().numpy()[:, :, -1], axis=0)) 220 | 221 | indicator = np.concatenate(loop.get_data_by_name(indicator_name)) 222 | x_mean = np.concatenate(loop.get_data_by_name("x_mean")) 223 | x_std = np.concatenate(loop.get_data_by_name("x_std")) 224 | 225 | indicator = np.concatenate([np.ones(shape=self.window_size - 1) * np.min(indicator), indicator]) 226 | if return_statistics: 227 | return indicator, x_mean, x_std 228 | else: 229 | return indicator 230 | 231 | def detect(self, kpi: KPISeries, train_kpi: KPISeries = None, return_threshold=False): 232 | indicators = self.predict(kpi) 233 | indicators_ignore_missing, *_ = ignore_missing(indicators, missing=kpi.missing) 234 | labels_ignore_missing, *_ = ignore_missing(kpi.label, missing=kpi.missing) 235 | threshold = threshold_ml(indicators_ignore_missing, labels_ignore_missing) 236 | 237 | predict = indicators >= threshold 238 | 239 | if return_threshold: 240 | return predict, threshold 241 | else: 242 | return predict 243 | 244 | 245 | class Donut: 246 | def __init__(self, max_epoch: int = 150, batch_size: int = 128, network_size: List[int] = None, 247 | latent_dims: int = 8, window_size: int = 120, cuda: bool = True, print_fn=print): 248 | if network_size is None: 249 | network_size = [100, 100] 250 | 251 | self.print_fn = print_fn 252 | self.window_size = window_size 253 | self.latent_dims = latent_dims 254 | self.network_size = network_size 255 | self.batch_size = batch_size 256 | self.max_epoch = max_epoch 257 | self.cuda = cuda 258 | 259 | self._model = BasicVAE( 260 | MultiLinearGaussianStatistic( 261 | self.window_size, self.latent_dims, self.network_size, eps=1e-4), 262 | MultiLinearGaussianStatistic( 263 | self.latent_dims, self.window_size, self.network_size, eps=1e-4), 264 | ) 265 | if self.cuda: 266 | self._model = self._model.cuda() 267 | 268 | if cuda: 269 | self.z_prior_dist = dist.Normal( 270 | Variable(torch.from_numpy(np.zeros((self.latent_dims,), np.float32)).cuda()), 271 | Variable(torch.from_numpy(np.ones((self.latent_dims,), np.float32)).cuda()) 272 | ) 273 | else: 274 | self.z_prior_dist = dist.Normal( 275 | Variable(torch.from_numpy(np.zeros((self.latent_dims,), np.float32))), 276 | Variable(torch.from_numpy(np.ones((self.latent_dims,), np.float32))) 277 | ) 278 | 279 | def fit(self, kpi: KPISeries, valid_kpi: KPISeries = None): 280 | self._model.train() 281 | with Loop(max_epochs=self.max_epoch, use_cuda=self.cuda, disp_epoch_freq=5, 282 | print_fn=self.print_fn).with_context() as loop: 283 | optimizer = Adam(self._model.parameters(), lr=1e-3) 284 | lr_scheduler = StepLR(optimizer, step_size=10, gamma=0.75) 285 | train_kpiframe_dataset = KpiFrameDataset(kpi, 286 | frame_size=self.window_size, missing_injection_rate=0.01) 287 | train_dataloader = KpiFrameDataLoader(train_kpiframe_dataset, batch_size=self.batch_size, shuffle=True, 288 | drop_last=True) 289 | if valid_kpi is not None: 290 | valid_kpiframe_dataset = KpiFrameDataset(valid_kpi, 291 | frame_size=self.window_size, missing_injection_rate=0.) 292 | valid_dataloader = KpiFrameDataLoader(valid_kpiframe_dataset, batch_size=256, shuffle=True) 293 | else: 294 | valid_dataloader = None 295 | 296 | for epoch in loop.iter_epochs(): 297 | lr_scheduler.step() 298 | for _, batch_data in loop.iter_steps(train_dataloader): 299 | optimizer.zero_grad() 300 | observe_x, observe_normal = batch_data 301 | 302 | p_xz, q_zx, observe_z = self._model(observe_x=observe_x) 303 | loss = m_elbo(observe_x, observe_z, observe_normal, p_xz, q_zx, 304 | self.z_prior_dist) + self._model.penalty() * 0.001 # type: Variable 305 | loss.backward() 306 | clip_grad_norm_(self._model.parameters(), max_norm=10.) 307 | optimizer.step() 308 | loop.submit_metric("train_loss", loss.data) 309 | if valid_kpi is not None: 310 | with torch.no_grad(): 311 | for _, batch_data in loop.iter_steps(valid_dataloader): 312 | observe_x, observe_normal = batch_data # type: Variable, Variable 313 | p_xz, q_zx, observe_z = self._model(observe_x=observe_x) 314 | loss = m_elbo(observe_x, observe_z, observe_normal, p_xz, q_zx, 315 | self.z_prior_dist) + self._model.penalty() * 0.001 # type: Variable 316 | loop.submit_metric("valid_loss", loss.data) 317 | # train_loss_epochs, train_loss = loop.get_metric_by_name("train_loss") 318 | # valid_loss_epochs, valid_loss = loop.get_metric_by_name("valid_loss") 319 | 320 | def predict(self, kpi: KPISeries, return_statistics=False, indicator_name="indicator"): 321 | """ 322 | :param kpi: 323 | :param return_statistics: 324 | :param indicator_name: 325 | default "indicator": Reconstructed probability 326 | "indicator_prior": E_q(z|x)[log p(x|z) * p(z) / q(z|x)] 327 | "indicator_erf": erf(abs(x - x_mean) / x_std * scale_factor) 328 | :return: 329 | """ 330 | with torch.no_grad(): 331 | with TestLoop(use_cuda=True, print_fn=self.print_fn).with_context() as loop: 332 | test_kpiframe_dataset = KpiFrameDataset(kpi, frame_size=self.window_size, missing_injection_rate=0.0) 333 | test_dataloader = KpiFrameDataLoader(test_kpiframe_dataset, batch_size=32, shuffle=False, 334 | drop_last=False) 335 | self._model.eval() 336 | for _, batch_data in loop.iter_steps(test_dataloader): 337 | observe_x, observe_normal = batch_data # type: Variable, Variable 338 | observe_x = mcmc_missing_imputation(observe_normal=observe_normal, 339 | vae=self._model, 340 | n_iteration=10, 341 | observe_x=observe_x, 342 | ) 343 | p_xz, q_zx, observe_z = self._model(observe_x=observe_x, 344 | n_sample=128, 345 | ) 346 | loss = m_elbo(observe_x, observe_z, observe_normal, 347 | p_xz, q_zx, self.z_prior_dist) # type: Variable 348 | loop.submit_metric("test_loss", loss.data.cpu()) 349 | 350 | log_p_xz = p_xz.log_prob(observe_x).data.cpu().numpy() 351 | 352 | log_p_x = log_p_xz * np.sum( 353 | torch.exp(self.z_prior_dist.log_prob(observe_z) - q_zx.log_prob(observe_z)).cpu().numpy(), 354 | axis=-1, keepdims=True) 355 | 356 | indicator_erf = erf((torch.abs(observe_x - p_xz.mean) / p_xz.stddev).cpu().numpy() * 0.1589967) 357 | 358 | loop.submit_data("indicator", -np.mean(log_p_xz[:, :, -1], axis=0)) 359 | loop.submit_data("indicator_prior", -np.mean(log_p_x[:, :, -1], axis=0)) 360 | loop.submit_data("indicator_erf", np.mean(indicator_erf[:, :, -1], axis=0)) 361 | 362 | loop.submit_data("x_mean", np.mean( 363 | p_xz.mean.data.cpu().numpy()[:, :, -1], axis=0)) 364 | loop.submit_data("x_std", np.mean( 365 | p_xz.stddev.data.cpu().numpy()[:, :, -1], axis=0)) 366 | 367 | indicator = np.concatenate(loop.get_data_by_name(indicator_name)) 368 | x_mean = np.concatenate(loop.get_data_by_name("x_mean")) 369 | x_std = np.concatenate(loop.get_data_by_name("x_std")) 370 | 371 | indicator = np.concatenate([np.ones(shape=self.window_size - 1) * np.min(indicator), indicator]) 372 | if return_statistics: 373 | return indicator, x_mean, x_std 374 | else: 375 | return indicator 376 | 377 | def detect(self, kpi: KPISeries, train_kpi: KPISeries = None, return_threshold=False): 378 | indicators = self.predict(kpi) 379 | indicators_ignore_missing, *_ = ignore_missing(indicators, missing=kpi.missing) 380 | labels_ignore_missing, *_ = ignore_missing(kpi.label, missing=kpi.missing) 381 | threshold = threshold_ml(indicators_ignore_missing, labels_ignore_missing) 382 | 383 | predict = indicators >= threshold 384 | 385 | if return_threshold: 386 | return predict, threshold 387 | else: 388 | return predict 389 | -------------------------------------------------------------------------------- /network/__init__.py: -------------------------------------------------------------------------------- 1 | from .fc_gaussian_statistic import MultiLinearGaussianStatistic 2 | from .mlp import MultilayerPerceptron 3 | -------------------------------------------------------------------------------- /network/fc_gaussian_statistic.py: -------------------------------------------------------------------------------- 1 | from typing import Iterable 2 | 3 | import torch 4 | import torch.nn as nn 5 | 6 | 7 | class MultiLinearGaussianStatistic(nn.Module): 8 | """ 9 | Generate a network with multi relu layers and a linear for mean, a linear + softplus for std 10 | """ 11 | def __init__(self, input_dims: int, output_dims: int, net_size: Iterable[int], eps: float = 1e-4): 12 | """ 13 | input: (batch_size, input_dims) 14 | output: (batch_size, output_dims), (batch_size, output_dims) 15 | :param input_dims: 16 | :param output_dims: 17 | :param net_size: sizes of the shared linear layers 18 | :param eps: std = softplus(_) + eps 19 | """ 20 | assert input_dims > 0, "input dims must be positive: {}".format(input_dims) 21 | assert output_dims > 0, "output dims must be positive: {}".format(output_dims) 22 | assert eps >= 0., "epsilon must be non-negative: {}".format(eps) 23 | super(MultiLinearGaussianStatistic, self).__init__() 24 | last_size = input_dims 25 | shared_layers = [] 26 | for current_size in net_size: 27 | shared_layers.append(nn.Linear(last_size, current_size)) 28 | shared_layers.append(nn.ReLU()) 29 | last_size = current_size 30 | self.shared_net = nn.Sequential(*shared_layers) 31 | self.z_mean_net = nn.Sequential( 32 | nn.Linear(last_size, output_dims), 33 | ) 34 | self.z_std_net = nn.Sequential( 35 | nn.Linear(last_size, output_dims), 36 | nn.Softplus(), 37 | ) 38 | self.eps = eps 39 | 40 | def forward(self, _x): 41 | x = self.shared_net(_x) 42 | z_mean = self.z_mean_net(x) 43 | z_std = self.z_std_net(x) + self.eps 44 | return z_mean, z_std 45 | 46 | def penalty(self): 47 | """ 48 | :return: l2 penalty on hidden layers 49 | """ 50 | penalty = 0. 51 | for p in self.shared_net.parameters(): 52 | penalty += torch.sum(p ** 2) 53 | return penalty 54 | -------------------------------------------------------------------------------- /network/loop.py: -------------------------------------------------------------------------------- 1 | from collections import Iterable 2 | import time 3 | from contextlib import contextmanager 4 | from typing import List, Dict, Union, Any 5 | 6 | import numpy as np 7 | from torch.autograd import Variable 8 | from torch.utils.data import DataLoader 9 | 10 | 11 | class Loop(object): 12 | __EPOCH_TIME_KEY = "epoch_time(s)" 13 | __STEP_TIME_KEY = "step_time(s)" 14 | 15 | def __init__(self, max_epochs: int = 32, max_steps: int = None, disp_epoch_freq=1, print_fn=print, use_cuda=False): 16 | def assert_positive_integer(v, name, can_be_none=False): 17 | type_tuple = (int, np.integer, type(None)) if can_be_none else (int, np.integer) 18 | assert isinstance(v, type_tuple) and (v is None or v > 0), "{} should be positive integer: {}".format(name, 19 | v) 20 | 21 | assert max_epochs is not None or max_steps is not None, "At least one of max_epochs and max_steps should not be None" 22 | assert_positive_integer(max_epochs, "max_epochs", True) 23 | assert_positive_integer(max_steps, "max_steps", True) 24 | assert_positive_integer(disp_epoch_freq, "disp_epoch_freq") 25 | self._max_epochs = max_epochs 26 | self._max_steps = max_steps 27 | self._print_fn = print_fn 28 | self._disp_epoch_freq = disp_epoch_freq 29 | self._use_cuda = use_cuda 30 | 31 | self._epoch_cnt = 0 # type: int 32 | self._step_cnt = 0 # type: int 33 | self._displayed_at_epoch = 0 # type: int 34 | 35 | self._metrics = [] # type: List[Dict[str, Any]] 36 | self._data = [] # type: List[Dict[str, Any]] 37 | 38 | self._within_epochs = False 39 | self._within_steps = False 40 | 41 | @contextmanager 42 | def with_context(self): 43 | yield self 44 | 45 | def _eta(self, epoch_time_estimate, step_time_estimate): 46 | estimate = float("inf") 47 | if self._max_epochs is not None: 48 | estimate = min(estimate, (self._max_epochs - self._epoch_cnt) * epoch_time_estimate) 49 | if self._max_steps is not None: 50 | estimate = min(estimate, (self._max_steps - self._step_cnt) * step_time_estimate) 51 | return estimate 52 | 53 | def iter_epochs(self): 54 | def loop_condition(): 55 | return (self._max_epochs is None or self._epoch_cnt < self._max_epochs) and ( 56 | self._max_steps is None or self._step_cnt < self._max_steps) 57 | 58 | def disp_condition(): 59 | return self._epoch_cnt % self._disp_epoch_freq == 0 60 | 61 | self._within_epochs = True 62 | 63 | try: 64 | while loop_condition(): 65 | self._epoch_cnt += 1 66 | self._metrics.append({}) 67 | self._data.append({}) 68 | tic = time.time() 69 | yield self._epoch_cnt 70 | toc = time.time() 71 | self.submit_metric(self.__EPOCH_TIME_KEY, toc - tic) 72 | if disp_condition(): 73 | self._print_log() 74 | self._displayed_at_epoch = self._epoch_cnt 75 | if not disp_condition(): 76 | self._print_log() 77 | finally: 78 | self._within_epochs = False 79 | 80 | def iter_steps(self, dataloader: DataLoader): 81 | assert self._within_epochs, "iter_steps must be called in an iter_epoch." 82 | self._within_steps = True 83 | try: 84 | for data in dataloader: 85 | self._step_cnt += 1 86 | tic = time.time() 87 | yield self._step_cnt, self.__make_variables(data) 88 | toc = time.time() 89 | self.submit_metric(self.__STEP_TIME_KEY, toc - tic) 90 | if self._max_steps is not None and self._step_cnt >= self._max_steps: 91 | break 92 | finally: 93 | self._within_steps = False 94 | 95 | @property 96 | def metrics(self): 97 | return self._metrics 98 | 99 | @property 100 | def data(self): 101 | return self._data 102 | 103 | def get_metric_by_epoch(self, epoch: int): 104 | assert isinstance(epoch, (int, np.integer)) and epoch > 0, "{} should be positive integer: {}".format("epoch", 105 | epoch) 106 | return self._metrics[epoch - 1] 107 | 108 | def get_metric_by_name(self, name: str): 109 | metric_list = [] 110 | epoch_list = [] 111 | for idx, data in enumerate(self.metrics): 112 | if name in data: 113 | metric_list.append(data[name]) 114 | epoch_list.append(idx + 1) 115 | return epoch_list, metric_list 116 | 117 | def get_data_by_epoch(self, epoch: int): 118 | assert isinstance(epoch, (int, np.integer)) and epoch > 0, "{} should be positive integer: {}".format("epoch", 119 | epoch) 120 | return self._data[epoch - 1] 121 | 122 | def get_data_by_name(self, name: str): 123 | data_list = [] 124 | epoch_list = [] 125 | for idx, data in enumerate(self.data): 126 | if name in data: 127 | data_list.append(data[name]) 128 | epoch_list.append(idx + 1) 129 | return epoch_list, data_list 130 | 131 | def submit_metric(self, name, value): 132 | if self._within_steps: 133 | d = self._metrics[-1] # type: dict 134 | if name not in d: 135 | d[name] = [] 136 | d[name].append(value) 137 | elif self._within_epochs: 138 | d = self._metrics[-1] # type: dict 139 | d[name] = value 140 | else: 141 | raise RuntimeError("Can't submit metric outside epoch or step") 142 | 143 | def submit_data(self, name, value): 144 | if self._within_steps: 145 | d = self._data[-1] # type: dict 146 | if name not in d: 147 | d[name] = [] 148 | d[name].append(value) 149 | elif self._within_epochs: 150 | d = self._data[-1] # type: dict 151 | d[name] = value 152 | else: 153 | raise RuntimeError("Can't submit data outside epoch or step") 154 | 155 | def _print_log(self): 156 | metrics_dict = {} 157 | for metrics in self._metrics[self._displayed_at_epoch:self._epoch_cnt]: 158 | for name, data in metrics.items(): 159 | if name not in metrics_dict: 160 | metrics_dict[name] = [] 161 | if isinstance(data, Iterable): 162 | metrics_dict[name].extend(list(data)) 163 | else: 164 | metrics_dict[name].append(data) 165 | metric_str_list = [] 166 | estimate_epoch_time, estimate_step_time = float("inf"), float("inf") 167 | for name, data in metrics_dict.items(): 168 | mean = np.mean(data) 169 | if name == self.__EPOCH_TIME_KEY: 170 | estimate_epoch_time = mean 171 | elif name == self.__STEP_TIME_KEY: 172 | estimate_step_time = mean 173 | if len(data) > 1: 174 | std = np.std(data) 175 | metric_str_list.append("{}: {:.6f}(±{:.6f})".format(name, mean, std)) 176 | else: 177 | metric_str_list.append("{}: {:.6f}".format(name, mean)) 178 | metric_str = " ".join(metric_str_list) 179 | 180 | if self._max_epochs is None: 181 | epoch_str = "{}".format(self._epoch_cnt) 182 | else: 183 | epoch_str = "{}/{}".format(self._epoch_cnt, self._max_epochs) 184 | if self._max_steps is None: 185 | step_str = "{}".format(self._step_cnt) 186 | else: 187 | step_str = "{}/{}".format(self._step_cnt, self._max_steps) 188 | process_str = "[epoch:{} step:{} ETA:{:.3f}s]".format(epoch_str, step_str, 189 | self._eta(estimate_epoch_time, estimate_step_time)) 190 | self._print_fn("{} {}".format(process_str, metric_str)) 191 | 192 | def __make_variables(self, data): 193 | if isinstance(data, Iterable): 194 | ret = [] 195 | for x in data: 196 | ret.append(self.__make_single_variable(x)) 197 | return tuple(ret) 198 | else: 199 | return self.__make_single_variable(data) 200 | 201 | def __make_single_variable(self, data): 202 | ret = Variable(data) 203 | if self._use_cuda: 204 | ret = ret.cuda() 205 | return ret 206 | 207 | 208 | TrainLoop = Loop 209 | 210 | 211 | class TestLoop(Loop): 212 | def __init__(self, print_fn=print, use_cuda=False): 213 | super(TestLoop, self).__init__(max_epochs=1, max_steps=None, disp_epoch_freq=1, print_fn=print_fn, use_cuda=use_cuda) 214 | 215 | def iter_epochs(self): 216 | raise RuntimeError("TestLoop don't need to iterate epochs.") 217 | 218 | @contextmanager 219 | def with_context(self): 220 | for _ in super(TestLoop, self).iter_epochs(): 221 | yield self 222 | 223 | def get_metric_by_name(self, name: str): 224 | return self.metrics[0][name] 225 | 226 | def get_data_by_name(self, name: str): 227 | return self.data[0][name] 228 | 229 | 230 | def __test(): 231 | with Loop(max_epochs=32) as loop: 232 | for epoch in loop.iter_epochs(): 233 | for step, data in loop.iter_steps([1, 2, 3, 4]): 234 | loop.submit_metric("step_loss", 1. / step) 235 | for step, data in loop.iter_steps([2, 3, 4, 5]): 236 | loop.submit_metric("valid_loss", 2 / step) 237 | print("") 238 | with Loop(max_epochs=32, max_steps=10, disp_epoch_freq=3) as loop: 239 | for epoch in loop.iter_epochs(): 240 | for step, data in loop.iter_steps([1, 2, 3, 4]): 241 | loop.submit_metric("step_loss", 1. / step) 242 | print("") 243 | with Loop(max_epochs=32, disp_epoch_freq=4) as loop: 244 | for epoch in loop.iter_epochs(): 245 | for step, data in loop.iter_steps([1, 2, 3, 4]): 246 | loop.submit_metric("step_loss", 1. / step) 247 | 248 | 249 | if __name__ == '__main__': 250 | __test() 251 | -------------------------------------------------------------------------------- /network/mlp.py: -------------------------------------------------------------------------------- 1 | from typing import Iterable 2 | import torch 3 | import torch.nn as nn 4 | 5 | 6 | class MultilayerPerceptron(nn.Module): 7 | def __init__(self, input_dims: int, output_dims: int, net_size: Iterable[int]): 8 | """ 9 | input: (batch_size, input_dims) 10 | output: (batch_size, output_dims), (batch_size, output_dims) 11 | :param input_dims: 12 | :param output_dims: 13 | :param net_size: sizes of the shared linear layers 14 | """ 15 | assert input_dims > 0, "input dims must be positive: {}".format(input_dims) 16 | assert output_dims > 0, "output dims must be positive: {}".format(output_dims) 17 | super().__init__() 18 | last_size = input_dims 19 | layers = [] 20 | for current_size in net_size: 21 | layers.append(nn.Linear(last_size, current_size)) 22 | layers.append(nn.ReLU()) 23 | last_size = current_size 24 | layers.append(nn.Linear(last_size, output_dims)) 25 | self.net = nn.Sequential(*layers) 26 | 27 | def forward(self, _x): 28 | return self.net(_x) 29 | 30 | def penalty(self): 31 | """ 32 | :return: l2 penalty on hidden layers 33 | """ 34 | penalty = 0. 35 | for p in self.net.parameters(): 36 | penalty += torch.sum(p ** 2) 37 | return penalty 38 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | mltoolkit 2 | torch==0.4 3 | torchvision 4 | visdom 5 | numba 6 | jupyterlab 7 | numpy 8 | scipy 9 | matplotlib 10 | seaborn 11 | numba 12 | pandas 13 | tables 14 | sklearn 15 | mltoolkit 16 | statsmodels 17 | dill 18 | pymongo 19 | click 20 | -------------------------------------------------------------------------------- /torch_util/__init__.py: -------------------------------------------------------------------------------- 1 | from .parallel_dataset import * -------------------------------------------------------------------------------- /torch_util/parallel_dataset.py: -------------------------------------------------------------------------------- 1 | from typing import List, Sequence 2 | 3 | import numpy as np 4 | from torch.utils.data import Dataset 5 | 6 | 7 | class DstackDataset(Dataset): 8 | """ 9 | Return data in several datasets in depth stack. 10 | eg. 11 | A return data in sample shape (10, ), B return data in sample shape (2, ), 12 | then ParallelDataset return data in sample shape (12, ) 13 | All the sub-datasets are supposed to have the same length 14 | 15 | At most ONE dataset can return more than one items 16 | eg. 17 | A return (10, ), (10, ), B return (4, ) 18 | then this will return (14, ), (10, ) 19 | """ 20 | def __init__(self, datasets: List[Dataset], complex_dataset=None): 21 | dataset_lengths = list(len(_) for _ in datasets) 22 | assert len(set(dataset_lengths)) == 1, "length of these datasets must be all the same: {}".format(dataset_lengths) 23 | self._length = dataset_lengths[0] 24 | self._datasets = datasets 25 | 26 | self._complex_dataset = complex_dataset 27 | 28 | def __getitem__(self, index): 29 | a = np.concatenate([dataset[index] if dataset is not self._complex_dataset else dataset[index][0] for dataset in self._datasets], axis=-1) 30 | if self._complex_dataset is None: 31 | return a 32 | else: 33 | return (a, ) + self._complex_dataset[index][1:] 34 | 35 | def __len__(self): 36 | return self._length 37 | 38 | 39 | class VstackDataset(Dataset): 40 | """ 41 | Return data in several datasets in vertical stack. 42 | eg. 43 | A return data in sample shape (10, ), B return data in sample shape (2, ), 44 | then ParallelDataset return data in sample shape (10, ), (2, ) 45 | All the sub-datasets are supposed to have the same length 46 | 47 | Dataset __getitem__ return type can't be tuple unless returning multiple array 48 | """ 49 | def __init__(self, datasets: List[Dataset]): 50 | dataset_lengths = list(len(_) for _ in datasets) 51 | assert len(set(dataset_lengths)) == 1, "length of these datasets must be all the same: {}".format(dataset_lengths) 52 | self._length = dataset_lengths[0] 53 | self._datasets = datasets 54 | 55 | def __getitem__(self, index): 56 | x = [dataset[index] if isinstance(dataset[index], tuple) else (dataset[index],) for dataset in self._datasets] 57 | return sum(x, tuple()) 58 | 59 | def __len__(self): 60 | return self._length 61 | -------------------------------------------------------------------------------- /utility/__init__.py: -------------------------------------------------------------------------------- 1 | from .timer import * 2 | -------------------------------------------------------------------------------- /utility/timer.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import time 3 | 4 | __TIC_TIME = None 5 | 6 | 7 | def tic(): 8 | global __TIC_TIME 9 | __TIC_TIME = time.time() 10 | 11 | 12 | def toc(): 13 | return time.time() - __TIC_TIME 14 | -------------------------------------------------------------------------------- /visual/__init__.py: -------------------------------------------------------------------------------- 1 | from .curve import * -------------------------------------------------------------------------------- /visual/curve.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence, Dict, Tuple 2 | 3 | import matplotlib.pyplot as plt 4 | from matplotlib import collections 5 | 6 | 7 | def x_y_curve(data: Sequence[Dict], title: str=None, xlabel: str=None, ylabel: str=None, fig_params=None): 8 | if fig_params is None: 9 | fig_params = {} 10 | fig = plt.figure(**fig_params) 11 | handles = [] 12 | for series in data: 13 | x = series.pop("x") 14 | y = series.pop("y") 15 | handle, = plt.plot(x, y, **series) 16 | if "label" in series: 17 | handles.append(handle) 18 | plt.title(title) 19 | plt.xlabel(xlabel) 20 | plt.ylabel(ylabel) 21 | plt.legend(handles=handles) 22 | return fig 23 | 24 | 25 | def kpi_curve(data: Sequence[Dict], title: str=None, xlabel:str = None, ylabel: str=None, fig_params=None, ylim=None): 26 | import numpy as np 27 | from kpi_anomaly_detection.kpi_series import KPISeries 28 | from datetime import datetime 29 | if fig_params is None: 30 | fig_params = {} 31 | fig, ax = plt.subplots(**fig_params) 32 | handles = [] 33 | for series in data: 34 | name = series.pop("name", None) # because KPI use the name "label" for anomaly points 35 | std = series.pop("std", None) 36 | kpi_series = KPISeries( 37 | value=series.pop("value", series.pop("y")), 38 | timestamp=series.pop("timestamp", series.pop("x")), 39 | label=series.pop("label", None), missing=series.pop("missing", None), name=name) 40 | handle, = plt.plot([datetime.fromtimestamp(_) for _ in kpi_series.timestamp], kpi_series.value, label=name, **series) 41 | if name is not None: 42 | handles.append(handle) 43 | if std is not None: 44 | plt.fill_between([datetime.fromtimestamp(_) for _ in kpi_series.timestamp], 45 | kpi_series.value - std, kpi_series.value + std, 46 | alpha=series.get("fill_alpha", "0.5") 47 | ) 48 | 49 | # plot anomaly 50 | def _plot_anomaly(_series, _color): 51 | split_index = np.where(np.diff(_series) != 0)[0] + 1 52 | points = np.vstack((kpi_series.timestamp, kpi_series.value)).T.reshape(-1, 2) 53 | segments = np.split(points, split_index) 54 | for i in range(len(segments) - 1): 55 | segments[i] = np.concatenate([segments[i], [segments[i + 1][0]]]) 56 | if _series[0] == 1: 57 | segments = segments[0::2] 58 | else: 59 | segments = segments[1::2] 60 | for line in segments: 61 | plt.plot([datetime.fromtimestamp(_) for _ in line[:, 0]], line[:, 1], _color) 62 | 63 | _plot_anomaly(kpi_series.label, "red") 64 | _plot_anomaly(kpi_series.missing, "orange") 65 | plt.title(title) 66 | plt.xlabel(xlabel) 67 | plt.ylabel(ylabel) 68 | plt.ylim(ylim) 69 | plt.legend(handles=handles) 70 | return fig 71 | --------------------------------------------------------------------------------