├── data ├── __init__.py ├── data_loader.py └── data_loader_dad.py ├── exp ├── __init__.py ├── exp_basic.py ├── exp_vrae_dad.py ├── exp_informer.py ├── exp_informer_dad.py └── exp_gta_dad.py ├── models ├── __init__.py ├── decoder.py ├── encoder.py ├── tconv.py ├── model.py ├── embed.py ├── attn.py ├── gta.py └── vrae.py ├── utils ├── __init__.py ├── masking.py ├── metrics.py ├── tools.py └── spot.py ├── requirements.txt ├── main_test_dad.py ├── README.md ├── main_vrae_dad.py ├── main_informer.py ├── main_informer_dad.py ├── main_gta_dad.py └── LICENSE /data/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /exp/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib == 3.1.1 2 | numpy == 1.19.4 3 | pandas == 0.25.1 4 | scikit_learn == 0.21.3 -------------------------------------------------------------------------------- /main_test_dad.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from models.gta import GraphTemporalEmbedding 3 | 4 | if __name__ == '__main__': 5 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 6 | x = torch.randn(32, 96, 122) 7 | model = GraphTemporalEmbedding(122, 96, 3) 8 | y = model(x) 9 | print(y.size()) 10 | # model = AdaGraphSage(num_nodes=10, seq_len=96, label_len=48, out_len=24) 11 | # model = model.double().to(device) 12 | # x = torch.randn(32, 96, 10, requires_grad=True).double().to(device) 13 | # y = torch.randn(32, 48, 10, requires_grad=True).double().to(device) 14 | # # print(out.size()) 15 | # out = model(x, y, None, None) 16 | # print(out.size()) -------------------------------------------------------------------------------- /exp/exp_basic.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import numpy as np 4 | 5 | class Exp_Basic(object): 6 | def __init__(self, args): 7 | self.args = args 8 | self.device = self._acquire_device() 9 | self.model = self._build_model().to(self.device) 10 | 11 | def _build_model(self): 12 | raise NotImplementedError 13 | return None 14 | 15 | def _acquire_device(self): 16 | if self.args.use_gpu: 17 | os.environ["CUDA_VISIBLE_DEVICES"] = str(self.args.gpu) 18 | device = torch.device('cuda:0') 19 | print('Use GPU: cuda:0') 20 | else: 21 | device = torch.device('cpu') 22 | print('Use CPU') 23 | return device 24 | 25 | def _get_data(self): 26 | pass 27 | 28 | def vali(self): 29 | pass 30 | 31 | def train(self): 32 | pass 33 | 34 | def test(self): 35 | pass 36 | -------------------------------------------------------------------------------- /utils/masking.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | class TriangularCausalMask(): 4 | def __init__(self, B, L, device="cpu"): 5 | mask_shape = [B, 1, L, L] 6 | with torch.no_grad(): 7 | self._mask = torch.triu(torch.ones(mask_shape, dtype=torch.bool), diagonal=1).to(device) 8 | 9 | @property 10 | def mask(self): 11 | return self._mask 12 | 13 | class ProbMask(): 14 | def __init__(self, B, H, L, index, scores, device="cpu"): 15 | _mask = torch.ones(L, scores.shape[-1], dytpe=torch.bool).to(device).triu(1) 16 | _mask_ex = _mask[None, None, :].expand(B, H, L, scores.shape[-1]) 17 | indicator = _mask_ex[torch.arange(B)[:, None, None], 18 | torch.arange(H)[None, :, None], 19 | index, :].to(device) 20 | self._mask = indicator.view(scores.shape).to(device) 21 | 22 | @property 23 | def mask(self): 24 | return self._mask -------------------------------------------------------------------------------- /utils/metrics.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def RSE(pred, true): 4 | return np.sqrt(np.sum((true-pred)**2)) / np.sqrt(np.sum((true-true.mean())**2)) 5 | 6 | def CORR(pred, true): 7 | u = ((true-true.mean(0))*(pred-pred.mean(0))).sum(0) 8 | d = np.sqrt(((true-true.mean(0))**2*(pred-pred.mean(0))**2).sum(0)) 9 | return (u/d).mean(-1) 10 | 11 | def MAE(pred, true): 12 | return np.mean(np.abs(pred-true)) 13 | 14 | def MSE(pred, true): 15 | return np.mean((pred-true)**2) 16 | 17 | def RMSE(pred, true): 18 | return np.sqrt(MSE(pred, true)) 19 | 20 | def MAPE(pred, true): 21 | return np.mean(np.abs((pred - true) / true)) 22 | 23 | def MSPE(pred, true): 24 | return np.mean(np.square((pred - true) / true)) 25 | 26 | def metric(pred, true): 27 | mae = MAE(pred, true) 28 | mse = MSE(pred, true) 29 | rmse = RMSE(pred, true) 30 | mape = MAPE(pred, true) 31 | mspe = MSPE(pred, true) 32 | 33 | return mae,mse,rmse,mape,mspe -------------------------------------------------------------------------------- /models/decoder.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | class DecoderLayer(nn.Module): 6 | def __init__(self, self_attention, cross_attention, d_model, d_ff=None, 7 | dropout=0.1, activation="relu"): 8 | super(DecoderLayer, self).__init__() 9 | d_ff = d_ff or 4*d_model 10 | self.self_attention = self_attention 11 | self.cross_attention = cross_attention 12 | self.conv1 = nn.Conv1d(in_channels=d_model, out_channels=d_ff, kernel_size=1) 13 | self.conv2 = nn.Conv1d(in_channels=d_ff, out_channels=d_model, kernel_size=1) 14 | self.norm1 = nn.LayerNorm(d_model) 15 | self.norm2 = nn.LayerNorm(d_model) 16 | self.norm3 = nn.LayerNorm(d_model) 17 | self.dropout = nn.Dropout(dropout) 18 | self.activation = F.relu if activation == "relu" else F.gelu 19 | 20 | def forward(self, x, cross, x_mask=None, cross_mask=None): 21 | x = x + self.dropout(self.self_attention( 22 | x, x, x, 23 | attn_mask=x_mask 24 | )) 25 | x = self.norm1(x) 26 | 27 | x = x + self.dropout(self.cross_attention( 28 | x, cross, cross, 29 | attn_mask=cross_mask 30 | )) 31 | 32 | y = x = self.norm2(x) 33 | y = self.dropout(self.activation(self.conv1(y.transpose(-1,1)))) 34 | y = self.dropout(self.conv2(y).transpose(-1,1)) 35 | 36 | return self.norm3(x+y) 37 | 38 | class Decoder(nn.Module): 39 | def __init__(self, layers, norm_layer=None): 40 | super(Decoder, self).__init__() 41 | self.layers = nn.ModuleList(layers) 42 | self.norm = norm_layer 43 | 44 | def forward(self, x, cross, x_mask=None, cross_mask=None): 45 | for layer in self.layers: 46 | x = layer(x, cross, x_mask=x_mask, cross_mask=cross_mask) 47 | 48 | if self.norm is not None: 49 | x = self.norm(x) 50 | 51 | return x -------------------------------------------------------------------------------- /utils/tools.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | def adjust_learning_rate(optimizer, epoch, args): 5 | # lr = args.learning_rate * (0.2 ** (epoch // 2)) 6 | if args.lradj=='type1': 7 | lr_adjust = {epoch: args.learning_rate * (0.5 ** ((epoch-1) // 1))} 8 | elif args.lradj=='type2': 9 | lr_adjust = { 10 | 2: 5e-5, 4: 1e-5, 6: 5e-6, 8: 1e-6, 11 | 10: 5e-7, 15: 1e-7, 20: 5e-8 12 | } 13 | if epoch in lr_adjust.keys(): 14 | lr = lr_adjust[epoch] 15 | for param_group in optimizer.param_groups: 16 | param_group['lr'] = lr 17 | print('Updating learning rate to {}'.format(lr)) 18 | 19 | class EarlyStopping: 20 | def __init__(self, patience=7, verbose=False, delta=0): 21 | self.patience = patience 22 | self.verbose = verbose 23 | self.counter = 0 24 | self.best_score = None 25 | self.early_stop = False 26 | self.val_loss_min = np.Inf 27 | self.delta = delta 28 | 29 | def __call__(self, val_loss, model, path): 30 | score = -val_loss 31 | if self.best_score is None: 32 | self.best_score = score 33 | self.save_checkpoint(val_loss, model, path) 34 | elif score < self.best_score + self.delta: 35 | self.counter += 1 36 | print(f'EarlyStopping counter: {self.counter} out of {self.patience}') 37 | if self.counter >= self.patience: 38 | self.early_stop = True 39 | else: 40 | self.best_score = score 41 | self.save_checkpoint(val_loss, model, path) 42 | self.counter = 0 43 | 44 | def save_checkpoint(self, val_loss, model, path): 45 | if self.verbose: 46 | print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}). Saving model ...') 47 | torch.save(model.state_dict(), path+'/'+'checkpoint.pth') 48 | self.val_loss_min = val_loss 49 | 50 | class dotdict(dict): 51 | """dot.notation access to dictionary attributes""" 52 | __getattr__ = dict.get 53 | __setattr__ = dict.__setitem__ 54 | __delattr__ = dict.__delitem__ 55 | 56 | class StandardScaler(): 57 | def __init__(self, mean, std): 58 | self.mean = mean 59 | self.std = std 60 | 61 | def transform(self, data): 62 | return (data - self.mean) / self.std 63 | 64 | def inverse_transform(self, data): 65 | return (data * self.std) + self.mean -------------------------------------------------------------------------------- /models/encoder.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | class ConvLayer(nn.Module): 6 | def __init__(self, c_in): 7 | super(ConvLayer, self).__init__() 8 | self.downConv = nn.Conv1d(in_channels=c_in, 9 | out_channels=c_in, 10 | kernel_size=3, 11 | padding=2, 12 | padding_mode='circular') 13 | self.norm = nn.BatchNorm1d(c_in) 14 | self.activation = nn.ELU() 15 | self.maxPool = nn.MaxPool1d(kernel_size=3, stride=2, padding=1) 16 | 17 | def forward(self, x): 18 | x = self.downConv(x.permute(0, 2, 1)) 19 | x = self.norm(x) 20 | x = self.activation(x) 21 | x = self.maxPool(x) 22 | x = x.transpose(1,2) 23 | return x 24 | 25 | class EncoderLayer(nn.Module): 26 | def __init__(self, attention, d_model, d_ff=None, dropout=0.1, activation="relu"): 27 | super(EncoderLayer, self).__init__() 28 | d_ff = d_ff or 4*d_model 29 | self.attention = attention 30 | self.conv1 = nn.Conv1d(in_channels=d_model, out_channels=d_ff, kernel_size=1) 31 | self.conv2 = nn.Conv1d(in_channels=d_ff, out_channels=d_model, kernel_size=1) 32 | self.norm1 = nn.LayerNorm(d_model) 33 | self.norm2 = nn.LayerNorm(d_model) 34 | self.dropout = nn.Dropout(dropout) 35 | self.activation = F.relu if activation == "relu" else F.gelu 36 | 37 | def forward(self, x, attn_mask=None): 38 | # x [B, L, D] 39 | x = x + self.dropout(self.attention( 40 | x, x, x, 41 | attn_mask = attn_mask 42 | )) 43 | 44 | y = x = self.norm1(x) 45 | y = self.dropout(self.activation(self.conv1(y.transpose(-1,1)))) 46 | y = self.dropout(self.conv2(y).transpose(-1,1)) 47 | 48 | return self.norm2(x+y) 49 | 50 | class Encoder(nn.Module): 51 | def __init__(self, attn_layers, conv_layers=None, norm_layer=None): 52 | super(Encoder, self).__init__() 53 | self.attn_layers = nn.ModuleList(attn_layers) 54 | self.conv_layers = nn.ModuleList(conv_layers) if conv_layers is not None else None 55 | self.norm = norm_layer 56 | 57 | def forward(self, x, attn_mask=None): 58 | # x [B, L, D] 59 | if self.conv_layers is not None: 60 | for attn_layer, conv_layer in zip(self.attn_layers, self.conv_layers): 61 | x = attn_layer(x, attn_mask=attn_mask) 62 | x = conv_layer(x) 63 | x = self.attn_layers[-1](x) 64 | else: 65 | for attn_layer in self.attn_layers: 66 | x = attn_layer(x, attn_mask=attn_mask) 67 | 68 | if self.norm is not None: 69 | x = self.norm(x) 70 | 71 | return x -------------------------------------------------------------------------------- /models/tconv.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.nn.utils import weight_norm 4 | 5 | 6 | # class Chomp1d(nn.Module): 7 | # def __init__(self, chomp_size): 8 | # super(Chomp1d, self).__init__() 9 | # self.chomp_size = chomp_size 10 | 11 | # def forward(self, x): 12 | # return x[:, :, :-self.chomp_size].contiguous() 13 | 14 | 15 | class TemporalBlock(nn.Module): 16 | def __init__(self, n_inputs, n_outputs, kernel_size=3, stride=1, dilation=1, padding=1, dropout=0.2): 17 | super(TemporalBlock, self).__init__() 18 | self.conv1 = weight_norm(nn.Conv1d(n_inputs, n_outputs, kernel_size, 19 | stride=stride, padding=padding, dilation=dilation, 20 | padding_mode='circular')) 21 | # self.chomp1 = Chomp1d(padding) 22 | self.relu1 = nn.ReLU() 23 | self.dropout1 = nn.Dropout(dropout) 24 | 25 | self.conv2 = weight_norm(nn.Conv1d(n_outputs, n_outputs, kernel_size, 26 | stride=stride, padding=padding, dilation=dilation, 27 | padding_mode='circular')) 28 | # self.chomp2 = Chomp1d(padding) 29 | self.relu2 = nn.ReLU() 30 | self.dropout2 = nn.Dropout(dropout) 31 | 32 | self.net = nn.Sequential(self.conv1, self.relu1, self.dropout1, 33 | self.conv2, self.relu2, self.dropout2) 34 | self.downsample = nn.Conv1d(n_inputs, n_outputs, 1) if n_inputs != n_outputs else None 35 | self.relu = nn.ReLU() 36 | self.init_weights() 37 | 38 | def init_weights(self): 39 | # self.conv1.weight.data.normal_(0, 0.01) 40 | # self.conv2.weight.data.normal_(0, 0.01) 41 | # if self.downsample is not None: 42 | # self.downsample.weight.data.normal_(0, 0.01) 43 | for m in self.modules(): 44 | if isinstance(m, nn.Conv1d): 45 | nn.init.kaiming_normal_(m.weight, mode='fan_in', nonlinearity='leaky_relu') 46 | 47 | def forward(self, x): 48 | out = self.net(x) 49 | res = x if self.downsample is None else self.downsample(x) 50 | return self.relu(out + res) 51 | 52 | 53 | class TemporalConvNet(nn.Module): 54 | def __init__(self, num_inputs, num_channels, kernel_size=3, dropout=0.2): 55 | super(TemporalConvNet, self).__init__() 56 | layers = [] 57 | num_levels = len(num_channels) 58 | for i in range(num_levels): 59 | dilation_size = 2 ** i 60 | in_channels = num_inputs if i == 0 else num_channels[i-1] 61 | out_channels = num_channels[i] 62 | layers += [TemporalBlock(in_channels, out_channels, kernel_size, stride=1, dilation=dilation_size, 63 | padding=(kernel_size-1) * dilation_size // 2, dropout=dropout)] 64 | 65 | self.network = nn.Sequential(*layers) 66 | 67 | def forward(self, x): 68 | return self.network(x) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Setup the computing environment 2 | ```shell 3 | # Conda Installation 4 | wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh 5 | bash miniconda.sh -b -p $HOME/miniconda 6 | source "$HOME/miniconda/etc/profile.d/conda.sh" 7 | hash -r 8 | conda config --set always_yes yes --set changeps1 no 9 | conda update -q conda 10 | conda info -a 11 | # Create the environment 12 | conda create -n python=3.8 13 | conda activate 14 | # Install independencies 15 | pip install -r requirements.txt 16 | ``` 17 | This repo is built upon **Pytorch** (deep neural networks) and **PytorchGeometrics** (graph learning). For [PyTorch](https://pytorch.org) and [PyG](https://pytorch-geometric.readthedocs.io/en/latest/notes/installation.html) installation, please follow the guide from the websites. Make sure you install the matched the version of Pytorch for PyG. 18 | 19 | # Transformer benchmark architecture (Informer -> AAAI'21 best paper) 20 | Our GTA's **Transformer** architecture is built on top of [**Informer**](https://github.com/zhouhaoyi/Informer2020) who won the best paper award of AAAI'21. One may refer to the link to receive more details. 21 | 22 | # Usage 23 | For training of GTA for deep anomaly detection (dad), please refer to `main_gta_dad.py` for more information. 24 | We also provided a more detailed and complete cli description for training the model: 25 | ``` 26 | python -u main_gta_dad.py --model --data 27 | --root_path --data_path --features 28 | --target --freq --checkpoints 29 | --seq_len --label_len --pred_len 30 | --enc_in --dec_in --c_out --d_model 31 | --n_heads --e_layers --d_layers 32 | --s_layers --d_ff --factor --dropout 33 | --attn --embed --activation 34 | --num_workers --train_epochs --itr 35 | --batch_size --patience --des 36 | --learning_rate --loss --lradj 37 | --use_gpu --gpu 38 | ``` 39 | We would update the repo by providing a version that supports multi-gpus using [Pytorch Lightning](https://www.pytorchlightning.ai). 40 | 41 | ## Citation 42 | If you find this repository useful in your research, please consider citing the following paper: 43 | 44 | ``` 45 | @ARTICLE{zekaietal-gta-2021, 46 | author = {Chen, Zekai and 47 | Chen, Dingshuo and 48 | Zhang, Xiao and 49 | Yuan, Zixuan and 50 | Cheng, Xiuzhen}, 51 | journal = {IEEE Internet of Things Journal}, 52 | title = {Learning Graph Structures with Transformer for Multivariate Time Series Anomaly Detection in IoT}, 53 | year = {2021}, 54 | pages = {1-1}, 55 | doi = {10.1109/JIOT.2021.3100509}} 56 | ``` 57 | 58 | ## Contact 59 | If you have any questions, feel free to contact Zekai Chen through Email (zekai.chen@bms.com) or Github issues. Pull requests are highly welcomed! 60 | -------------------------------------------------------------------------------- /models/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | from utils.masking import TriangularCausalMask, ProbMask 6 | from models.encoder import Encoder, EncoderLayer, ConvLayer 7 | from models.decoder import Decoder, DecoderLayer 8 | from models.attn import FullAttention, ProbAttention, AttentionLayer 9 | from models.embed import DataEmbedding 10 | 11 | class Informer(nn.Module): 12 | def __init__(self, enc_in, dec_in, c_out, seq_len, label_len, out_len, 13 | factor=5, d_model=512, n_heads=8, e_layers=3, d_layers=2, d_ff=512, 14 | dropout=0.0, attn='prob', embed='fixed', data='ETTh', activation='gelu', 15 | device=torch.device('cuda:0')): 16 | super(Informer, self).__init__() 17 | self.pred_len = out_len 18 | self.attn = attn 19 | 20 | # Encoding 21 | self.enc_embedding = DataEmbedding(enc_in, d_model, embed, data, dropout) 22 | self.dec_embedding = DataEmbedding(dec_in, d_model, embed, data, dropout) 23 | # Attention 24 | Attn = ProbAttention if attn=='prob' else FullAttention 25 | # Encoder 26 | self.encoder = Encoder( 27 | [ 28 | EncoderLayer( 29 | AttentionLayer(Attn(False, factor, attention_dropout=dropout), 30 | d_model, n_heads), 31 | d_model, 32 | d_ff, 33 | dropout=dropout, 34 | activation=activation 35 | ) for l in range(e_layers) 36 | ], 37 | [ 38 | ConvLayer( 39 | d_model 40 | ) for l in range(e_layers-1) 41 | ], 42 | norm_layer=torch.nn.LayerNorm(d_model) 43 | ) 44 | # Decoder 45 | self.decoder = Decoder( 46 | [ 47 | DecoderLayer( 48 | AttentionLayer(FullAttention(True, factor, attention_dropout=dropout), 49 | d_model, n_heads), 50 | AttentionLayer(FullAttention(False, factor, attention_dropout=dropout), 51 | d_model, n_heads), 52 | d_model, 53 | d_ff, 54 | dropout=dropout, 55 | activation=activation, 56 | ) 57 | for l in range(d_layers) 58 | ], 59 | norm_layer=torch.nn.LayerNorm(d_model) 60 | ) 61 | # self.end_conv1 = nn.Conv1d(in_channels=label_len+out_len, out_channels=out_len, kernel_size=1, bias=True) 62 | # self.end_conv2 = nn.Conv1d(in_channels=d_model, out_channels=c_out, kernel_size=1, bias=True) 63 | self.projection = nn.Linear(d_model, c_out, bias=True) 64 | 65 | def forward(self, x_enc, x_mark_enc, x_dec, x_mark_dec, 66 | enc_self_mask=None, dec_self_mask=None, dec_enc_mask=None): 67 | enc_out = self.enc_embedding(x_enc, x_mark_enc) 68 | enc_out = self.encoder(enc_out, attn_mask=enc_self_mask) 69 | 70 | dec_out = self.dec_embedding(x_dec, x_mark_dec) 71 | dec_out = self.decoder(dec_out, enc_out, x_mask=dec_self_mask, cross_mask=dec_enc_mask) 72 | dec_out = self.projection(dec_out) 73 | 74 | # dec_out = self.end_conv1(dec_out) 75 | # dec_out = self.end_conv2(dec_out.transpose(2,1)).transpose(1,2) 76 | return dec_out[:,-self.pred_len:,:] # [B, L, D] 77 | -------------------------------------------------------------------------------- /main_vrae_dad.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | from exp.exp_vrae_dad import Exp_VRAE_DAD 5 | 6 | parser = argparse.ArgumentParser(description='[VRAE] LSTM-VAE for time series modeling') 7 | 8 | parser.add_argument('--model', type=str, required=True, default='vrae',help='model of the experiment') 9 | 10 | parser.add_argument('--data', type=str, required=True, default='WADI', help='data') 11 | parser.add_argument('--root_path', type=str, default='./data/graph_data/', help='root path of the data file') 12 | parser.add_argument('--data_path', type=str, default='WADI_14days_downsampled.csv', help='location of the data file') 13 | parser.add_argument('--features', type=str, default='M', help='features [S, M]') 14 | parser.add_argument('--target', type=str, default='OT', help='target feature') 15 | 16 | parser.add_argument('--seq_len', type=int, default=48, help='input series length') 17 | parser.add_argument('--label_len', type=int, default=12, help='help series length') 18 | parser.add_argument('--pred_len', type=int, default=6, help='predict series length') 19 | parser.add_argument('--enc_in', type=int, default=122, help='encoder input size') 20 | parser.add_argument('--enc_hid', type=int, default=256, help='encoder block hidden size') 21 | parser.add_argument('--dec_hid', type=int, default=256, help='decoder block hidden size') 22 | parser.add_argument('--e_layers', type=int, default=2, help='num of encoder layers') 23 | parser.add_argument('--d_layers', type=int, default=2, help='num of decoder layers') 24 | parser.add_argument('--d_lat', type=int, default=64, help='dimensionality of latent space') 25 | parser.add_argument('--block', type=str, default='LSTM', help='rnn cell type') 26 | 27 | parser.add_argument('--dropout', type=float, default=0.05, help='dropout') 28 | parser.add_argument('--num_workers', type=int, default=0, help='data loader num workers') 29 | 30 | parser.add_argument('--itr', type=int, default=2, help='each params run iteration') 31 | parser.add_argument('--train_epochs', type=int, default=6, help='train epochs') 32 | parser.add_argument('--batch_size', type=int, default=32, help='input data batch size') 33 | parser.add_argument('--patience', type=int, default=3, help='early stopping patience') 34 | parser.add_argument('--learning_rate', type=float, default=0.0001, help='optimizer learning rate') 35 | parser.add_argument('--des', type=str, default='test',help='exp description') 36 | parser.add_argument('--loss', type=str, default='mse',help='loss function') 37 | parser.add_argument('--lradj', type=str, default='type1',help='adjust learning rate') 38 | parser.add_argument('--clip', type=bool, default=False, help='if clipping the gradients') 39 | parser.add_argument('--max_grad_norm', type=int, default=5, help='max gradient normalization coefficient') 40 | 41 | parser.add_argument('--use_gpu', type=bool, default=True, help='use gpu') 42 | parser.add_argument('--gpu', type=int, default=0, help='gpu') 43 | 44 | args = parser.parse_args() 45 | 46 | data_parser = { 47 | 'ETTh1':{'data':'ETTh1.csv','T':'OT','M':[7,7,7],'S':[1,1,1]}, 48 | 'ETTh2':{'data':'ETTh2.csv','T':'OT','M':[7,7,7],'S':[1,1,1]}, 49 | 'ETTm1':{'data':'ETTm1.csv','T':'OT','M':[7,7,7],'S':[1,1,1]}, 50 | 'SolarEnergy':{'data':'solar_energy.csv','T':'0','M':[137,137,137],'S':[1,1,1]}, 51 | 'WADI':{'data':'WADI_14days_downsampled.csv','T':'1_AIT_005_PV','M':[122,122,122],'S':[1,1,1]}, 52 | 'SMAP':{'data':'SMAP','T':0,'M':[25,25,25],'S':[1,1,1]}, 53 | 'MSL':{'data':'MSL','T':0,'M':[55,55,55],'S':[1,1,1]}, 54 | } 55 | if args.data in data_parser.keys(): 56 | data_info = data_parser[args.data] 57 | args.data_path = data_info['data'] 58 | args.target = data_info['T'] 59 | args.enc_in, _, _ = data_info[args.features] 60 | 61 | Exp = Exp_VRAE_DAD 62 | 63 | for ii in range(args.itr): 64 | setting = '{}_{}_ft{}_sl{}_eh{}_dh{}_el{}_dl{}_la{}_bl{}_{}_{}'.format(args.model, args.data, args.features, 65 | args.seq_len, args.enc_hid, args.dec_hid, args.e_layers, args.d_layers, args.d_lat, args.block, args.des, ii) 66 | 67 | exp = Exp(args) 68 | print('>>>>>>>start training : {}>>>>>>>>>>>>>>>>>>>>>>>>>>'.format(setting)) 69 | exp.train(setting) 70 | 71 | print('>>>>>>>testing : {}<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'.format(setting)) 72 | exp.test(setting) 73 | -------------------------------------------------------------------------------- /main_informer.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | from exp.exp_informer import Exp_Informer 5 | 6 | parser = argparse.ArgumentParser(description='[Informer] Long Sequences Forecasting') 7 | 8 | parser.add_argument('--model', type=str, required=True, default='informer',help='model of the experiment') 9 | 10 | parser.add_argument('--data', type=str, required=True, default='ETTh1', help='data') 11 | parser.add_argument('--root_path', type=str, default='./data/ETT-small/', help='root path of the data file') 12 | parser.add_argument('--data_path', type=str, default='ETTh1.csv', help='location of the data file') 13 | parser.add_argument('--features', type=str, default='S', help='features [S, M]') 14 | parser.add_argument('--target', type=str, default='OT', help='target feature') 15 | 16 | parser.add_argument('--seq_len', type=int, default=96, help='input series length') 17 | parser.add_argument('--label_len', type=int, default=48, help='help series length') 18 | parser.add_argument('--pred_len', type=int, default=24, help='predict series length') 19 | parser.add_argument('--enc_in', type=int, default=7, help='encoder input size') 20 | parser.add_argument('--dec_in', type=int, default=7, help='decoder input size') 21 | parser.add_argument('--c_out', type=int, default=7, help='output size') 22 | parser.add_argument('--d_model', type=int, default=128, help='dimension of model') 23 | parser.add_argument('--n_heads', type=int, default=8, help='num of heads') 24 | parser.add_argument('--e_layers', type=int, default=3, help='num of encoder layers') 25 | parser.add_argument('--d_layers', type=int, default=2, help='num of decoder layers') 26 | parser.add_argument('--d_ff', type=int, default=128, help='dimension of fcn') 27 | parser.add_argument('--factor', type=int, default=5, help='prob sparse factor') 28 | 29 | parser.add_argument('--dropout', type=float, default=0.05, help='dropout') 30 | parser.add_argument('--attn', type=str, default='prob', help='attention [prob, full]') 31 | parser.add_argument('--embed', type=str, default='fixed', help='embedding type [fixed, learned]') 32 | parser.add_argument('--activation', type=str, default='gelu',help='activation') 33 | parser.add_argument('--num_workers', type=int, default=0, help='data loader num workers') 34 | 35 | parser.add_argument('--itr', type=int, default=2, help='each params run iteration') 36 | parser.add_argument('--train_epochs', type=int, default=6, help='train epochs') 37 | parser.add_argument('--batch_size', type=int, default=32, help='input data batch size') 38 | parser.add_argument('--patience', type=int, default=3, help='early stopping patience') 39 | parser.add_argument('--learning_rate', type=float, default=0.0001, help='optimizer learning rate') 40 | parser.add_argument('--des', type=str, default='test',help='exp description') 41 | parser.add_argument('--loss', type=str, default='mse',help='loss function') 42 | parser.add_argument('--lradj', type=str, default='type1',help='adjust learning rate') 43 | 44 | parser.add_argument('--use_gpu', type=bool, default=True, help='use gpu') 45 | parser.add_argument('--gpu', type=int, default=0, help='gpu') 46 | 47 | args = parser.parse_args() 48 | 49 | data_parser = { 50 | 'ETTh1':{'data':'ETTh1.csv','T':'OT','M':[7,7,7],'S':[1,1,1]}, 51 | 'ETTh2':{'data':'ETTh2.csv','T':'OT','M':[7,7,7],'S':[1,1,1]}, 52 | 'ETTm1':{'data':'ETTm1.csv','T':'OT','M':[7,7,7],'S':[1,1,1]}, 53 | 'SolarEnergy':{'data':'solar_energy.csv','T':'0','M':[137,137,137],'S':[1,1,1]}, 54 | 'WADI':{'data':'WADI_14days_downsampled.csv','T':'1_LS_001_AL','M':[122,122,122],'S':[1,1,1]}, 55 | 'SMAP':{'data':'SMAP_train.pkl','T':0,'M':[25,25,25],'S':[1,1,1]}, 56 | } 57 | if args.data in data_parser.keys(): 58 | data_info = data_parser[args.data] 59 | args.data_path = data_info['data'] 60 | args.target = data_info['T'] 61 | args.enc_in, args.dec_in, args.c_out = data_info[args.features] 62 | 63 | Exp = Exp_Informer 64 | 65 | for ii in range(args.itr): 66 | setting = '{}_{}_ft{}_sl{}_ll{}_pl{}_dm{}_nh{}_el{}_dl{}_df{}_at{}_eb{}_{}_{}'.format(args.model, args.data, args.features, 67 | args.seq_len, args.label_len, args.pred_len, 68 | args.d_model, args.n_heads, args.e_layers, args.d_layers, args.d_ff, args.attn, args.embed, args.des, ii) 69 | 70 | exp = Exp(args) 71 | print('>>>>>>>start training : {}>>>>>>>>>>>>>>>>>>>>>>>>>>'.format(setting)) 72 | exp.train(setting) 73 | 74 | print('>>>>>>>testing : {}<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'.format(setting)) 75 | exp.test(setting) 76 | -------------------------------------------------------------------------------- /main_informer_dad.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | from exp.exp_informer_dad import Exp_Informer_DAD 5 | 6 | parser = argparse.ArgumentParser(description='[Informer] Long Sequences Forecasting') 7 | 8 | parser.add_argument('--model', type=str, required=True, default='informer',help='model of the experiment') 9 | 10 | parser.add_argument('--data', type=str, required=True, default='ETTh1', help='data') 11 | parser.add_argument('--root_path', type=str, default='./data/ETT-small/', help='root path of the data file') 12 | parser.add_argument('--data_path', type=str, default='ETTh1.csv', help='location of the data file') 13 | parser.add_argument('--features', type=str, default='M', help='features [S, M]') 14 | parser.add_argument('--target', type=str, default='OT', help='target feature') 15 | 16 | parser.add_argument('--seq_len', type=int, default=96, help='input series length') 17 | parser.add_argument('--label_len', type=int, default=48, help='help series length') 18 | parser.add_argument('--pred_len', type=int, default=24, help='predict series length') 19 | parser.add_argument('--enc_in', type=int, default=7, help='encoder input size') 20 | parser.add_argument('--dec_in', type=int, default=7, help='decoder input size') 21 | parser.add_argument('--c_out', type=int, default=7, help='output size') 22 | parser.add_argument('--d_model', type=int, default=128, help='dimension of model') 23 | parser.add_argument('--n_heads', type=int, default=8, help='num of heads') 24 | parser.add_argument('--e_layers', type=int, default=3, help='num of encoder layers') 25 | parser.add_argument('--d_layers', type=int, default=2, help='num of decoder layers') 26 | parser.add_argument('--d_ff', type=int, default=128, help='dimension of fcn') 27 | parser.add_argument('--factor', type=int, default=5, help='prob sparse factor') 28 | 29 | parser.add_argument('--dropout', type=float, default=0.05, help='dropout') 30 | parser.add_argument('--attn', type=str, default='prob', help='attention [prob, full]') 31 | parser.add_argument('--embed', type=str, default='fixed', help='embedding type [fixed, learned]') 32 | parser.add_argument('--activation', type=str, default='gelu',help='activation') 33 | parser.add_argument('--num_workers', type=int, default=0, help='data loader num workers') 34 | 35 | parser.add_argument('--itr', type=int, default=2, help='each params run iteration') 36 | parser.add_argument('--train_epochs', type=int, default=6, help='train epochs') 37 | parser.add_argument('--batch_size', type=int, default=32, help='input data batch size') 38 | parser.add_argument('--patience', type=int, default=3, help='early stopping patience') 39 | parser.add_argument('--learning_rate', type=float, default=0.0001, help='optimizer learning rate') 40 | parser.add_argument('--des', type=str, default='test',help='exp description') 41 | parser.add_argument('--loss', type=str, default='mse',help='loss function') 42 | parser.add_argument('--lradj', type=str, default='type1',help='adjust learning rate') 43 | 44 | parser.add_argument('--use_gpu', type=bool, default=True, help='use gpu') 45 | parser.add_argument('--gpu', type=int, default=0, help='gpu') 46 | 47 | args = parser.parse_args() 48 | 49 | data_parser = { 50 | 'ETTh1':{'data':'ETTh1.csv','T':'OT','M':[7,7,7],'S':[1,1,1]}, 51 | 'ETTh2':{'data':'ETTh2.csv','T':'OT','M':[7,7,7],'S':[1,1,1]}, 52 | 'ETTm1':{'data':'ETTm1.csv','T':'OT','M':[7,7,7],'S':[1,1,1]}, 53 | 'SolarEnergy':{'data':'solar_energy.csv','T':'0','M':[137,137,137],'S':[1,1,1]}, 54 | 'WADI':{'data':'WADI_14days_downsampled.csv','T':'1_LS_001_AL','M':[122,122,122],'S':[1,1,1]}, 55 | 'SMAP':{'data':'SMAP','T':0,'M':[25,25,25],'S':[1,1,1]}, 56 | 'MSL':{'data':'MSL','T':0,'M':[55,55,55],'S':[1,1,1]}, 57 | } 58 | if args.data in data_parser.keys(): 59 | data_info = data_parser[args.data] 60 | args.data_path = data_info['data'] 61 | args.target = data_info['T'] 62 | args.enc_in, args.dec_in, args.c_out = data_info[args.features] 63 | 64 | Exp = Exp_Informer_DAD 65 | 66 | for ii in range(args.itr): 67 | setting = '{}_{}_ft{}_sl{}_ll{}_pl{}_dm{}_nh{}_el{}_dl{}_df{}_at{}_eb{}_{}_{}'.format(args.model, args.data, args.features, 68 | args.seq_len, args.label_len, args.pred_len, 69 | args.d_model, args.n_heads, args.e_layers, args.d_layers, args.d_ff, args.attn, args.embed, args.des, ii) 70 | 71 | exp = Exp(args) 72 | print('>>>>>>>start training : {}>>>>>>>>>>>>>>>>>>>>>>>>>>'.format(setting)) 73 | exp.train(setting) 74 | 75 | print('>>>>>>>testing : {}<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'.format(setting)) 76 | exp.test(setting) 77 | -------------------------------------------------------------------------------- /main_gta_dad.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | from exp.exp_gta_dad import Exp_GTA_DAD 5 | 6 | parser = argparse.ArgumentParser(description='[GTA] GTA for DAD') 7 | 8 | parser.add_argument('--model', type=str, required=True, default='gta',help='model of the experiment') 9 | 10 | parser.add_argument('--data', type=str, required=True, default='ETTh1', help='data') 11 | parser.add_argument('--root_path', type=str, default='./data/ETT-small/', help='root path of the data file') 12 | parser.add_argument('--data_path', type=str, default='ETTh1.csv', help='location of the data file') 13 | parser.add_argument('--features', type=str, default='M', help='features [S, M]') 14 | parser.add_argument('--target', type=str, default='OT', help='target feature') 15 | 16 | parser.add_argument('--seq_len', type=int, default=60, help='input series length') 17 | parser.add_argument('--label_len', type=int, default=30, help='help series length') 18 | parser.add_argument('--pred_len', type=int, default=24, help='predict series length') 19 | parser.add_argument('--num_nodes', type=int, default=7, help='encoder input size') 20 | parser.add_argument('--num_levels', type=int, default=3, help='number of dilated levels for graph embedding') 21 | # parser.add_argument('--dec_in', type=int, default=7, help='decoder input size') 22 | # parser.add_argument('--c_out', type=int, default=7, help='output size') 23 | parser.add_argument('--d_model', type=int, default=128, help='dimension of model') 24 | parser.add_argument('--n_heads', type=int, default=8, help='num of heads') 25 | parser.add_argument('--e_layers', type=int, default=3, help='num of encoder layers') 26 | parser.add_argument('--d_layers', type=int, default=2, help='num of decoder layers') 27 | parser.add_argument('--d_ff', type=int, default=128, help='dimension of fcn') 28 | parser.add_argument('--factor', type=int, default=5, help='prob sparse factor') 29 | 30 | parser.add_argument('--dropout', type=float, default=0.05, help='dropout') 31 | parser.add_argument('--attn', type=str, default='prob', help='attention [prob, full]') 32 | parser.add_argument('--embed', type=str, default='fixed', help='embedding type [fixed, learned]') 33 | parser.add_argument('--activation', type=str, default='gelu',help='activation') 34 | parser.add_argument('--num_workers', type=int, default=0, help='data loader num workers') 35 | 36 | parser.add_argument('--itr', type=int, default=2, help='each params run iteration') 37 | parser.add_argument('--train_epochs', type=int, default=6, help='train epochs') 38 | parser.add_argument('--batch_size', type=int, default=32, help='input data batch size') 39 | parser.add_argument('--patience', type=int, default=3, help='early stopping patience') 40 | parser.add_argument('--learning_rate', type=float, default=0.0001, help='optimizer learning rate') 41 | parser.add_argument('--des', type=str, default='test',help='exp description') 42 | parser.add_argument('--loss', type=str, default='mse',help='loss function') 43 | parser.add_argument('--lradj', type=str, default='type1',help='adjust learning rate') 44 | 45 | parser.add_argument('--use_gpu', type=bool, default=True, help='use gpu') 46 | parser.add_argument('--gpu', type=int, default=0, help='gpu') 47 | 48 | args = parser.parse_args() 49 | 50 | data_parser = { 51 | 'ETTh1':{'data':'ETTh1.csv','T':'OT','M':[7,7,7],'S':[1,1,1]}, 52 | 'ETTh2':{'data':'ETTh2.csv','T':'OT','M':[7,7,7],'S':[1,1,1]}, 53 | 'ETTm1':{'data':'ETTm1.csv','T':'OT','M':[7,7,7],'S':[1,1,1]}, 54 | 'SolarEnergy':{'data':'solar_energy.csv','T':'0','M':[137,137,137],'S':[1,1,1]}, 55 | 'WADI':{'data':'WADI_14days_downsampled.csv','T':'1_LS_001_AL','M':112,'S':1}, 56 | 'SMAP':{'data':'SMAP','T':0,'M':25,'S':1}, 57 | 'MSL':{'data':'MSL','T':0,'M':55,'S':1}, 58 | 'SWaT':{'data':'SWaT','T':'FIT_101','M':51,'S':1} 59 | } 60 | if args.data in data_parser.keys(): 61 | data_info = data_parser[args.data] 62 | args.data_path = data_info['data'] 63 | args.target = data_info['T'] 64 | args.num_nodes = data_info[args.features] 65 | 66 | Exp = Exp_GTA_DAD 67 | 68 | for ii in range(args.itr): 69 | setting = '{}_{}_ft{}_sl{}_ll{}_pl{}_nl{}_dm{}_nh{}_el{}_dl{}_df{}_at{}_eb{}_{}_{}'.format(args.model, args.data, args.features, 70 | args.seq_len, args.label_len, args.pred_len, args.num_levels, 71 | args.d_model, args.n_heads, args.e_layers, args.d_layers, args.d_ff, args.attn, args.embed, args.des, ii) 72 | 73 | exp = Exp(args) 74 | print('>>>>>>>start training : {}>>>>>>>>>>>>>>>>>>>>>>>>>>'.format(setting)) 75 | exp.train(setting) 76 | 77 | print('>>>>>>>testing : {}<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'.format(setting)) 78 | exp.test(setting) 79 | -------------------------------------------------------------------------------- /models/embed.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | import math 6 | 7 | class PositionalEmbedding(nn.Module): 8 | def __init__(self, d_model, max_len=5000): 9 | super(PositionalEmbedding, self).__init__() 10 | # Compute the positional encodings once in log space. 11 | pe = torch.zeros(max_len, d_model).float() 12 | pe.require_grad = False 13 | 14 | position = torch.arange(0, max_len).float().unsqueeze(1) 15 | div_term = (torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model)).exp() 16 | 17 | pe[:, 0::2] = torch.sin(position * div_term) 18 | pe[:, 1::2] = torch.cos(position * div_term) 19 | 20 | pe = pe.unsqueeze(0) 21 | self.register_buffer('pe', pe) 22 | 23 | def forward(self, x): 24 | return self.pe[:, :x.size(1)] 25 | 26 | class TokenEmbedding(nn.Module): 27 | def __init__(self, c_in, d_model): 28 | super(TokenEmbedding, self).__init__() 29 | padding = 1 if torch.__version__>='1.5.0' else 2 30 | self.tokenConv = nn.Conv1d(in_channels=c_in, out_channels=d_model, 31 | kernel_size=3, padding=padding, padding_mode='circular') 32 | for m in self.modules(): 33 | if isinstance(m, nn.Conv1d): 34 | nn.init.kaiming_normal_(m.weight, mode='fan_in', nonlinearity='leaky_relu') 35 | 36 | def forward(self, x): 37 | x = self.tokenConv(x.permute(0, 2, 1)).transpose(1,2) 38 | return x 39 | 40 | class FixedEmbedding(nn.Module): 41 | def __init__(self, c_in, d_model): 42 | super(FixedEmbedding, self).__init__() 43 | 44 | w = torch.zeros(c_in, d_model).float() 45 | w.require_grad = False 46 | 47 | position = torch.arange(0, c_in).float().unsqueeze(1) 48 | div_term = (torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model)).exp() 49 | 50 | w[:, 0::2] = torch.sin(position * div_term) 51 | w[:, 1::2] = torch.cos(position * div_term) 52 | 53 | self.emb = nn.Embedding(c_in, d_model) 54 | self.emb.weight = nn.Parameter(w, requires_grad=False) 55 | 56 | def forward(self, x): 57 | return self.emb(x).detach() 58 | 59 | class TemporalEmbedding(nn.Module): 60 | def __init__(self, d_model, embed_type='fixed', data='ETTh'): 61 | super(TemporalEmbedding, self).__init__() 62 | 63 | minute_size = 4; hour_size = 24 64 | weekday_size = 7; day_size = 32; month_size = 13 65 | 66 | Embed = FixedEmbedding if embed_type=='fixed' else nn.Embedding 67 | if data == 'ETTm': 68 | self.minute_embed = Embed(minute_size, d_model) 69 | elif data == 'SolarEnergy': 70 | minute_size = 6 71 | self.minute_embed = Embed(minute_size, d_model) 72 | elif data == 'WADI': 73 | minute_size = 60 74 | second_size = 6 75 | self.minute_embed = Embed(minute_size, d_model) 76 | self.second_emebd = Embed(second_size, d_model) 77 | elif data == 'SMAP': 78 | minute_size = 60 79 | second_size = 15 80 | self.minute_embed = Embed(minute_size, d_model) 81 | self.second_emebd = Embed(second_size, d_model) 82 | self.hour_embed = Embed(hour_size, d_model) 83 | self.weekday_embed = Embed(weekday_size, d_model) 84 | self.day_embed = Embed(day_size, d_model) 85 | self.month_embed = Embed(month_size, d_model) 86 | 87 | def forward(self, x): 88 | x = x.long() 89 | 90 | second_x = self.second_emebd(x[:,:,5]) if hasattr(self, 'second_embed') else 0. 91 | minute_x = self.minute_embed(x[:,:,4]) if hasattr(self, 'minute_embed') else 0. 92 | hour_x = self.hour_embed(x[:,:,3]) 93 | weekday_x = self.weekday_embed(x[:,:,2]) 94 | day_x = self.day_embed(x[:,:,1]) 95 | month_x = self.month_embed(x[:,:,0]) 96 | 97 | return hour_x + weekday_x + day_x + month_x + minute_x + second_x 98 | 99 | class DataEmbedding(nn.Module): 100 | def __init__(self, c_in, d_model, embed_type='fixed', data='ETTh', dropout=0.1): 101 | super(DataEmbedding, self).__init__() 102 | 103 | self.value_embedding = TokenEmbedding(c_in=c_in, d_model=d_model) 104 | self.position_embedding = PositionalEmbedding(d_model=d_model) 105 | # self.temporal_embedding = TemporalEmbedding(d_model=d_model, embed_type=embed_type, data=data) 106 | 107 | self.dropout = nn.Dropout(p=dropout) 108 | 109 | def forward(self, x, x_mark): 110 | # x = self.value_embedding(x) + self.position_embedding(x) + self.temporal_embedding(x_mark) 111 | x = self.value_embedding(x) + self.position_embedding(x) 112 | 113 | return self.dropout(x) 114 | -------------------------------------------------------------------------------- /models/attn.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | import numpy as np 6 | 7 | from math import sqrt 8 | from utils.masking import TriangularCausalMask, ProbMask 9 | 10 | class FullAttention(nn.Module): 11 | def __init__(self, mask_flag=True, factor=5, scale=None, attention_dropout=0.1): 12 | super(FullAttention, self).__init__() 13 | self.scale = scale 14 | self.mask_flag = mask_flag 15 | self.dropout = nn.Dropout(attention_dropout) 16 | 17 | def forward(self, queries, keys, values, attn_mask): 18 | B, L, H, E = queries.shape 19 | _, S, _, D = values.shape 20 | scale = self.scale or 1./sqrt(E) 21 | 22 | scores = torch.einsum("blhe,bshe->bhls", queries, keys) 23 | if self.mask_flag: 24 | if attn_mask is None: 25 | attn_mask = TriangularCausalMask(B, L, device=queries.device) 26 | 27 | scores.masked_fill_(attn_mask.mask, -np.inf) 28 | 29 | A = self.dropout(torch.softmax(scale * scores, dim=-1)) 30 | V = torch.einsum("bhls,bshd->blhd", A, values) 31 | 32 | return V.contiguous() 33 | 34 | class ProbAttention(nn.Module): 35 | def __init__(self, mask_flag=True, factor=5, scale=None, attention_dropout=0.1): 36 | super(ProbAttention, self).__init__() 37 | self.factor = factor 38 | self.scale = scale 39 | self.mask_flag = mask_flag 40 | self.dropout = nn.Dropout(attention_dropout) 41 | 42 | def _prob_QK(self, Q, K, sample_k, n_top): 43 | # Q [B, H, L, D] 44 | B, H, L, E = K.shape 45 | _, _, S, _ = Q.shape 46 | 47 | # calculate the sampled Q_K 48 | K_expand = K.unsqueeze(-3).expand(B, H, S, L, E) 49 | indx_sample = torch.randint(L, (S, sample_k)) 50 | K_sample = K_expand[:, :, torch.arange(S).unsqueeze(1), indx_sample, :] 51 | Q_K_sample = torch.matmul(Q.unsqueeze(-2), K_sample.transpose(-2, -1)).squeeze() 52 | 53 | # find the Top_k query with sparisty measurement 54 | M = Q_K_sample.max(-1)[0] - torch.div(Q_K_sample.sum(-1), L) 55 | M_top = M.topk(n_top, sorted=False)[1] 56 | 57 | # use the reduced Q to calculate Q_K 58 | Q_reduce = Q[torch.arange(B)[:, None, None], 59 | torch.arange(H)[None, :, None], 60 | M_top, :] 61 | Q_K = torch.matmul(Q_reduce, K.transpose(-2, -1)) 62 | 63 | return Q_K, M_top 64 | 65 | def _get_initial_context(self, V, L_Q): 66 | B, H, L_V, D = V.shape 67 | if not self.mask_flag: 68 | V_sum = V.sum(dim=-2) 69 | contex = V_sum.unsqueeze(-2).expand(B, H, L_Q, V_sum.shape[-1]).clone() 70 | else: # use mask 71 | assert(L_Q == L_V) # requires that L_Q == L_V, i.e. for self-attention only 72 | contex = V.cumsum(dim=-1) 73 | return contex 74 | 75 | def _update_context(self, context_in, V, scores, index, L_Q, attn_mask): 76 | B, H, L_V, D = V.shape 77 | 78 | if self.mask_flag: 79 | attn_mask = ProbMask(B, H, L_Q, index, scores, device=V.device) 80 | scores.masked_fill_(attn_mask.mask, -np.inf) 81 | 82 | attn = torch.softmax(scores, dim=-1) # nn.Softmax(dim=-1)(scores) 83 | 84 | context_in[torch.arange(B)[:, None, None], 85 | torch.arange(H)[None, :, None], 86 | index, :] = torch.matmul(attn, V) 87 | return context_in 88 | 89 | def forward(self, queries, keys, values, attn_mask): 90 | B, L, H, D = queries.shape 91 | _, S, _, _ = keys.shape 92 | 93 | queries = queries.view(B, H, L, -1) 94 | keys = keys.view(B, H, S, -1) 95 | values = values.view(B, H, S, -1) 96 | 97 | U = self.factor * np.ceil(np.log(S)).astype('int').item() 98 | u = self.factor * np.ceil(np.log(L)).astype('int').item() 99 | 100 | scores_top, index = self._prob_QK(queries, keys, u, U) 101 | # add scale factor 102 | scale = self.scale or 1./sqrt(D) 103 | if scale is not None: 104 | scores_top = scores_top * scale 105 | # get the context 106 | context = self._get_initial_context(values, L) 107 | # update the context with selected top_k queries 108 | context = self._update_context(context, values, scores_top, index, L, attn_mask) 109 | 110 | return context.contiguous() 111 | 112 | 113 | class AttentionLayer(nn.Module): 114 | def __init__(self, attention, d_model, n_heads, d_keys=None, 115 | d_values=None): 116 | super(AttentionLayer, self).__init__() 117 | 118 | d_keys = d_keys or (d_model//n_heads) 119 | d_values = d_values or (d_model//n_heads) 120 | 121 | self.inner_attention = attention 122 | self.query_projection = nn.Linear(d_model, d_keys * n_heads) 123 | self.key_projection = nn.Linear(d_model, d_keys * n_heads) 124 | self.value_projection = nn.Linear(d_model, d_values * n_heads) 125 | self.out_projection = nn.Linear(d_values * n_heads, d_model) 126 | self.n_heads = n_heads 127 | 128 | def forward(self, queries, keys, values, attn_mask): 129 | B, L, _ = queries.shape 130 | _, S, _ = keys.shape 131 | H = self.n_heads 132 | 133 | queries = self.query_projection(queries).view(B, L, H, -1) 134 | keys = self.key_projection(keys).view(B, S, H, -1) 135 | values = self.value_projection(values).view(B, S, H, -1) 136 | 137 | out = self.inner_attention( 138 | queries, 139 | keys, 140 | values, 141 | attn_mask 142 | ).view(B, L, -1) 143 | 144 | return self.out_projection(out) -------------------------------------------------------------------------------- /models/gta.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.nn import Sequential as Seq, Linear, ReLU, Parameter 3 | from torch_geometric.nn import MessagePassing, GCNConv 4 | from torch_geometric.nn.conv.gcn_conv import gcn_norm 5 | from torch_geometric.nn.inits import glorot, zeros 6 | from torch_geometric.utils import remove_self_loops, add_self_loops 7 | from models.model import Informer 8 | from models.tconv import TemporalBlock 9 | import torch.nn.functional as F 10 | 11 | 12 | class AdaGCNConv(MessagePassing): 13 | def __init__(self, num_nodes, in_channels, out_channels, improved=False, 14 | add_self_loops=False, normalize=True, bias=True, init_method='all'): 15 | super(AdaGCNConv, self).__init__(aggr='add', node_dim=0) # "Max" aggregation. 16 | self.num_nodes = num_nodes 17 | self.in_channels = in_channels 18 | self.out_channels = out_channels 19 | self.improved = improved 20 | self.add_self_loops = add_self_loops 21 | self.normalize = normalize 22 | self.bias = bias 23 | self.init_method = init_method 24 | 25 | self.weight = Parameter(torch.Tensor(in_channels, out_channels)) 26 | 27 | if bias: 28 | self.bias = Parameter(torch.Tensor(out_channels)) 29 | else: 30 | self.register_parameter('bias', None) 31 | 32 | self._init_graph_logits_() 33 | 34 | self.reset_parameters() 35 | 36 | def _init_graph_logits_(self): 37 | if self.init_method == 'all': 38 | logits = .8 * torch.ones(self.num_nodes ** 2, 2) 39 | logits[:, 1] = 0 40 | elif self.init_method == 'random': 41 | logits = 1e-3 * torch.randn(self.num_nodes ** 2, 2) 42 | elif self.init_method == 'equal': 43 | logits = .5 * torch.ones(self.num_nodes ** 2, 2) 44 | else: 45 | raise NotImplementedError('Initial Method %s is not implemented' % self.init_method) 46 | 47 | self.register_parameter('logits', Parameter(logits, requires_grad=True)) 48 | 49 | def reset_parameters(self): 50 | glorot(self.weight) 51 | zeros(self.bias) 52 | 53 | def forward(self, x, edge_index, edge_weight=None): 54 | # x has shape [N, in_channels] 55 | # edge_index has shape [2, E] 56 | if self.normalize: 57 | edge_index, edge_weight = gcn_norm( # yapf: disable 58 | edge_index, edge_weight, x.size(self.node_dim), 59 | self.improved, self.add_self_loops, dtype=x.dtype) 60 | 61 | z = torch.nn.functional.gumbel_softmax(self.logits, hard=True) 62 | 63 | x = torch.matmul(x, self.weight) 64 | 65 | # propagate_type: (x: Tensor, edge_weight: OptTensor) 66 | out = self.propagate(edge_index, x=x, edge_weight=edge_weight, 67 | size=None, z=z) 68 | 69 | if self.bias is not None: 70 | out += self.bias 71 | 72 | return out 73 | 74 | def message(self, x_j, edge_weight, z): 75 | if edge_weight is None: 76 | return x_j * z[:, 0].contiguous().view([-1] + [1] * (x_j.dim() - 1)) 77 | else: 78 | return edge_weight.view([-1] + [1] * (x_j.dim() - 1)) * x_j * z[:, 0].contiguous().view([-1] + [1] * (x_j.dim() - 1)) 79 | 80 | def __repr__(self): 81 | return '{}({}, {})'.format(self.__class__.__name__, self.in_channels, 82 | self.out_channels) 83 | 84 | 85 | class GraphTemporalEmbedding(torch.nn.Module): 86 | def __init__(self, num_nodes, seq_len, num_levels, kernel_size=3, dropout=0.02, device=torch.device('cuda:0')): 87 | super(GraphTemporalEmbedding, self).__init__() 88 | self.num_nodes = num_nodes 89 | self.seq_len = seq_len 90 | self.num_levels = num_levels 91 | self.device = device 92 | assert (kernel_size - 1) // 2 93 | 94 | self.tc_modules = torch.nn.ModuleList([]) 95 | self.gc_module = AdaGCNConv(num_nodes, seq_len, seq_len) 96 | for i in range(num_levels): 97 | dilation_size = 2 ** i 98 | self.tc_modules.extend([TemporalBlock(num_nodes, num_nodes, kernel_size=kernel_size, stride=1, dilation=dilation_size, 99 | padding=(kernel_size-1) * dilation_size // 2, dropout=dropout)]) 100 | # self.gc_modules.extend([AdaGCNConv(num_nodes, seq_len, seq_len)]) 101 | 102 | source_nodes, target_nodes = [], [] 103 | for i in range(num_nodes): 104 | for j in range(num_nodes): 105 | source_nodes.append(j) 106 | target_nodes.append(i) 107 | self.edge_index = torch.tensor([source_nodes, target_nodes], dtype=torch.long, device=self.device) 108 | 109 | def forward(self, x): 110 | # >> (bsz, seq_len, num_nodes) 111 | x = x.permute(0, 2, 1) # >> (bsz, num_nodes, seq_len) 112 | 113 | x = self.tc_modules[0](x) # >> (bsz, num_nodes, seq_len) 114 | x = self.gc_modules[0](x.transpose(0, 1), self.edge_index).transpose(0, 1) # >> (bsz, num_nodes, seq_len) 115 | # output = x 116 | 117 | for i in range(1, self.num_levels): 118 | x = self.tc_modules[i](x) # >> (bsz, num_nodes, seq_len) 119 | x = self.gc_module(x.transpose(0, 1), self.edge_index).transpose(0, 1) # >> (bsz, num_nodes, seq_len) 120 | # output += x 121 | 122 | # return output.transpose(1, 2) # >> (bsz, seq_len, num_nodes) 123 | return x.transpose(1, 2) 124 | 125 | 126 | class GTA(torch.nn.Module): 127 | def __init__(self, num_nodes, seq_len, label_len, out_len, num_levels, 128 | factor=5, d_model=512, n_heads=8, e_layers=3, d_layers=2, d_ff=512, 129 | dropout=0.0, attn='prob', embed='fixed', data='ETTh', activation='gelu', 130 | device=torch.device('cuda:0')): 131 | super(GTA, self).__init__() 132 | self.num_nodes = num_nodes 133 | self.seq_len = seq_len 134 | self.label_len = label_len 135 | self.out_len = out_len 136 | self.num_levels = num_levels 137 | self.device = device 138 | 139 | self.gt_embedding = GraphTemporalEmbedding(num_nodes, seq_len, num_levels, kernel_size=3, dropout=dropout, device=device) 140 | self.model = Informer(num_nodes, num_nodes, num_nodes, seq_len, label_len, out_len, 141 | factor, d_model, n_heads, e_layers, d_layers, d_ff, 142 | dropout, attn, embed, data, activation, device) 143 | 144 | def forward(self, batch_x, batch_y, batch_x_mark, batch_y_mark): 145 | # >> (bsz, seq_len, num_nodes) 146 | # batch_x = batch_x.contiguous().transpose(1, 2).transpose(0, 1) # >> (bsz, num_nodes, seq_len) >> (num_nodes, bsz, seq_len) 147 | # print(batch_x.size()) 148 | batch_x = self.gt_embedding(batch_x) # >> (bsz, seq, num_nodes) 149 | # batch_x = batch_x.transpose(0, 1).transpose(1, 2) # >> (bsz, seq_len, num_nodes) 150 | dec_inp = torch.zeros_like(batch_y[:,-self.out_len:,:]).double().to(self.device) 151 | dec_inp = torch.cat([batch_y[:,:self.label_len,:], dec_inp], dim=1).double().to(self.device) 152 | output = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark) 153 | 154 | return output -------------------------------------------------------------------------------- /exp/exp_vrae_dad.py: -------------------------------------------------------------------------------- 1 | from data.data_loader_dad import ( 2 | NASA_Anomaly, 3 | WADI 4 | ) 5 | from exp.exp_basic import Exp_Basic 6 | from models.vrae import VRAE 7 | 8 | from utils.tools import EarlyStopping, adjust_learning_rate 9 | from utils.metrics import metric 10 | 11 | import numpy as np 12 | 13 | import torch 14 | import torch.nn as nn 15 | from torch import optim 16 | from torch.utils.data import DataLoader 17 | 18 | import os 19 | import time 20 | 21 | import warnings 22 | warnings.filterwarnings('ignore') 23 | 24 | class Exp_VRAE_DAD(Exp_Basic): 25 | def __init__(self, args): 26 | super(Exp_VRAE_DAD, self).__init__(args) 27 | 28 | def _build_model(self): 29 | model_dict = { 30 | 'vrae':VRAE, 31 | } 32 | if self.args.model=='vrae': 33 | model = model_dict[self.args.model]( 34 | self.args.seq_len, 35 | self.args.enc_in, 36 | self.args.batch_size, 37 | self.args.enc_hid, 38 | self.args.dec_hid, 39 | self.args.e_layers, 40 | self.args.d_layers, 41 | self.args.d_lat, 42 | self.args.block, 43 | self.args.dropout, 44 | self.device 45 | ) 46 | 47 | return model.double() 48 | 49 | def _get_data(self, flag): 50 | args = self.args 51 | 52 | data_dict = { 53 | 'SMAP':NASA_Anomaly, 54 | 'MSL':NASA_Anomaly, 55 | 'WADI':WADI, 56 | } 57 | Data = data_dict[self.args.data] 58 | 59 | if flag == 'test': 60 | shuffle_flag = False; drop_last = True; batch_size = args.batch_size 61 | else: 62 | shuffle_flag = True; drop_last = True; batch_size = args.batch_size 63 | 64 | data_set = Data( 65 | root_path=args.root_path, 66 | data_path=args.data_path, 67 | flag=flag, 68 | size=[args.seq_len, args.label_len, args.pred_len], 69 | features=args.features, 70 | target=args.target 71 | ) 72 | print(flag, len(data_set)) 73 | data_loader = DataLoader( 74 | data_set, 75 | batch_size=batch_size, 76 | shuffle=shuffle_flag, 77 | num_workers=args.num_workers, 78 | drop_last=drop_last) 79 | 80 | return data_set, data_loader 81 | 82 | def _select_optimizer(self): 83 | model_optim = optim.Adam(self.model.parameters(), lr=self.args.learning_rate) 84 | return model_optim 85 | 86 | def _select_criterion(self): 87 | criterion = nn.MSELoss() 88 | return criterion 89 | 90 | def vali(self, vali_data, vali_loader, criterion): 91 | self.model.eval() 92 | total_loss = [] 93 | 94 | for i, (batch_x,batch_y,batch_x_mark,batch_y_mark,batch_label) in enumerate(vali_loader): 95 | batch_x = batch_x.double().to(self.device) 96 | 97 | # encoder - decoder 98 | outputs, latent = self.model(batch_x) 99 | 100 | pred = outputs.detach().cpu() 101 | true = batch_x.detach().cpu() 102 | 103 | loss = criterion(pred, true) 104 | 105 | total_loss.append(loss) 106 | 107 | total_loss = np.average(total_loss) 108 | self.model.train() 109 | return total_loss 110 | 111 | def train(self, setting): 112 | train_data, train_loader = self._get_data(flag = 'train') 113 | vali_data, vali_loader = self._get_data(flag = 'val') 114 | test_data, test_loader = self._get_data(flag = 'test') 115 | 116 | path = './checkpoints/'+setting 117 | if not os.path.exists(path): 118 | os.makedirs(path) 119 | 120 | time_now = time.time() 121 | 122 | train_steps = len(train_loader) 123 | early_stopping = EarlyStopping(patience=self.args.patience, verbose=True) 124 | 125 | model_optim = self._select_optimizer() 126 | criterion = self._select_criterion() 127 | 128 | for epoch in range(self.args.train_epochs): 129 | iter_count = 0 130 | train_loss = [] 131 | 132 | self.model.train() 133 | for i, (batch_x,batch_y,batch_x_mark,batch_y_mark) in enumerate(train_loader): 134 | iter_count += 1 135 | 136 | model_optim.zero_grad() 137 | 138 | batch_x = batch_x.double().to(self.device) 139 | 140 | # encoder - decoder 141 | loss, recon_loss, kl_loss, _ = self.model.compute_loss(batch_x, criterion) 142 | train_loss.append(loss.item()) 143 | 144 | if (i+1) % 100==0: 145 | print("\titers: {0}, epoch: {1} | loss: {2:.7f}".format(i + 1, epoch + 1, loss.item())) 146 | speed = (time.time()-time_now)/iter_count 147 | left_time = speed*((self.args.train_epochs - epoch)*train_steps - i) 148 | print('\tspeed: {:.4f}s/iter; left time: {:.4f}s'.format(speed, left_time)) 149 | iter_count = 0 150 | time_now = time.time() 151 | 152 | loss.backward() 153 | if self.args.clip: 154 | torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=self.args.max_grad_norm) 155 | model_optim.step() 156 | 157 | train_loss = np.average(train_loss) 158 | vali_loss = self.vali(vali_data, vali_loader, criterion) 159 | test_loss = self.vali(test_data, test_loader, criterion) 160 | 161 | print("Epoch: {0}, Steps: {1} | Train Loss: {2:.7f} Vali Loss: {3:.7f} Test Loss: {4:.7f}".format( 162 | epoch + 1, train_steps, train_loss, vali_loss, test_loss)) 163 | early_stopping(vali_loss, self.model, path) 164 | if early_stopping.early_stop: 165 | print("Early stopping") 166 | break 167 | 168 | adjust_learning_rate(model_optim, epoch+1, self.args) 169 | 170 | best_model_path = path+'/'+'checkpoint.pth' 171 | self.model.load_state_dict(torch.load(best_model_path)) 172 | 173 | return self.model 174 | 175 | def test(self, setting): 176 | test_data, test_loader = self._get_data(flag='test') 177 | 178 | self.model.eval() 179 | 180 | preds = [] 181 | trues = [] 182 | labels = [] 183 | 184 | with torch.no_grad(): 185 | for i, (batch_x,batch_y,batch_x_mark,batch_y_mark,batch_label) in enumerate(test_loader): 186 | batch_x = batch_x.double().to(self.device) 187 | 188 | # encoder - decoder 189 | outputs, latent = self.model(batch_x) 190 | 191 | pred = outputs.detach().cpu().numpy()#.squeeze() 192 | true = batch_x.detach().cpu().numpy()#.squeeze() 193 | batch_label = batch_label.long().detach().numpy() 194 | 195 | preds.append(pred) 196 | trues.append(true) 197 | labels.append(batch_label) 198 | 199 | preds = np.array(preds) 200 | trues = np.array(trues) 201 | labels = np.array(labels) 202 | print('test shape:', preds.shape, trues.shape) 203 | preds = preds.reshape(-1, preds.shape[-2], preds.shape[-1]) 204 | trues = trues.reshape(-1, trues.shape[-2], trues.shape[-1]) 205 | labels = labels.reshape(-1, labels.shape[-1]) 206 | print('test shape:', preds.shape, trues.shape) 207 | 208 | # result save 209 | folder_path = './results/' + setting +'/' 210 | if not os.path.exists(folder_path): 211 | os.makedirs(folder_path) 212 | 213 | mae, mse, rmse, mape, mspe = metric(preds, trues) 214 | print('mse:{}, mae:{}'.format(mse, mae)) 215 | 216 | np.save(folder_path+'metrics.npy', np.array([mae, mse, rmse, mape, mspe])) 217 | np.save(folder_path+'pred.npy', preds) 218 | np.save(folder_path+'true.npy', trues) 219 | np.save(folder_path+'label.npy', labels) 220 | 221 | return -------------------------------------------------------------------------------- /exp/exp_informer.py: -------------------------------------------------------------------------------- 1 | from data.data_loader import ( 2 | Dataset_ETT_hour, 3 | Dataset_ETT_minute, 4 | SolarEnergy, 5 | WADI, 6 | NASA_data 7 | ) 8 | from exp.exp_basic import Exp_Basic 9 | from models.model import Informer 10 | 11 | from utils.tools import EarlyStopping, adjust_learning_rate 12 | from utils.metrics import metric 13 | 14 | import numpy as np 15 | 16 | import torch 17 | import torch.nn as nn 18 | from torch import optim 19 | from torch.utils.data import DataLoader 20 | 21 | import os 22 | import time 23 | 24 | import warnings 25 | warnings.filterwarnings('ignore') 26 | 27 | class Exp_Informer(Exp_Basic): 28 | def __init__(self, args): 29 | super(Exp_Informer, self).__init__(args) 30 | 31 | def _build_model(self): 32 | model_dict = { 33 | 'informer':Informer, 34 | } 35 | if self.args.model=='informer': 36 | model = model_dict[self.args.model]( 37 | self.args.enc_in, 38 | self.args.dec_in, 39 | self.args.c_out, 40 | self.args.seq_len, 41 | self.args.label_len, 42 | self.args.pred_len, 43 | self.args.factor, 44 | self.args.d_model, 45 | self.args.n_heads, 46 | self.args.e_layers, 47 | self.args.d_layers, 48 | self.args.d_ff, 49 | self.args.dropout, 50 | self.args.attn, 51 | self.args.embed, 52 | self.args.data[:-1], 53 | self.args.activation, 54 | self.device 55 | ) 56 | 57 | return model.double() 58 | 59 | def _get_data(self, flag): 60 | args = self.args 61 | 62 | data_dict = { 63 | 'ETTh1':Dataset_ETT_hour, 64 | 'ETTh2':Dataset_ETT_hour, 65 | 'ETTm1':Dataset_ETT_minute, 66 | 'SolarEnergy':SolarEnergy, 67 | 'WADI':WADI, 68 | 'SMAP':NASA_data 69 | } 70 | Data = data_dict[self.args.data] 71 | 72 | if flag == 'test': 73 | shuffle_flag = False; drop_last = True; batch_size = args.batch_size 74 | else: 75 | shuffle_flag = True; drop_last = True; batch_size = args.batch_size 76 | 77 | data_set = Data( 78 | root_path=args.root_path, 79 | data_path=args.data_path, 80 | flag=flag, 81 | size=[args.seq_len, args.label_len, args.pred_len], 82 | features=args.features, 83 | target=args.target 84 | ) 85 | print(flag, len(data_set)) 86 | data_loader = DataLoader( 87 | data_set, 88 | batch_size=batch_size, 89 | shuffle=shuffle_flag, 90 | num_workers=args.num_workers, 91 | drop_last=drop_last) 92 | 93 | return data_set, data_loader 94 | 95 | def _select_optimizer(self): 96 | model_optim = optim.Adam(self.model.parameters(), lr=self.args.learning_rate) 97 | return model_optim 98 | 99 | def _select_criterion(self): 100 | criterion = nn.MSELoss() 101 | return criterion 102 | 103 | def vali(self, vali_data, vali_loader, criterion): 104 | self.model.eval() 105 | total_loss = [] 106 | for i, (batch_x,batch_y,batch_x_mark,batch_y_mark) in enumerate(vali_loader): 107 | batch_x = batch_x.double().to(self.device) 108 | batch_y = batch_y.double() 109 | 110 | batch_x_mark = batch_x_mark.double().to(self.device) 111 | batch_y_mark = batch_y_mark.double().to(self.device) 112 | 113 | # decoder input 114 | dec_inp = torch.zeros_like(batch_y[:,-self.args.pred_len:,:]).double() 115 | dec_inp = torch.cat([batch_y[:,:self.args.label_len,:], dec_inp], dim=1).double().to(self.device) 116 | # encoder - decoder 117 | outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark) 118 | batch_y = batch_y[:,-self.args.pred_len:,:].to(self.device) 119 | 120 | pred = outputs.detach().cpu() 121 | true = batch_y.detach().cpu() 122 | 123 | loss = criterion(pred, true) 124 | 125 | total_loss.append(loss) 126 | total_loss = np.average(total_loss) 127 | self.model.train() 128 | return total_loss 129 | 130 | def train(self, setting): 131 | train_data, train_loader = self._get_data(flag = 'train') 132 | vali_data, vali_loader = self._get_data(flag = 'val') 133 | test_data, test_loader = self._get_data(flag = 'test') 134 | 135 | path = './checkpoints/'+setting 136 | if not os.path.exists(path): 137 | os.makedirs(path) 138 | 139 | time_now = time.time() 140 | 141 | train_steps = len(train_loader) 142 | early_stopping = EarlyStopping(patience=self.args.patience, verbose=True) 143 | 144 | model_optim = self._select_optimizer() 145 | criterion = self._select_criterion() 146 | 147 | for epoch in range(self.args.train_epochs): 148 | iter_count = 0 149 | train_loss = [] 150 | 151 | self.model.train() 152 | for i, (batch_x,batch_y,batch_x_mark,batch_y_mark) in enumerate(train_loader): 153 | iter_count += 1 154 | 155 | model_optim.zero_grad() 156 | 157 | batch_x = batch_x.double().to(self.device) 158 | batch_y = batch_y.double() 159 | 160 | batch_x_mark = batch_x_mark.double().to(self.device) 161 | batch_y_mark = batch_y_mark.double().to(self.device) 162 | 163 | # decoder input 164 | dec_inp = torch.zeros_like(batch_y[:,-self.args.pred_len:,:]).double() 165 | dec_inp = torch.cat([batch_y[:,:self.args.label_len,:], dec_inp], dim=1).double().to(self.device) 166 | # encoder - decoder 167 | outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark) 168 | 169 | batch_y = batch_y[:,-self.args.pred_len:,:].to(self.device) 170 | loss = criterion(outputs, batch_y) 171 | train_loss.append(loss.item()) 172 | 173 | if (i+1) % 100==0: 174 | print("\titers: {0}, epoch: {1} | loss: {2:.7f}".format(i + 1, epoch + 1, loss.item())) 175 | speed = (time.time()-time_now)/iter_count 176 | left_time = speed*((self.args.train_epochs - epoch)*train_steps - i) 177 | print('\tspeed: {:.4f}s/iter; left time: {:.4f}s'.format(speed, left_time)) 178 | iter_count = 0 179 | time_now = time.time() 180 | 181 | loss.backward() 182 | model_optim.step() 183 | 184 | train_loss = np.average(train_loss) 185 | vali_loss = self.vali(vali_data, vali_loader, criterion) 186 | test_loss = self.vali(test_data, test_loader, criterion) 187 | 188 | print("Epoch: {0}, Steps: {1} | Train Loss: {2:.7f} Vali Loss: {3:.7f} Test Loss: {4:.7f}".format( 189 | epoch + 1, train_steps, train_loss, vali_loss, test_loss)) 190 | early_stopping(vali_loss, self.model, path) 191 | if early_stopping.early_stop: 192 | print("Early stopping") 193 | break 194 | 195 | adjust_learning_rate(model_optim, epoch+1, self.args) 196 | 197 | best_model_path = path+'/'+'checkpoint.pth' 198 | self.model.load_state_dict(torch.load(best_model_path)) 199 | 200 | return self.model 201 | 202 | def test(self, setting): 203 | test_data, test_loader = self._get_data(flag='test') 204 | 205 | self.model.eval() 206 | 207 | preds = [] 208 | trues = [] 209 | 210 | for i, (batch_x,batch_y,batch_x_mark,batch_y_mark) in enumerate(test_loader): 211 | batch_x = batch_x.double().to(self.device) 212 | batch_y = batch_y.double() 213 | batch_x_mark = batch_x_mark.double().to(self.device) 214 | batch_y_mark = batch_y_mark.double().to(self.device) 215 | 216 | # decoder input 217 | dec_inp = torch.zeros_like(batch_y[:,-self.args.pred_len:,:]).double() 218 | dec_inp = torch.cat([batch_y[:,:self.args.label_len,:], dec_inp], dim=1).double().to(self.device) 219 | # encoder - decoder 220 | outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark) 221 | batch_y = batch_y[:,-self.args.pred_len:,:].to(self.device) 222 | 223 | pred = outputs.detach().cpu().numpy()#.squeeze() 224 | true = batch_y.detach().cpu().numpy()#.squeeze() 225 | 226 | preds.append(pred) 227 | trues.append(true) 228 | 229 | preds = np.array(preds) 230 | trues = np.array(trues) 231 | print('test shape:', preds.shape, trues.shape) 232 | preds = preds.reshape(-1, preds.shape[-2], preds.shape[-1]) 233 | trues = trues.reshape(-1, trues.shape[-2], trues.shape[-1]) 234 | print('test shape:', preds.shape, trues.shape) 235 | 236 | # result save 237 | folder_path = './results/' + setting +'/' 238 | if not os.path.exists(folder_path): 239 | os.makedirs(folder_path) 240 | 241 | mae, mse, rmse, mape, mspe = metric(preds, trues) 242 | print('mse:{}, mae:{}'.format(mse, mae)) 243 | 244 | np.save(folder_path+'metrics.npy', np.array([mae, mse, rmse, mape, mspe])) 245 | np.save(folder_path+'pred.npy', preds) 246 | np.save(folder_path+'true.npy', trues) 247 | 248 | return -------------------------------------------------------------------------------- /exp/exp_informer_dad.py: -------------------------------------------------------------------------------- 1 | from data.data_loader_dad import ( 2 | NASA_Anomaly, 3 | WADI 4 | ) 5 | from exp.exp_basic import Exp_Basic 6 | from models.model import Informer 7 | 8 | from utils.tools import EarlyStopping, adjust_learning_rate 9 | from utils.metrics import metric 10 | from sklearn.metrics import classification_report 11 | 12 | import numpy as np 13 | 14 | import torch 15 | import torch.nn as nn 16 | from torch import optim 17 | from torch.utils.data import DataLoader 18 | 19 | import os 20 | import time 21 | 22 | import warnings 23 | warnings.filterwarnings('ignore') 24 | 25 | class Exp_Informer_DAD(Exp_Basic): 26 | def __init__(self, args): 27 | super(Exp_Informer_DAD, self).__init__(args) 28 | 29 | def _build_model(self): 30 | model_dict = { 31 | 'informer':Informer, 32 | } 33 | if self.args.model=='informer': 34 | model = model_dict[self.args.model]( 35 | self.args.enc_in, 36 | self.args.dec_in, 37 | self.args.c_out, 38 | self.args.seq_len, 39 | self.args.label_len, 40 | self.args.pred_len, 41 | self.args.factor, 42 | self.args.d_model, 43 | self.args.n_heads, 44 | self.args.e_layers, 45 | self.args.d_layers, 46 | self.args.d_ff, 47 | self.args.dropout, 48 | self.args.attn, 49 | self.args.embed, 50 | self.args.data[:-1], 51 | self.args.activation, 52 | self.device 53 | ) 54 | 55 | return model.double() 56 | 57 | def _get_data(self, flag): 58 | args = self.args 59 | 60 | data_dict = { 61 | 'SMAP':NASA_Anomaly, 62 | 'MSL':NASA_Anomaly, 63 | 'WADI':WADI, 64 | } 65 | Data = data_dict[self.args.data] 66 | 67 | if flag == 'test': 68 | shuffle_flag = False; drop_last = True; batch_size = args.batch_size 69 | else: 70 | shuffle_flag = True; drop_last = True; batch_size = args.batch_size 71 | 72 | data_set = Data( 73 | root_path=args.root_path, 74 | data_path=args.data_path, 75 | flag=flag, 76 | size=[args.seq_len, args.label_len, args.pred_len], 77 | features=args.features, 78 | target=args.target 79 | ) 80 | print(flag, len(data_set)) 81 | data_loader = DataLoader( 82 | data_set, 83 | batch_size=batch_size, 84 | shuffle=shuffle_flag, 85 | num_workers=args.num_workers, 86 | drop_last=drop_last) 87 | 88 | return data_set, data_loader 89 | 90 | def _select_optimizer(self): 91 | model_optim = optim.Adam(self.model.parameters(), lr=self.args.learning_rate) 92 | return model_optim 93 | 94 | def _select_criterion(self): 95 | criterion = nn.MSELoss() 96 | return criterion 97 | 98 | def vali(self, vali_data, vali_loader, criterion): 99 | self.model.eval() 100 | total_loss = [] 101 | 102 | for i, (batch_x,batch_y,batch_x_mark,batch_y_mark,batch_label) in enumerate(vali_loader): 103 | batch_x = batch_x.double().to(self.device) 104 | batch_y = batch_y.double() 105 | 106 | batch_x_mark = batch_x_mark.double().to(self.device) 107 | batch_y_mark = batch_y_mark.double().to(self.device) 108 | 109 | # decoder input 110 | dec_inp = torch.zeros_like(batch_y[:,-self.args.pred_len:,:]).double() 111 | dec_inp = torch.cat([batch_y[:,:self.args.label_len,:], dec_inp], dim=1).double().to(self.device) 112 | # encoder - decoder 113 | outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark) 114 | batch_y = batch_y[:,-self.args.pred_len:,:].to(self.device) 115 | 116 | pred = outputs.detach().cpu() 117 | true = batch_y.detach().cpu() 118 | 119 | loss = criterion(pred, true) 120 | 121 | total_loss.append(loss) 122 | 123 | total_loss = np.average(total_loss) 124 | self.model.train() 125 | return total_loss 126 | 127 | def train(self, setting): 128 | train_data, train_loader = self._get_data(flag = 'train') 129 | vali_data, vali_loader = self._get_data(flag = 'val') 130 | test_data, test_loader = self._get_data(flag = 'test') 131 | 132 | path = './checkpoints/'+setting 133 | if not os.path.exists(path): 134 | os.makedirs(path) 135 | 136 | time_now = time.time() 137 | 138 | train_steps = len(train_loader) 139 | early_stopping = EarlyStopping(patience=self.args.patience, verbose=True) 140 | 141 | model_optim = self._select_optimizer() 142 | criterion = self._select_criterion() 143 | 144 | for epoch in range(self.args.train_epochs): 145 | iter_count = 0 146 | train_loss = [] 147 | 148 | self.model.train() 149 | for i, (batch_x,batch_y,batch_x_mark,batch_y_mark) in enumerate(train_loader): 150 | iter_count += 1 151 | 152 | model_optim.zero_grad() 153 | 154 | batch_x = batch_x.double().to(self.device) 155 | batch_y = batch_y.double() 156 | 157 | batch_x_mark = batch_x_mark.double().to(self.device) 158 | batch_y_mark = batch_y_mark.double().to(self.device) 159 | 160 | # decoder input 161 | dec_inp = torch.zeros_like(batch_y[:,-self.args.pred_len:,:]).double() 162 | dec_inp = torch.cat([batch_y[:,:self.args.label_len,:], dec_inp], dim=1).double().to(self.device) 163 | # encoder - decoder 164 | outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark) 165 | 166 | batch_y = batch_y[:,-self.args.pred_len:,:].to(self.device) 167 | loss = criterion(outputs, batch_y) 168 | train_loss.append(loss.item()) 169 | 170 | if (i+1) % 100==0: 171 | print("\titers: {0}, epoch: {1} | loss: {2:.7f}".format(i + 1, epoch + 1, loss.item())) 172 | speed = (time.time()-time_now)/iter_count 173 | left_time = speed*((self.args.train_epochs - epoch)*train_steps - i) 174 | print('\tspeed: {:.4f}s/iter; left time: {:.4f}s'.format(speed, left_time)) 175 | iter_count = 0 176 | time_now = time.time() 177 | 178 | loss.backward() 179 | model_optim.step() 180 | 181 | train_loss = np.average(train_loss) 182 | vali_loss = self.vali(vali_data, vali_loader, criterion) 183 | test_loss = self.vali(test_data, test_loader, criterion) 184 | 185 | print("Epoch: {0}, Steps: {1} | Train Loss: {2:.7f} Vali Loss: {3:.7f} Test Loss: {4:.7f}".format( 186 | epoch + 1, train_steps, train_loss, vali_loss, test_loss)) 187 | early_stopping(vali_loss, self.model, path) 188 | if early_stopping.early_stop: 189 | print("Early stopping") 190 | break 191 | 192 | adjust_learning_rate(model_optim, epoch+1, self.args) 193 | 194 | best_model_path = path+'/'+'checkpoint.pth' 195 | self.model.load_state_dict(torch.load(best_model_path)) 196 | 197 | return self.model 198 | 199 | def test(self, setting): 200 | test_data, test_loader = self._get_data(flag='test') 201 | 202 | self.model.eval() 203 | 204 | preds = [] 205 | trues = [] 206 | labels = [] 207 | 208 | with torch.no_grad(): 209 | for i, (batch_x,batch_y,batch_x_mark,batch_y_mark,batch_label) in enumerate(test_loader): 210 | batch_x = batch_x.double().to(self.device) 211 | batch_y = batch_y.double() 212 | batch_x_mark = batch_x_mark.double().to(self.device) 213 | batch_y_mark = batch_y_mark.double().to(self.device) 214 | 215 | # decoder input 216 | dec_inp = torch.zeros_like(batch_y[:,-self.args.pred_len:,:]).double() 217 | dec_inp = torch.cat([batch_y[:,:self.args.label_len,:], dec_inp], dim=1).double().to(self.device) 218 | # encoder - decoder 219 | outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark) 220 | batch_y = batch_y[:,-self.args.pred_len:,:].to(self.device) 221 | 222 | pred = outputs.detach().cpu().numpy()#.squeeze() 223 | true = batch_y.detach().cpu().numpy()#.squeeze() 224 | batch_label = batch_label.long().detach().numpy() 225 | 226 | preds.append(pred) 227 | trues.append(true) 228 | labels.append(batch_label) 229 | 230 | preds = np.array(preds) 231 | trues = np.array(trues) 232 | labels = np.array(labels) 233 | print('test shape:', preds.shape, trues.shape) 234 | preds = preds.reshape(-1, preds.shape[-2], preds.shape[-1]) 235 | trues = trues.reshape(-1, trues.shape[-2], trues.shape[-1]) 236 | labels = labels.reshape(-1, labels.shape[-1]) 237 | print('test shape:', preds.shape, trues.shape) 238 | 239 | # result save 240 | folder_path = './results/' + setting +'/' 241 | if not os.path.exists(folder_path): 242 | os.makedirs(folder_path) 243 | 244 | mae, mse, rmse, mape, mspe = metric(preds, trues) 245 | print('mse:{}, mae:{}'.format(mse, mae)) 246 | 247 | np.save(folder_path+'metrics.npy', np.array([mae, mse, rmse, mape, mspe])) 248 | np.save(folder_path+'pred.npy', preds) 249 | np.save(folder_path+'true.npy', trues) 250 | np.save(folder_path+'label.npy', labels) 251 | 252 | return -------------------------------------------------------------------------------- /exp/exp_gta_dad.py: -------------------------------------------------------------------------------- 1 | from data.data_loader_dad import ( 2 | NASA_Anomaly, 3 | WADI, 4 | SWaT 5 | ) 6 | from exp.exp_basic import Exp_Basic 7 | from models.gta import GTA 8 | 9 | from utils.tools import EarlyStopping, adjust_learning_rate 10 | from utils.metrics import metric 11 | from sklearn.metrics import classification_report 12 | 13 | import numpy as np 14 | 15 | import torch 16 | import torch.nn as nn 17 | from torch import optim 18 | from torch.utils.data import DataLoader 19 | 20 | import os 21 | import time 22 | 23 | import warnings 24 | warnings.filterwarnings('ignore') 25 | 26 | class Exp_GTA_DAD(Exp_Basic): 27 | def __init__(self, args): 28 | super(Exp_GTA_DAD, self).__init__(args) 29 | 30 | def _build_model(self): 31 | model_dict = { 32 | 'gta':GTA, 33 | } 34 | if self.args.model=='gta': 35 | model = model_dict[self.args.model]( 36 | self.args.num_nodes, 37 | self.args.seq_len, 38 | self.args.label_len, 39 | self.args.pred_len, 40 | self.args.num_levels, 41 | self.args.factor, 42 | self.args.d_model, 43 | self.args.n_heads, 44 | self.args.e_layers, 45 | self.args.d_layers, 46 | self.args.d_ff, 47 | self.args.dropout, 48 | self.args.attn, 49 | self.args.embed, 50 | self.args.data, 51 | self.args.activation, 52 | self.device 53 | ) 54 | 55 | return model.double() 56 | 57 | def _get_data(self, flag): 58 | args = self.args 59 | 60 | data_dict = { 61 | 'SMAP':NASA_Anomaly, 62 | 'MSL':NASA_Anomaly, 63 | 'WADI':WADI, 64 | 'SWaT':SWaT, 65 | } 66 | Data = data_dict[self.args.data] 67 | 68 | if flag == 'test': 69 | shuffle_flag = False; drop_last = True; batch_size = args.batch_size 70 | else: 71 | shuffle_flag = True; drop_last = True; batch_size = args.batch_size 72 | 73 | data_set = Data( 74 | root_path=args.root_path, 75 | data_path=args.data_path, 76 | flag=flag, 77 | size=[args.seq_len, args.label_len, args.pred_len], 78 | features=args.features, 79 | target=args.target 80 | ) 81 | print(flag, len(data_set)) 82 | data_loader = DataLoader( 83 | data_set, 84 | batch_size=batch_size, 85 | shuffle=shuffle_flag, 86 | num_workers=args.num_workers, 87 | drop_last=drop_last) 88 | 89 | return data_set, data_loader 90 | 91 | def _select_optimizer(self): 92 | model_optim = optim.Adam(self.model.parameters(), lr=self.args.learning_rate) 93 | return model_optim 94 | 95 | def _select_criterion(self): 96 | criterion = nn.MSELoss() 97 | return criterion 98 | 99 | def vali(self, vali_data, vali_loader, criterion): 100 | self.model.eval() 101 | total_loss = [] 102 | 103 | for i, (batch_x,batch_y,batch_x_mark,batch_y_mark,batch_label) in enumerate(vali_loader): 104 | batch_x = batch_x.double().to(self.device) 105 | batch_y = batch_y.double().to(self.device) 106 | 107 | batch_x_mark = batch_x_mark.double().to(self.device) 108 | batch_y_mark = batch_y_mark.double().to(self.device) 109 | 110 | # decoder input 111 | # dec_inp = torch.zeros_like(batch_y[:,-self.args.pred_len:,:]).double() 112 | # dec_inp = torch.cat([batch_y[:,:self.args.label_len,:], dec_inp], dim=1).double().to(self.device) 113 | # encoder - decoder 114 | # outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark) 115 | outputs = self.model(batch_x, batch_y, batch_x_mark, batch_y_mark) 116 | batch_y = batch_y[:,-self.args.pred_len:,:].to(self.device) 117 | 118 | pred = outputs.detach().cpu() 119 | true = batch_y.detach().cpu() 120 | 121 | loss = criterion(pred, true) 122 | 123 | total_loss.append(loss) 124 | 125 | total_loss = np.average(total_loss) 126 | self.model.train() 127 | return total_loss 128 | 129 | def train(self, setting): 130 | train_data, train_loader = self._get_data(flag = 'train') 131 | vali_data, vali_loader = self._get_data(flag = 'val') 132 | test_data, test_loader = self._get_data(flag = 'test') 133 | 134 | path = './checkpoints/'+setting 135 | if not os.path.exists(path): 136 | os.makedirs(path) 137 | 138 | time_now = time.time() 139 | 140 | train_steps = len(train_loader) 141 | early_stopping = EarlyStopping(patience=self.args.patience, verbose=True) 142 | 143 | model_optim = self._select_optimizer() 144 | criterion = self._select_criterion() 145 | 146 | for epoch in range(self.args.train_epochs): 147 | iter_count = 0 148 | train_loss = [] 149 | 150 | self.model.train() 151 | for i, (batch_x,batch_y,batch_x_mark,batch_y_mark) in enumerate(train_loader): 152 | iter_count += 1 153 | 154 | model_optim.zero_grad() 155 | 156 | batch_x = batch_x.double().to(self.device) 157 | batch_y = batch_y.double().to(self.device) 158 | 159 | batch_x_mark = batch_x_mark.double().to(self.device) 160 | batch_y_mark = batch_y_mark.double().to(self.device) 161 | 162 | # decoder input 163 | # dec_inp = torch.zeros_like(batch_y[:,-self.args.pred_len:,:]).double() 164 | # dec_inp = torch.cat([batch_y[:,:self.args.label_len,:], dec_inp], dim=1).double().to(self.device) 165 | # encoder - decoder 166 | # outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark) 167 | outputs = self.model(batch_x, batch_y, batch_x_mark, batch_y_mark) 168 | batch_y = batch_y[:,-self.args.pred_len:,:].to(self.device) 169 | 170 | loss = criterion(outputs, batch_y) + \ 171 | torch.sum(torch.abs(self.model.gt_embedding.gc_module.logits[:, 0])) 172 | train_loss.append(loss.item()) 173 | 174 | if (i+1) % 100==0: 175 | print("\titers: {0}, epoch: {1} | loss: {2:.7f}".format(i + 1, epoch + 1, loss.item())) 176 | speed = (time.time()-time_now)/iter_count 177 | left_time = speed*((self.args.train_epochs - epoch)*train_steps - i) 178 | print('\tspeed: {:.4f}s/iter; left time: {:.4f}s'.format(speed, left_time)) 179 | iter_count = 0 180 | time_now = time.time() 181 | 182 | loss.backward() 183 | model_optim.step() 184 | 185 | train_loss = np.average(train_loss) 186 | vali_loss = self.vali(vali_data, vali_loader, criterion) 187 | test_loss = self.vali(test_data, test_loader, criterion) 188 | 189 | print("Epoch: {0}, Steps: {1} | Train Loss: {2:.7f} Vali Loss: {3:.7f} Test Loss: {4:.7f}".format( 190 | epoch + 1, train_steps, train_loss, vali_loss, test_loss)) 191 | early_stopping(vali_loss, self.model, path) 192 | if early_stopping.early_stop: 193 | print("Early stopping") 194 | break 195 | 196 | adjust_learning_rate(model_optim, epoch+1, self.args) 197 | 198 | best_model_path = path+'/'+'checkpoint.pth' 199 | self.model.load_state_dict(torch.load(best_model_path)) 200 | 201 | return self.model 202 | 203 | def test(self, setting): 204 | test_data, test_loader = self._get_data(flag='test') 205 | 206 | self.model.eval() 207 | 208 | preds = [] 209 | trues = [] 210 | labels = [] 211 | 212 | with torch.no_grad(): 213 | for i, (batch_x,batch_y,batch_x_mark,batch_y_mark,batch_label) in enumerate(test_loader): 214 | batch_x = batch_x.double().to(self.device) 215 | batch_y = batch_y.double().to(self.device) 216 | batch_x_mark = batch_x_mark.double().to(self.device) 217 | batch_y_mark = batch_y_mark.double().to(self.device) 218 | 219 | # decoder input 220 | # dec_inp = torch.zeros_like(batch_y[:,-self.args.pred_len:,:]).double() 221 | # dec_inp = torch.cat([batch_y[:,:self.args.label_len,:], dec_inp], dim=1).double().to(self.device) 222 | # encoder - decoder 223 | # outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark) 224 | outputs = self.model(batch_x, batch_y, batch_x_mark, batch_y_mark) 225 | batch_y = batch_y[:,-self.args.pred_len:,:].to(self.device) 226 | 227 | pred = outputs.detach().cpu().numpy()#.squeeze() 228 | true = batch_y.detach().cpu().numpy()#.squeeze() 229 | batch_label = batch_label.long().detach().numpy() 230 | 231 | preds.append(pred) 232 | trues.append(true) 233 | labels.append(batch_label) 234 | 235 | preds = np.array(preds) 236 | trues = np.array(trues) 237 | labels = np.array(labels) 238 | print('test shape:', preds.shape, trues.shape) 239 | preds = preds.reshape(-1, preds.shape[-2], preds.shape[-1]) 240 | trues = trues.reshape(-1, trues.shape[-2], trues.shape[-1]) 241 | labels = labels.reshape(-1, labels.shape[-1]) 242 | print('test shape:', preds.shape, trues.shape) 243 | 244 | # result save 245 | folder_path = './results/' + setting +'/' 246 | if not os.path.exists(folder_path): 247 | os.makedirs(folder_path) 248 | 249 | mae, mse, rmse, mape, mspe = metric(preds, trues) 250 | print('mse:{}, mae:{}'.format(mse, mae)) 251 | 252 | np.save(folder_path+'metrics.npy', np.array([mae, mse, rmse, mape, mspe])) 253 | np.save(folder_path+'pred.npy', preds) 254 | np.save(folder_path+'true.npy', trues) 255 | np.save(folder_path+'label.npy', labels) 256 | 257 | return -------------------------------------------------------------------------------- /models/vrae.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import torch 4 | import torch.nn as nn 5 | from torch.autograd import Variable 6 | 7 | 8 | class Encoder(nn.Module): 9 | """ 10 | Encoder network containing enrolled LSTM/GRU 11 | :param number_of_features: number of input features 12 | :param hidden_size: hidden size of the RNN 13 | :param hidden_layer_depth: number of layers in RNN 14 | :param latent_length: latent vector length 15 | :param dropout: percentage of nodes to dropout 16 | :param block: LSTM/GRU block 17 | """ 18 | def __init__(self, number_of_features, hidden_size, hidden_layer_depth, latent_length, dropout, block='LSTM'): 19 | 20 | super(Encoder, self).__init__() 21 | 22 | self.number_of_features = number_of_features 23 | self.hidden_size = hidden_size 24 | self.hidden_layer_depth = hidden_layer_depth 25 | self.latent_length = latent_length 26 | 27 | if block == 'LSTM': 28 | self.model = nn.LSTM(self.number_of_features, self.hidden_size, self.hidden_layer_depth, dropout=dropout, batch_first=True) 29 | elif block == 'GRU': 30 | self.model = nn.GRU(self.number_of_features, self.hidden_size, self.hidden_layer_depth, dropout=dropout, batch_first=True) 31 | else: 32 | raise NotImplementedError 33 | 34 | def forward(self, x): 35 | """Forward propagation of encoder. Given input, outputs the last hidden state of encoder 36 | :param x: input to the encoder, of shape (sequence_length, batch_size, number_of_features) 37 | :return: last hidden state of encoder, of shape (batch_size, hidden_size) 38 | """ 39 | 40 | _, (h_end, c_end) = self.model(x) 41 | 42 | h_end = h_end[-1, :, :] 43 | return h_end 44 | 45 | 46 | class Lambda(nn.Module): 47 | """Lambda module converts output of encoder to latent vector 48 | :param hidden_size: hidden size of the encoder 49 | :param latent_length: latent vector length 50 | """ 51 | def __init__(self, hidden_size, latent_length): 52 | super(Lambda, self).__init__() 53 | 54 | self.hidden_size = hidden_size 55 | self.latent_length = latent_length 56 | 57 | self.hidden_to_mean = nn.Linear(self.hidden_size, self.latent_length) 58 | self.hidden_to_logvar = nn.Linear(self.hidden_size, self.latent_length) 59 | 60 | nn.init.xavier_uniform_(self.hidden_to_mean.weight) 61 | nn.init.xavier_uniform_(self.hidden_to_logvar.weight) 62 | 63 | def forward(self, cell_output): 64 | """Given last hidden state of encoder, passes through a linear layer, and finds the mean and variance 65 | :param cell_output: last hidden state of encoder 66 | :return: latent vector 67 | """ 68 | 69 | self.latent_mean = self.hidden_to_mean(cell_output) 70 | self.latent_logvar = self.hidden_to_logvar(cell_output) 71 | 72 | if self.training: 73 | std = torch.exp(0.5 * self.latent_logvar) 74 | eps = torch.randn_like(std) 75 | return eps.mul(std).add_(self.latent_mean) 76 | else: 77 | return self.latent_mean 78 | 79 | class Decoder(nn.Module): 80 | """Converts latent vector into output 81 | :param sequence_length: length of the input sequence 82 | :param batch_size: batch size of the input sequence 83 | :param hidden_size: hidden size of the RNN 84 | :param hidden_layer_depth: number of layers in RNN 85 | :param latent_length: latent vector length 86 | :param output_size: 2, one representing the mean, other log std dev of the output 87 | :param block: GRU/LSTM - use the same which you've used in the encoder 88 | :param dtype: Depending on cuda enabled/disabled, create the tensor 89 | """ 90 | def __init__(self, num_of_features, sequence_length, batch_size, hidden_size, hidden_layer_depth, latent_length, output_size, device, block='LSTM'): 91 | 92 | super(Decoder, self).__init__() 93 | 94 | self.hidden_size = hidden_size 95 | self.batch_size = batch_size 96 | self.num_of_features = num_of_features 97 | self.sequence_length = sequence_length 98 | self.hidden_layer_depth = hidden_layer_depth 99 | self.latent_length = latent_length 100 | self.output_size = output_size 101 | self.device = device 102 | 103 | if block == 'LSTM': 104 | self.model = nn.LSTM(self.num_of_features, self.hidden_size, self.hidden_layer_depth, batch_first=True) 105 | elif block == 'GRU': 106 | self.model = nn.GRU(self.num_of_features, self.hidden_size, self.hidden_layer_depth, batch_first=True) 107 | else: 108 | raise NotImplementedError 109 | 110 | self.latent_to_hidden = nn.Linear(self.latent_length, self.hidden_size * 2) 111 | self.hidden_to_output = nn.Linear(self.hidden_size, self.output_size) 112 | 113 | self.decoder_inputs = torch.randn(self.batch_size, self.sequence_length, self.num_of_features, requires_grad=True).double().to(self.device) 114 | # self.c_0 = torch.zeros(self.hidden_layer_depth, self.batch_size, self.hidden_size, requires_grad=True).double().to(self.device) 115 | 116 | nn.init.xavier_uniform_(self.latent_to_hidden.weight) 117 | nn.init.xavier_uniform_(self.hidden_to_output.weight) 118 | 119 | def forward(self, latent): 120 | """Converts latent to hidden to output 121 | :param latent: latent vector 122 | :return: outputs consisting of mean and std dev of vector 123 | """ 124 | h_states = self.latent_to_hidden(latent) 125 | h_state, c_state = h_states[:, :self.hidden_size].contiguous(), h_states[:, self.hidden_size:].contiguous() 126 | 127 | if isinstance(self.model, nn.LSTM): 128 | h_0 = torch.stack([h_state for _ in range(self.hidden_layer_depth)]) 129 | c_0 = torch.stack([c_state for _ in range(self.hidden_layer_depth)]) 130 | decoder_output, _ = self.model(self.decoder_inputs, (h_0, c_0)) 131 | elif isinstance(self.model, nn.GRU): 132 | h_0 = torch.stack([h_state for _ in range(self.hidden_layer_depth)]) 133 | decoder_output, _ = self.model(self.decoder_inputs, h_0) 134 | else: 135 | raise NotImplementedError 136 | 137 | out = self.hidden_to_output(decoder_output) 138 | return out 139 | 140 | def _assert_no_grad(tensor): 141 | assert not tensor.requires_grad, \ 142 | "nn criterions don't compute the gradient w.r.t. targets - please " \ 143 | "mark these tensors as not requiring gradients" 144 | 145 | class VRAE(nn.Module): 146 | """Variational recurrent auto-encoder. This module is used for dimensionality reduction of timeseries 147 | :param sequence_length: length of the input sequence 148 | :param number_of_features: number of input features 149 | :param hidden_size: hidden size of the RNN 150 | :param hidden_layer_depth: number of layers in RNN 151 | :param latent_length: latent vector length 152 | :param batch_size: number of timeseries in a single batch 153 | :param learning_rate: the learning rate of the module 154 | :param block: GRU/LSTM to be used as a basic building block 155 | :param n_epochs: Number of iterations/epochs 156 | :param dropout_rate: The probability of a node being dropped-out 157 | :param optimizer: ADAM/ SGD optimizer to reduce the loss function 158 | :param loss: SmoothL1Loss / MSELoss / ReconLoss / any custom loss which inherits from `_Loss` class 159 | :param boolean cuda: to be run on GPU or not 160 | :param print_every: The number of iterations after which loss should be printed 161 | :param boolean clip: Gradient clipping to overcome explosion 162 | :param max_grad_norm: The grad-norm to be clipped 163 | :param dload: Download directory where models are to be dumped 164 | """ 165 | def __init__(self, sequence_length, number_of_features, batch_size, enc_hidden_size=90, dec_hidden_size=90, 166 | enc_hidden_layer_depth=2, dec_hidden_layer_depth=2, latent_length=20, 167 | block='LSTM', dropout_rate=0.2, device=torch.device('cuda:0')): 168 | 169 | super(VRAE, self).__init__() 170 | 171 | self.device = device 172 | 173 | self.encoder = Encoder(number_of_features=number_of_features, 174 | hidden_size=enc_hidden_size, 175 | hidden_layer_depth=enc_hidden_layer_depth, 176 | latent_length=latent_length, 177 | dropout=dropout_rate, 178 | block=block) 179 | 180 | self.lmbd = Lambda(hidden_size=enc_hidden_size, 181 | latent_length=latent_length) 182 | 183 | self.decoder = Decoder(num_of_features=number_of_features, 184 | sequence_length=sequence_length, 185 | batch_size=batch_size, 186 | hidden_size=dec_hidden_size, 187 | hidden_layer_depth=dec_hidden_layer_depth, 188 | latent_length=latent_length, 189 | output_size=number_of_features, 190 | block=block, 191 | device=self.device) 192 | 193 | def forward(self, x): 194 | """ 195 | Forward propagation which involves one pass from inputs to encoder to lambda to decoder 196 | :param x:input tensor 197 | :return: the decoded output, latent vector 198 | """ 199 | cell_output = self.encoder(x) 200 | latent = self.lmbd(cell_output) 201 | x_decoded = self.decoder(latent) 202 | 203 | return x_decoded, latent 204 | 205 | def _rec(self, x_decoded, x, loss_fn): 206 | """ 207 | Compute the loss given output x decoded, input x and the specified loss function 208 | :param x_decoded: output of the decoder 209 | :param x: input to the encoder 210 | :param loss_fn: loss function specified 211 | :return: joint loss, reconstruction loss and kl-divergence loss 212 | """ 213 | latent_mean, latent_logvar = self.lmbd.latent_mean, self.lmbd.latent_logvar 214 | 215 | kl_loss = -0.5 * torch.mean(1 + latent_logvar - latent_mean.pow(2) - latent_logvar.exp()) 216 | recon_loss = loss_fn(x_decoded.transpose(0, 1), x.transpose(0, 1)) 217 | 218 | return kl_loss + recon_loss, recon_loss, kl_loss 219 | 220 | def compute_loss(self, X, loss_fn): 221 | """ 222 | Given input tensor, forward propagate, compute the loss, and backward propagate. 223 | Represents the lifecycle of a single iteration 224 | :param X: Input tensor 225 | :return: total loss, reconstruction loss, kl-divergence loss and original input 226 | """ 227 | x = Variable(X[:, :, :].to(self.device), requires_grad=True) 228 | 229 | x_decoded, _ = self(x) 230 | loss, recon_loss, kl_loss = self._rec(x_decoded, x.detach(), loss_fn) 231 | 232 | return loss, recon_loss, kl_loss, x 233 | 234 | 235 | if __name__ == '__main__': 236 | device = torch.device('cuda:0') 237 | model = VRAE(sequence_length=120, number_of_features=22, batch_size=32).double().to(device) 238 | x = torch.randn(32, 120, 22).double().to(device) 239 | # outputs, _ = model(x) 240 | loss_fn = torch.nn.MSELoss() 241 | loss, recon_loss, kl_loss, _ = model.compute_loss(x, loss_fn) 242 | print(loss) 243 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /data/data_loader.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import numpy as np 4 | import pandas as pd 5 | 6 | import torch 7 | from torch.utils.data import Dataset, DataLoader 8 | from sklearn.preprocessing import StandardScaler, MinMaxScaler 9 | 10 | import warnings 11 | warnings.filterwarnings('ignore') 12 | 13 | class Dataset_ETT_hour(Dataset): 14 | def __init__(self, root_path, flag='train', size=None, 15 | features='S', data_path='ETTh1.csv', 16 | target='OT', scale=True): 17 | # size [seq_len, label_len pred_len] 18 | # info 19 | if size == None: 20 | self.seq_len = 24*4*4 21 | self.label_len = 24*4 22 | self.pred_len = 24*4 23 | else: 24 | self.seq_len = size[0] 25 | self.label_len = size[1] 26 | self.pred_len = size[2] 27 | # init 28 | assert flag in ['train', 'test', 'val'] 29 | type_map = {'train':0, 'val':1, 'test':2} 30 | self.set_type = type_map[flag] 31 | 32 | self.features = features 33 | self.target = target 34 | self.scale = scale 35 | 36 | self.root_path = root_path 37 | self.data_path = data_path 38 | self.__read_data__() 39 | 40 | def __read_data__(self): 41 | scaler = StandardScaler() 42 | df_raw = pd.read_csv(os.path.join(self.root_path, 43 | self.data_path)) 44 | 45 | border1s = [0, 12*30*24 - self.seq_len, 12*30*24+4*30*24 - self.seq_len] 46 | border2s = [12*30*24, 12*30*24+4*30*24, 12*30*24+8*30*24] 47 | border1 = border1s[self.set_type] 48 | border2 = border2s[self.set_type] 49 | 50 | if self.features=='M': 51 | cols_data = df_raw.columns[1:] 52 | df_data = df_raw[cols_data] 53 | elif self.features=='S': 54 | df_data = df_raw[[self.target]] 55 | 56 | if self.scale: 57 | data = scaler.fit_transform(df_data.values) 58 | else: 59 | data = df_data.values 60 | 61 | df_stamp = df_raw[['date']][border1:border2] 62 | df_stamp['date'] = pd.to_datetime(df_stamp.date) 63 | df_stamp['month'] = df_stamp.date.apply(lambda row:row.month,1) 64 | df_stamp['day'] = df_stamp.date.apply(lambda row:row.day,1) 65 | df_stamp['weekday'] = df_stamp.date.apply(lambda row:row.weekday(),1) 66 | df_stamp['hour'] = df_stamp.date.apply(lambda row:row.hour,1) 67 | data_stamp = df_stamp.drop(['date'],1).values 68 | 69 | self.data_x = data[border1:border2] 70 | self.data_y = data[border1:border2] 71 | self.data_stamp = data_stamp 72 | 73 | def __getitem__(self, index): 74 | s_begin = index 75 | s_end = s_begin + self.seq_len 76 | r_begin = s_end - self.label_len 77 | r_end = r_begin + self.label_len + self.pred_len 78 | 79 | seq_x = self.data_x[s_begin:s_end] 80 | seq_y = self.data_y[r_begin:r_end] 81 | seq_x_mark = self.data_stamp[s_begin:s_end] 82 | seq_y_mark = self.data_stamp[r_begin:r_end] 83 | 84 | return seq_x, seq_y, seq_x_mark, seq_y_mark 85 | 86 | def __len__(self): 87 | return len(self.data_x) - self.seq_len - self.pred_len + 1 88 | 89 | class Dataset_ETT_minute(Dataset): 90 | def __init__(self, root_path, flag='train', size=None, 91 | features='S', data_path='ETTm1.csv', 92 | target='OT', scale=True): 93 | # size [seq_len, label_len pred_len] 94 | # info 95 | if size == None: 96 | self.seq_len = 24*4*4 97 | self.label_len = 24*4 98 | self.pred_len = 24*4 99 | else: 100 | self.seq_len = size[0] 101 | self.label_len = size[1] 102 | self.pred_len = size[2] 103 | # init 104 | assert flag in ['train', 'test', 'val'] 105 | type_map = {'train':0, 'val':1, 'test':2} 106 | self.set_type = type_map[flag] 107 | 108 | self.features = features 109 | self.target = target 110 | self.scale = scale 111 | 112 | self.root_path = root_path 113 | self.data_path = data_path 114 | self.__read_data__() 115 | 116 | def __read_data__(self): 117 | scaler = StandardScaler() 118 | df_raw = pd.read_csv(os.path.join(self.root_path, 119 | self.data_path)) 120 | 121 | border1s = [0, 12*30*24*4 - self.seq_len, 12*30*24*4+4*30*24*4 - self.seq_len] 122 | border2s = [12*30*24*4, 12*30*24*4+4*30*24*4, 12*30*24*4+8*30*24*4] 123 | border1 = border1s[self.set_type] 124 | border2 = border2s[self.set_type] 125 | 126 | if self.features=='M': 127 | cols_data = df_raw.columns[1:] 128 | df_data = df_raw[cols_data] 129 | elif self.features=='S': 130 | df_data = df_raw[[self.target]] 131 | 132 | if self.scale: 133 | data = scaler.fit_transform(df_data.values) 134 | else: 135 | data = df_data.values 136 | 137 | df_stamp = df_raw[['date']][border1:border2] 138 | df_stamp['date'] = pd.to_datetime(df_stamp.date) 139 | df_stamp['month'] = df_stamp.date.apply(lambda row:row.month,1) 140 | df_stamp['day'] = df_stamp.date.apply(lambda row:row.day,1) 141 | df_stamp['weekday'] = df_stamp.date.apply(lambda row:row.weekday(),1) 142 | df_stamp['hour'] = df_stamp.date.apply(lambda row:row.hour,1) 143 | df_stamp['minute'] = df_stamp.date.apply(lambda row:row.minute,1) 144 | df_stamp['minute'] = df_stamp.minute.map(lambda x:x//15) 145 | data_stamp = df_stamp.drop(['date'],1).values 146 | 147 | self.data_x = data[border1:border2] 148 | self.data_y = data[border1:border2] 149 | self.data_stamp = data_stamp 150 | 151 | def __getitem__(self, index): 152 | s_begin = index 153 | s_end = s_begin + self.seq_len 154 | r_begin = s_end - self.label_len 155 | r_end = r_begin + self.label_len + self.pred_len 156 | 157 | seq_x = self.data_x[s_begin:s_end] 158 | seq_y = self.data_y[r_begin:r_end] 159 | seq_x_mark = self.data_stamp[s_begin:s_end] 160 | seq_y_mark = self.data_stamp[r_begin:r_end] 161 | 162 | return seq_x, seq_y, seq_x_mark, seq_y_mark 163 | 164 | def __len__(self): 165 | return len(self.data_x) - self.seq_len - self.pred_len + 1 166 | 167 | class SolarEnergy(Dataset): 168 | def __init__(self, root_path, flag='train', size=None, 169 | features='M', data_path='solar_energy.csv', 170 | target='0', scale=True): 171 | # size [seq_len, label_len pred_len] 172 | # info 173 | if size == None: 174 | self.seq_len = 12*4*6 175 | self.label_len = 12*6 176 | self.pred_len = 12*6 177 | else: 178 | self.seq_len = size[0] 179 | self.label_len = size[1] 180 | self.pred_len = size[2] 181 | # init 182 | assert flag in ['train', 'test', 'val'] 183 | type_map = {'train':0, 'val':1, 'test':2} 184 | self.set_type = type_map[flag] 185 | 186 | self.features = features 187 | self.target = target 188 | self.scale = scale 189 | 190 | self.root_path = root_path 191 | self.data_path = data_path 192 | self.__read_data__() 193 | 194 | def __read_data__(self): 195 | scaler = StandardScaler() 196 | df_raw = pd.read_csv(os.path.join(self.root_path, 197 | self.data_path)) 198 | 199 | border1s = [0, 6*30*24*6 - self.seq_len, 6*30*24*6+2*30*24*6 - self.seq_len] 200 | border2s = [6*30*24*6, 6*30*24*6+2*30*24*6, 8*30*24*6+4*30*24*6] 201 | border1 = border1s[self.set_type] 202 | border2 = border2s[self.set_type] 203 | 204 | if self.features=='M': 205 | cols_data = df_raw.columns[1:] 206 | df_data = df_raw[cols_data] 207 | elif self.features=='S': 208 | df_data = df_raw[[self.target]] 209 | 210 | if self.scale: 211 | data = scaler.fit_transform(df_data.values) 212 | else: 213 | data = df_data.values 214 | 215 | df_stamp = df_raw[['date']][border1:border2] 216 | df_stamp['date'] = pd.to_datetime(df_stamp.date) 217 | df_stamp['month'] = df_stamp.date.apply(lambda row:row.month,1) 218 | df_stamp['day'] = df_stamp.date.apply(lambda row:row.day,1) 219 | df_stamp['weekday'] = df_stamp.date.apply(lambda row:row.weekday(),1) 220 | df_stamp['hour'] = df_stamp.date.apply(lambda row:row.hour,1) 221 | df_stamp['minute'] = df_stamp.date.apply(lambda row:row.minute,1) 222 | df_stamp['minute'] = df_stamp.minute.map(lambda x:x//10) 223 | data_stamp = df_stamp.drop(['date'],1).values 224 | 225 | self.data_x = data[border1:border2] 226 | self.data_y = data[border1:border2] 227 | self.data_stamp = data_stamp 228 | 229 | def __getitem__(self, index): 230 | s_begin = index 231 | s_end = s_begin + self.seq_len 232 | r_begin = s_end - self.label_len 233 | r_end = r_begin + self.label_len + self.pred_len 234 | 235 | seq_x = self.data_x[s_begin:s_end] 236 | seq_y = self.data_y[r_begin:r_end] 237 | seq_x_mark = self.data_stamp[s_begin:s_end] 238 | seq_y_mark = self.data_stamp[r_begin:r_end] 239 | 240 | return seq_x, seq_y, seq_x_mark, seq_y_mark 241 | 242 | def __len__(self): 243 | return len(self.data_x) - self.seq_len- self.pred_len + 1 244 | 245 | 246 | class DogeCoin(Dataset): 247 | def __init__(self, root_path, flag='train', size=None, 248 | features='M', data_path='DOGEUSDT-5m-data.csv', 249 | target='0', scale=True): 250 | # size [seq_len, label_len pred_len] 251 | # info 252 | if size == None: 253 | self.seq_len = 12*4*6 254 | self.label_len = 12*6 255 | self.pred_len = 12*6 256 | else: 257 | self.seq_len = size[0] 258 | self.label_len = size[1] 259 | self.pred_len = size[2] 260 | # init 261 | assert flag in ['train', 'test', 'val'] 262 | type_map = {'train':0, 'val':1, 'test':2} 263 | self.set_type = type_map[flag] 264 | 265 | self.features = features 266 | self.target = target 267 | self.scale = scale 268 | 269 | self.root_path = root_path 270 | self.data_path = data_path 271 | self.__read_data__() 272 | 273 | def __read_data__(self): 274 | scaler = StandardScaler() 275 | df_raw = pd.read_csv(os.path.join(self.root_path, 276 | self.data_path)) 277 | 278 | border1s = [0, 6*30*24*6 - self.seq_len, 6*30*24*6+2*30*24*6 - self.seq_len] 279 | border2s = [6*30*24*6, 6*30*24*6+2*30*24*6, 8*30*24*6+4*30*24*6] 280 | border1 = border1s[self.set_type] 281 | border2 = border2s[self.set_type] 282 | 283 | if self.features=='M': 284 | cols_data = df_raw.columns[1:] 285 | df_data = df_raw[cols_data] 286 | elif self.features=='S': 287 | df_data = df_raw[[self.target]] 288 | 289 | if self.scale: 290 | data = scaler.fit_transform(df_data.values) 291 | else: 292 | data = df_data.values 293 | 294 | df_stamp = df_raw[['date']][border1:border2] 295 | df_stamp['date'] = pd.to_datetime(df_stamp.date) 296 | df_stamp['month'] = df_stamp.date.apply(lambda row:row.month,1) 297 | df_stamp['day'] = df_stamp.date.apply(lambda row:row.day,1) 298 | df_stamp['weekday'] = df_stamp.date.apply(lambda row:row.weekday(),1) 299 | df_stamp['hour'] = df_stamp.date.apply(lambda row:row.hour,1) 300 | df_stamp['minute'] = df_stamp.date.apply(lambda row:row.minute,1) 301 | df_stamp['minute'] = df_stamp.minute.map(lambda x:x//10) 302 | data_stamp = df_stamp.drop(['date'],1).values 303 | 304 | self.data_x = data[border1:border2] 305 | self.data_y = data[border1:border2] 306 | self.data_stamp = data_stamp 307 | 308 | def __getitem__(self, index): 309 | s_begin = index 310 | s_end = s_begin + self.seq_len 311 | r_begin = s_end - self.label_len 312 | r_end = r_begin + self.label_len + self.pred_len 313 | 314 | seq_x = self.data_x[s_begin:s_end] 315 | seq_y = self.data_y[r_begin:r_end] 316 | seq_x_mark = self.data_stamp[s_begin:s_end] 317 | seq_y_mark = self.data_stamp[r_begin:r_end] 318 | 319 | return seq_x, seq_y, seq_x_mark, seq_y_mark 320 | 321 | def __len__(self): 322 | return len(self.data_x) - self.seq_len- self.pred_len + 1 323 | 324 | 325 | if __name__ == '__main__': 326 | # dataset = Dataset_ETT_hour(root_path='./data/ETT-small', features='M') 327 | dataset = NASA_data(root_path='./data/SMAP', flag='train', size=(96, 48, 24)) 328 | data_loader = DataLoader( 329 | dataset, 330 | batch_size=32, 331 | shuffle=True, 332 | num_workers=2, 333 | drop_last=True) 334 | for (x, y, x_stamp, y_stamp) in data_loader: 335 | print(x.size(), y.size(), x_stamp.size(), y_stamp.size()) 336 | break 337 | # print(dataset[0][0].shape) 338 | # print(dataset[0][1].shape) 339 | # print(dataset[0][2]) 340 | # print(len(dataset)) -------------------------------------------------------------------------------- /data/data_loader_dad.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import numpy as np 4 | import pandas as pd 5 | 6 | import torch 7 | from torch.utils.data import Dataset, DataLoader 8 | from sklearn.preprocessing import StandardScaler, MinMaxScaler 9 | 10 | import warnings 11 | warnings.filterwarnings('ignore') 12 | 13 | 14 | class NASA_Anomaly(Dataset): 15 | def __init__(self, root_path, flag='train', size=None, 16 | features='M', data_path='SMAP', 17 | target=0, scale=True): 18 | # size [seq_len, label_len pred_len] 19 | # info 20 | if size == None: 21 | self.seq_len = 8*60 22 | self.label_len = 2*60 23 | self.pred_len = 2*60 24 | else: 25 | self.seq_len = size[0] 26 | self.label_len = size[1] 27 | self.pred_len = size[2] 28 | # init 29 | assert flag in ['train', 'test', 'val'] 30 | 31 | type_map = {'train':0, 'val':1, 'test':2} 32 | self.set_type = type_map[flag] 33 | self.flag = flag 34 | 35 | self.features = features 36 | self.target = target 37 | self.scale = scale 38 | 39 | self.root_path = root_path 40 | self.data_path = data_path 41 | self.__read_data__() 42 | 43 | def get_data_dim(self, dataset): 44 | if dataset == 'SMAP': 45 | return 25 46 | elif dataset == 'MSL': 47 | return 55 48 | elif str(dataset).startswith('machine'): 49 | return 38 50 | else: 51 | raise ValueError('unknown dataset '+str(dataset)) 52 | 53 | def __read_data__(self): 54 | """ 55 | get data from pkl files 56 | 57 | return shape: (([train_size, x_dim], [train_size] or None), ([test_size, x_dim], [test_size])) 58 | """ 59 | 60 | x_dim = self.get_data_dim(self.data_path) 61 | if self.flag == 'train': 62 | f = open(os.path.join(self.root_path, self.data_path, '{}_train.pkl'.format(self.data_path)), "rb") 63 | data = pickle.load(f).reshape((-1, x_dim)) 64 | f.close() 65 | elif self.flag in ['val', 'test']: 66 | try: 67 | f = open(os.path.join(self.root_path, self.data_path, '{}_test.pkl'.format(self.data_path)), "rb") 68 | data = pickle.load(f).reshape((-1, x_dim)) 69 | f.close() 70 | except (KeyError, FileNotFoundError): 71 | data = None 72 | try: 73 | f = open(os.path.join(self.root_path, self.data_path, '{}_test_label.pkl'.format(self.data_path)), "rb") 74 | label = pickle.load(f).reshape((-1)) 75 | f.close() 76 | except (KeyError, FileNotFoundError): 77 | label = None 78 | assert len(data) == len(label), "length of test data shoube the same as label" 79 | if self.scale: 80 | data = self.preprocess(data) 81 | 82 | df_stamp = pd.DataFrame(columns=['date']) 83 | date = pd.date_range(start='1/1/2015', periods=len(data), freq='4s') 84 | df_stamp['date'] = date 85 | df_stamp['month'] = df_stamp.date.apply(lambda row:row.month,1) 86 | df_stamp['day'] = df_stamp.date.apply(lambda row:row.day,1) 87 | df_stamp['weekday'] = df_stamp.date.apply(lambda row:row.weekday(),1) 88 | df_stamp['hour'] = df_stamp.date.apply(lambda row:row.hour,1) 89 | df_stamp['minute'] = df_stamp.date.apply(lambda row:row.minute,1) 90 | # df_stamp['minute'] = df_stamp.minute.map(lambda x:x//10) 91 | df_stamp['second'] = df_stamp.date.apply(lambda row:row.second,1) 92 | data_stamp = df_stamp.drop(['date'],1).values 93 | 94 | if self.flag == 'train': 95 | if self.features=='M': 96 | self.data_x = data 97 | self.data_y = data 98 | elif self.features=='S': 99 | df_data = data[:, [self.target]] 100 | self.data_x = df_data 101 | self.data_y = df_data 102 | else: 103 | border1s = [0, 0, 0] 104 | border2s = [None, len(data)//4, len(data)] 105 | border1 = border1s[self.set_type] 106 | border2 = border2s[self.set_type] 107 | if self.features=='M': 108 | self.data_x = data[border1:border2] 109 | self.data_y = data[border1:border2] 110 | self.label = label[border1:border2] 111 | elif self.features=='S': 112 | df_data = data[:, [self.target]] 113 | self.data_x = df_data[border1:border2] 114 | self.data_y = df_data[border1:border2] 115 | self.label = label[border1:border2] 116 | self.data_stamp = data_stamp 117 | 118 | def preprocess(self, df): 119 | """returns normalized and standardized data. 120 | """ 121 | 122 | df = np.asarray(df, dtype=np.float32) 123 | 124 | if len(df.shape) == 1: 125 | raise ValueError('Data must be a 2-D array') 126 | 127 | if np.any(sum(np.isnan(df)) != 0): 128 | print('Data contains null values. Will be replaced with 0') 129 | df = np.nan_to_num() 130 | 131 | # normalize data 132 | df = MinMaxScaler().fit_transform(df) 133 | print('Data normalized') 134 | 135 | return df 136 | 137 | def __getitem__(self, index): 138 | s_begin = index 139 | s_end = s_begin + self.seq_len 140 | r_begin = s_end - self.label_len 141 | r_end = s_end + self.pred_len 142 | 143 | seq_x = self.data_x[s_begin:s_end] 144 | seq_y = self.data_y[r_begin:r_end] 145 | seq_x_mark = self.data_stamp[s_begin:s_end] 146 | seq_y_mark = self.data_stamp[r_begin:r_end] 147 | 148 | if self.flag == 'train': 149 | return seq_x, seq_y, seq_x_mark, seq_y_mark 150 | else: 151 | seq_label = self.label[s_end:r_end] 152 | return seq_x, seq_y, seq_x_mark, seq_y_mark, seq_label 153 | 154 | def __len__(self): 155 | return len(self.data_x) - self.seq_len - self.pred_len + 1 156 | 157 | 158 | class WADI(Dataset): 159 | def __init__(self, root_path, flag='train', size=None, 160 | features='M', data_path='WADI_14days_downsampled.csv', 161 | target='1_AIT_001_PV', scale=True): 162 | # size [seq_len, label_len pred_len] 163 | # info 164 | if size == None: 165 | self.seq_len = 8*60 166 | self.label_len = 2*60 167 | self.pred_len = 2*60 168 | else: 169 | self.seq_len = size[0] 170 | self.label_len = size[1] 171 | self.pred_len = size[2] 172 | # init 173 | assert flag in ['train', 'test', 'val'] 174 | self.flag = flag 175 | type_map = {'train':0, 'val':1, 'test':2} 176 | self.set_type = type_map[flag] 177 | 178 | self.features = features 179 | self.target = target 180 | self.scale = scale 181 | 182 | self.root_path = root_path 183 | self.data_path = data_path 184 | self.__read_data__() 185 | 186 | def __read_data__(self): 187 | scaler = MinMaxScaler() 188 | if self.flag == 'train': 189 | df_raw = pd.read_csv(os.path.join(self.root_path, 190 | 'WADI_14days_downsampled.csv')) 191 | if self.features=='M': 192 | cols_data = df_raw.columns[1:] 193 | df_data = df_raw[cols_data] 194 | elif self.features=='S': 195 | df_data = df_raw[[self.target]] 196 | 197 | df_stamp = df_raw[['date']] 198 | 199 | if self.scale: 200 | data = scaler.fit_transform(df_data.values) 201 | else: 202 | data = df_data.values 203 | 204 | self.data_x = data 205 | self.data_y = data 206 | else: 207 | df_raw = pd.read_csv(os.path.join(self.root_path, 208 | 'WADI_attackdata_downsampled.csv')) 209 | 210 | border1s = [0, 0, 0] 211 | border2s = [None, len(df_raw)//4, len(df_raw)] 212 | border1 = border1s[self.set_type] 213 | border2 = border2s[self.set_type] 214 | 215 | df_stamp = df_raw[['date']][border1:border2] 216 | 217 | if self.features=='M': 218 | cols_data = df_raw.columns[1:-1] 219 | df_data = df_raw[cols_data] 220 | label = df_raw['label'].values 221 | elif self.features=='S': 222 | df_data = df_raw[[self.target]] 223 | label = df_raw['label'].values 224 | 225 | if self.scale: 226 | data = scaler.fit_transform(df_data.values) 227 | else: 228 | data = df_data.values 229 | 230 | self.data_x = data[border1:border2] 231 | self.data_y = data[border1:border2] 232 | self.label = label[border1:border2] 233 | 234 | df_stamp['date'] = pd.to_datetime(df_stamp.date) 235 | df_stamp['month'] = df_stamp.date.apply(lambda row:row.month,1) 236 | df_stamp['day'] = df_stamp.date.apply(lambda row:row.day,1) 237 | df_stamp['weekday'] = df_stamp.date.apply(lambda row:row.weekday(),1) 238 | df_stamp['hour'] = df_stamp.date.apply(lambda row:row.hour,1) 239 | df_stamp['minute'] = df_stamp.date.apply(lambda row:row.minute,1) 240 | # df_stamp['minute'] = df_stamp.minute.map(lambda x:x//10) 241 | df_stamp['second'] = df_stamp.date.apply(lambda row:row.second,1) 242 | df_stamp['second'] = df_stamp.second.map(lambda x:x//10) 243 | data_stamp = df_stamp.drop(['date'],1).values 244 | 245 | self.data_stamp = data_stamp 246 | 247 | def __getitem__(self, index): 248 | s_begin = index 249 | s_end = s_begin + self.seq_len 250 | r_begin = s_end - self.label_len 251 | r_end = s_end + self.pred_len 252 | 253 | seq_x = self.data_x[s_begin:s_end] 254 | seq_y = self.data_y[r_begin:r_end] 255 | seq_x_mark = self.data_stamp[s_begin:s_end] 256 | seq_y_mark = self.data_stamp[r_begin:r_end] 257 | 258 | if self.flag == 'train': 259 | return seq_x, seq_y, seq_x_mark, seq_y_mark 260 | else: 261 | seq_label = self.label[s_end:r_end] 262 | return seq_x, seq_y, seq_x_mark, seq_y_mark, seq_label 263 | 264 | def __len__(self): 265 | return len(self.data_x) - self.seq_len - self.pred_len + 1 266 | 267 | 268 | class SWaT(Dataset): 269 | def __init__(self, root_path, flag='train', size=None, 270 | features='M', data_path='SWaT_normaldata_downsampled.csv', 271 | target='FIT_101', scale=True): 272 | # size [seq_len, label_len pred_len] 273 | # info 274 | if size == None: 275 | self.seq_len = 8*60 276 | self.label_len = 2*60 277 | self.pred_len = 2*60 278 | else: 279 | self.seq_len = size[0] 280 | self.label_len = size[1] 281 | self.pred_len = size[2] 282 | # init 283 | assert flag in ['train', 'test', 'val'] 284 | self.flag = flag 285 | type_map = {'train':0, 'val':1, 'test':2} 286 | self.set_type = type_map[flag] 287 | 288 | self.features = features 289 | self.target = target 290 | self.scale = scale 291 | 292 | self.root_path = root_path 293 | self.data_path = data_path 294 | self.__read_data__() 295 | 296 | def __read_data__(self): 297 | scaler = MinMaxScaler() 298 | if self.flag == 'train': 299 | df_raw = pd.read_csv(os.path.join(self.root_path, 300 | 'SWaT_normaldata_downsampled.csv')) 301 | if self.features=='M': 302 | cols_data = df_raw.columns[1:] 303 | df_data = df_raw[cols_data] 304 | elif self.features=='S': 305 | df_data = df_raw[[self.target]] 306 | 307 | df_stamp = df_raw[[' Timestamp']] 308 | 309 | if self.scale: 310 | data = scaler.fit_transform(df_data.values) 311 | else: 312 | data = df_data.values 313 | 314 | self.data_x = data 315 | self.data_y = data 316 | else: 317 | df_raw = pd.read_csv(os.path.join(self.root_path, 318 | 'SWaT_attackdata_downsampled.csv')) 319 | 320 | border1s = [0, 0, 0] 321 | border2s = [None, len(df_raw)//4, len(df_raw)] 322 | border1 = border1s[self.set_type] 323 | border2 = border2s[self.set_type] 324 | 325 | df_stamp = df_raw[[' Timestamp']][border1:border2] 326 | 327 | if self.features=='M': 328 | cols_data = df_raw.columns[1:-1] 329 | df_data = df_raw[cols_data] 330 | label = df_raw['Normal/Attack'].values 331 | elif self.features=='S': 332 | df_data = df_raw[[self.target]] 333 | label = df_raw['Normal/Attack'].values 334 | 335 | if self.scale: 336 | data = scaler.fit_transform(df_data.values) 337 | else: 338 | data = df_data.values 339 | 340 | self.data_x = data[border1:border2] 341 | self.data_y = data[border1:border2] 342 | self.label = label[border1:border2] 343 | 344 | df_stamp[' Timestamp'] = pd.to_datetime(df_stamp[' Timestamp']) 345 | df_stamp['month'] = df_stamp[' Timestamp'].apply(lambda row:row.month,1) 346 | df_stamp['day'] = df_stamp[' Timestamp'].apply(lambda row:row.day,1) 347 | df_stamp['weekday'] = df_stamp[' Timestamp'].apply(lambda row:row.weekday(),1) 348 | df_stamp['hour'] = df_stamp[' Timestamp'].apply(lambda row:row.hour,1) 349 | df_stamp['minute'] = df_stamp[' Timestamp'].apply(lambda row:row.minute,1) 350 | # df_stamp['minute'] = df_stamp.minute.map(lambda x:x//10) 351 | df_stamp['second'] = df_stamp[' Timestamp'].apply(lambda row:row.second,1) 352 | df_stamp['second'] = df_stamp.second.map(lambda x:x//10) 353 | data_stamp = df_stamp.drop([' Timestamp'],1).values 354 | 355 | self.data_stamp = data_stamp 356 | 357 | def __getitem__(self, index): 358 | s_begin = index 359 | s_end = s_begin + self.seq_len 360 | r_begin = s_end - self.label_len 361 | r_end = s_end + self.pred_len 362 | 363 | seq_x = self.data_x[s_begin:s_end] 364 | seq_y = self.data_y[r_begin:r_end] 365 | seq_x_mark = self.data_stamp[s_begin:s_end] 366 | seq_y_mark = self.data_stamp[r_begin:r_end] 367 | 368 | if self.flag == 'train': 369 | return seq_x, seq_y, seq_x_mark, seq_y_mark 370 | else: 371 | seq_label = self.label[s_end:r_end] 372 | return seq_x, seq_y, seq_x_mark, seq_y_mark, seq_label 373 | 374 | def __len__(self): 375 | return len(self.data_x) - self.seq_len - self.pred_len + 1 376 | 377 | 378 | 379 | if __name__ == '__main__': 380 | flag = 'test' 381 | dataset = NASA_Anomaly(root_path='./data/', data_path='MSL', flag=flag, size=(60, 30, 1)) 382 | print(flag, len(dataset)) 383 | # data_loader = DataLoader( 384 | # dataset, 385 | # batch_size=32, 386 | # shuffle=True, 387 | # num_workers=2, 388 | # drop_last=True) 389 | # for (x, y, x_stamp, y_stamp, label) in data_loader: 390 | # print(x.size(), y.size(), x_stamp.size(), y_stamp.size(), label.size()) 391 | # break -------------------------------------------------------------------------------- /utils/spot.py: -------------------------------------------------------------------------------- 1 | from scipy.optimize import minimize 2 | from math import log,floor 3 | import numpy as np 4 | import pandas as pd 5 | import matplotlib.pyplot as plt 6 | import tqdm 7 | 8 | # colors for plot 9 | deep_saffron = '#FF9933' 10 | air_force_blue = '#5D8AA8' 11 | 12 | 13 | """ 14 | ================================= MAIN CLASS ================================== 15 | """ 16 | 17 | class SPOT: 18 | """ 19 | This class allows to run SPOT algorithm on univariate dataset (upper-bound) 20 | 21 | Attributes 22 | ---------- 23 | proba : float 24 | Detection level (risk), chosen by the user 25 | 26 | extreme_quantile : float 27 | current threshold (bound between normal and abnormal events) 28 | 29 | data : numpy.array 30 | stream 31 | 32 | init_data : numpy.array 33 | initial batch of observations (for the calibration/initialization step) 34 | 35 | init_threshold : float 36 | initial threshold computed during the calibration step 37 | 38 | peaks : numpy.array 39 | array of peaks (excesses above the initial threshold) 40 | 41 | n : int 42 | number of observed values 43 | 44 | Nt : int 45 | number of observed peaks 46 | """ 47 | 48 | def __init__(self, q = 1e-4): 49 | """ 50 | Constructor 51 | Parameters 52 | ---------- 53 | q 54 | Detection level (risk) 55 | 56 | Returns 57 | ---------- 58 | SPOT object 59 | """ 60 | self.proba = q 61 | self.extreme_quantile = None 62 | self.data = None 63 | self.init_data = None 64 | self.init_threshold = None 65 | self.peaks = None 66 | self.n = 0 67 | self.Nt = 0 68 | 69 | def __str__(self): 70 | s = '' 71 | s += 'Streaming Peaks-Over-Threshold Object\n' 72 | s += 'Detection level q = %s\n' % self.proba 73 | if self.data is not None: 74 | s += 'Data imported : Yes\n' 75 | s += '\t initialization : %s values\n' % self.init_data.size 76 | s += '\t stream : %s values\n' % self.data.size 77 | else: 78 | s += 'Data imported : No\n' 79 | return s 80 | 81 | if self.n == 0: 82 | s += 'Algorithm initialized : No\n' 83 | else: 84 | s += 'Algorithm initialized : Yes\n' 85 | s += '\t initial threshold : %s\n' % self.init_threshold 86 | 87 | r = self.n-self.init_data.size 88 | if r > 0: 89 | s += 'Algorithm run : Yes\n' 90 | s += '\t number of observations : %s (%.2f %%)\n' % (r,100*r/self.n) 91 | else: 92 | s += '\t number of peaks : %s\n' % self.Nt 93 | s += '\t extreme quantile : %s\n' % self.extreme_quantile 94 | s += 'Algorithm run : No\n' 95 | return s 96 | 97 | 98 | def fit(self,init_data,data): 99 | """ 100 | Import data to SPOT object 101 | 102 | Parameters 103 | ---------- 104 | init_data : list, numpy.array or pandas.Series 105 | initial batch to calibrate the algorithm 106 | 107 | data : numpy.array 108 | data for the run (list, np.array or pd.series) 109 | 110 | """ 111 | if isinstance(data,list): 112 | self.data = np.array(data) 113 | elif isinstance(data,np.ndarray): 114 | self.data = data 115 | elif isinstance(data,pd.Series): 116 | self.data = data.values 117 | else: 118 | print('This data format (%s) is not supported' % type(data)) 119 | return 120 | 121 | if isinstance(init_data,list): 122 | self.init_data = np.array(init_data) 123 | elif isinstance(init_data,np.ndarray): 124 | self.init_data = init_data 125 | elif isinstance(init_data,pd.Series): 126 | self.init_data = init_data.values 127 | elif isinstance(init_data,int): 128 | self.init_data = self.data[:init_data] 129 | self.data = self.data[init_data:] 130 | elif isinstance(init_data,float) & (init_data<1) & (init_data>0): 131 | r = int(init_data*data.size) 132 | self.init_data = self.data[:r] 133 | self.data = self.data[r:] 134 | else: 135 | print('The initial data cannot be set') 136 | return 137 | 138 | def add(self,data): 139 | """ 140 | This function allows to append data to the already fitted data 141 | 142 | Parameters 143 | ---------- 144 | data : list, numpy.array, pandas.Series 145 | data to append 146 | """ 147 | if isinstance(data,list): 148 | data = np.array(data) 149 | elif isinstance(data,np.ndarray): 150 | data = data 151 | elif isinstance(data,pd.Series): 152 | data = data.values 153 | else: 154 | print('This data format (%s) is not supported' % type(data)) 155 | return 156 | 157 | self.data = np.append(self.data,data) 158 | return 159 | 160 | def initialize(self, level = 0.98, verbose = True): 161 | """ 162 | Run the calibration (initialization) step 163 | 164 | Parameters 165 | ---------- 166 | level : float 167 | (default 0.98) Probability associated with the initial threshold t 168 | verbose : bool 169 | (default = True) If True, gives details about the batch initialization 170 | """ 171 | level = level-floor(level) 172 | 173 | n_init = self.init_data.size 174 | 175 | S = np.sort(self.init_data) # we sort X to get the empirical quantile 176 | self.init_threshold = S[int(level*n_init)] # t is fixed for the whole algorithm 177 | 178 | # initial peaks 179 | self.peaks = self.init_data[self.init_data>self.init_threshold]-self.init_threshold 180 | self.Nt = self.peaks.size 181 | self.n = n_init 182 | 183 | if verbose: 184 | print('Initial threshold : %s' % self.init_threshold) 185 | print('Number of peaks : %s' % self.Nt) 186 | print('Grimshaw maximum log-likelihood estimation ... ', end = '') 187 | 188 | g,s,l = self._grimshaw() 189 | self.extreme_quantile = self._quantile(g,s) 190 | 191 | if verbose: 192 | print('[done]') 193 | print('\t'+chr(0x03B3) + ' = ' + str(g)) 194 | print('\t'+chr(0x03C3) + ' = ' + str(s)) 195 | print('\tL = ' + str(l)) 196 | print('Extreme quantile (probability = %s): %s' % (self.proba,self.extreme_quantile)) 197 | 198 | return 199 | 200 | 201 | 202 | 203 | def _rootsFinder(fun,jac,bounds,npoints,method): 204 | """ 205 | Find possible roots of a scalar function 206 | 207 | Parameters 208 | ---------- 209 | fun : function 210 | scalar function 211 | jac : function 212 | first order derivative of the function 213 | bounds : tuple 214 | (min,max) interval for the roots search 215 | npoints : int 216 | maximum number of roots to output 217 | method : str 218 | 'regular' : regular sample of the search interval, 'random' : uniform (distribution) sample of the search interval 219 | 220 | Returns 221 | ---------- 222 | numpy.array 223 | possible roots of the function 224 | """ 225 | if method == 'regular': 226 | step = (bounds[1]-bounds[0])/(npoints+1) 227 | X0 = np.arange(bounds[0]+step,bounds[1],step) 228 | elif method == 'random': 229 | X0 = np.random.uniform(bounds[0],bounds[1],npoints) 230 | 231 | def objFun(X,f,jac): 232 | g = 0 233 | j = np.zeros(X.shape) 234 | i = 0 235 | for x in X: 236 | fx = f(x) 237 | g = g+fx**2 238 | j[i] = 2*fx*jac(x) 239 | i = i+1 240 | return g,j 241 | 242 | opt = minimize(lambda X:objFun(X,fun,jac), X0, 243 | method='L-BFGS-B', 244 | jac=True, bounds=[bounds]*len(X0)) 245 | 246 | X = opt.x 247 | np.round(X,decimals = 5) 248 | return np.unique(X) 249 | 250 | 251 | def _log_likelihood(Y,gamma,sigma): 252 | """ 253 | Compute the log-likelihood for the Generalized Pareto Distribution (μ=0) 254 | 255 | Parameters 256 | ---------- 257 | Y : numpy.array 258 | observations 259 | gamma : float 260 | GPD index parameter 261 | sigma : float 262 | GPD scale parameter (>0) 263 | Returns 264 | ---------- 265 | float 266 | log-likelihood of the sample Y to be drawn from a GPD(γ,σ,μ=0) 267 | """ 268 | n = Y.size 269 | if gamma != 0: 270 | tau = gamma/sigma 271 | L = -n * log(sigma) - ( 1 + (1/gamma) ) * ( np.log(1+tau*Y) ).sum() 272 | else: 273 | L = n * ( 1 + log(Y.mean()) ) 274 | return L 275 | 276 | 277 | def _grimshaw(self,epsilon = 1e-8, n_points = 10): 278 | """ 279 | Compute the GPD parameters estimation with the Grimshaw's trick 280 | 281 | Parameters 282 | ---------- 283 | epsilon : float 284 | numerical parameter to perform (default : 1e-8) 285 | n_points : int 286 | maximum number of candidates for maximum likelihood (default : 10) 287 | Returns 288 | ---------- 289 | gamma_best,sigma_best,ll_best 290 | gamma estimates, sigma estimates and corresponding log-likelihood 291 | """ 292 | def u(s): 293 | return 1 + np.log(s).mean() 294 | 295 | def v(s): 296 | return np.mean(1/s) 297 | 298 | def w(Y,t): 299 | s = 1+t*Y 300 | us = u(s) 301 | vs = v(s) 302 | return us*vs-1 303 | 304 | def jac_w(Y,t): 305 | s = 1+t*Y 306 | us = u(s) 307 | vs = v(s) 308 | jac_us = (1/t)*(1-vs) 309 | jac_vs = (1/t)*(-vs+np.mean(1/s**2)) 310 | return us*jac_vs+vs*jac_us 311 | 312 | 313 | Ym = self.peaks.min() 314 | YM = self.peaks.max() 315 | Ymean = self.peaks.mean() 316 | 317 | 318 | a = -1/YM 319 | if abs(a)<2*epsilon: 320 | epsilon = abs(a)/n_points 321 | 322 | a = a + epsilon 323 | b = 2*(Ymean-Ym)/(Ymean*Ym) 324 | c = 2*(Ymean-Ym)/(Ym**2) 325 | 326 | # We look for possible roots 327 | left_zeros = SPOT._rootsFinder(lambda t: w(self.peaks,t), 328 | lambda t: jac_w(self.peaks,t), 329 | (a+epsilon,-epsilon), 330 | n_points,'regular') 331 | 332 | right_zeros = SPOT._rootsFinder(lambda t: w(self.peaks,t), 333 | lambda t: jac_w(self.peaks,t), 334 | (b,c), 335 | n_points,'regular') 336 | 337 | # all the possible roots 338 | zeros = np.concatenate((left_zeros,right_zeros)) 339 | 340 | # 0 is always a solution so we initialize with it 341 | gamma_best = 0 342 | sigma_best = Ymean 343 | ll_best = SPOT._log_likelihood(self.peaks,gamma_best,sigma_best) 344 | 345 | # we look for better candidates 346 | for z in zeros: 347 | gamma = u(1+z*self.peaks)-1 348 | sigma = gamma/z 349 | ll = SPOT._log_likelihood(self.peaks,gamma,sigma) 350 | if ll>ll_best: 351 | gamma_best = gamma 352 | sigma_best = sigma 353 | ll_best = ll 354 | 355 | return gamma_best,sigma_best,ll_best 356 | 357 | 358 | 359 | def _quantile(self,gamma,sigma): 360 | """ 361 | Compute the quantile at level 1-q 362 | 363 | Parameters 364 | ---------- 365 | gamma : float 366 | GPD parameter 367 | sigma : float 368 | GPD parameter 369 | Returns 370 | ---------- 371 | float 372 | quantile at level 1-q for the GPD(γ,σ,μ=0) 373 | """ 374 | r = self.n * self.proba / self.Nt 375 | if gamma != 0: 376 | return self.init_threshold + (sigma/gamma)*(pow(r,-gamma)-1) 377 | else: 378 | return self.init_threshold - sigma*log(r) 379 | 380 | 381 | def run(self, with_alarm = True): 382 | """ 383 | Run SPOT on the stream 384 | 385 | Parameters 386 | ---------- 387 | with_alarm : bool 388 | (default = True) If False, SPOT will adapt the threshold assuming \ 389 | there is no abnormal values 390 | Returns 391 | ---------- 392 | dict 393 | keys : 'thresholds' and 'alarms' 394 | 395 | 'thresholds' contains the extreme quantiles and 'alarms' contains \ 396 | the indexes of the values which have triggered alarms 397 | 398 | """ 399 | if (self.n>self.init_data.size): 400 | print('Warning : the algorithm seems to have already been run, you \ 401 | should initialize before running again') 402 | return {} 403 | 404 | # list of the thresholds 405 | th = [] 406 | alarm = [] 407 | # Loop over the stream 408 | for i in tqdm.tqdm(range(self.data.size)): 409 | 410 | # If the observed value exceeds the current threshold (alarm case) 411 | if self.data[i]>self.extreme_quantile: 412 | # if we want to alarm, we put it in the alarm list 413 | if with_alarm: 414 | alarm.append(i) 415 | # otherwise we add it in the peaks 416 | else: 417 | self.peaks = np.append(self.peaks,self.data[i]-self.init_threshold) 418 | self.Nt += 1 419 | self.n += 1 420 | # and we update the thresholds 421 | 422 | g,s,l = self._grimshaw() 423 | self.extreme_quantile = self._quantile(g,s) 424 | 425 | # case where the value exceeds the initial threshold but not the alarm ones 426 | elif self.data[i]>self.init_threshold: 427 | # we add it in the peaks 428 | self.peaks = np.append(self.peaks,self.data[i]-self.init_threshold) 429 | self.Nt += 1 430 | self.n += 1 431 | # and we update the thresholds 432 | 433 | g,s,l = self._grimshaw() 434 | self.extreme_quantile = self._quantile(g,s) 435 | else: 436 | self.n += 1 437 | 438 | 439 | th.append(self.extreme_quantile) # thresholds record 440 | 441 | return {'thresholds' : th, 'alarms': alarm} 442 | 443 | 444 | def plot(self,run_results,with_alarm = True): 445 | """ 446 | Plot the results of given by the run 447 | 448 | Parameters 449 | ---------- 450 | run_results : dict 451 | results given by the 'run' method 452 | with_alarm : bool 453 | (default = True) If True, alarms are plotted. 454 | Returns 455 | ---------- 456 | list 457 | list of the plots 458 | 459 | """ 460 | x = range(self.data.size) 461 | K = run_results.keys() 462 | 463 | ts_fig, = plt.plot(x,self.data,color=air_force_blue) 464 | fig = [ts_fig] 465 | 466 | if 'thresholds' in K: 467 | th = run_results['thresholds'] 468 | th_fig, = plt.plot(x,th,color=deep_saffron,lw=2,ls='dashed') 469 | fig.append(th_fig) 470 | 471 | if with_alarm and ('alarms' in K): 472 | alarm = run_results['alarms'] 473 | al_fig = plt.scatter(alarm,self.data[alarm],color='red') 474 | fig.append(al_fig) 475 | 476 | plt.xlim((0,self.data.size)) 477 | 478 | 479 | return fig 480 | 481 | 482 | 483 | 484 | 485 | 486 | 487 | 488 | 489 | 490 | """ 491 | ============================ UPPER & LOWER BOUNDS ============================= 492 | """ 493 | 494 | 495 | 496 | 497 | class biSPOT: 498 | """ 499 | This class allows to run biSPOT algorithm on univariate dataset (upper and lower bounds) 500 | 501 | Attributes 502 | ---------- 503 | proba : float 504 | Detection level (risk), chosen by the user 505 | 506 | extreme_quantile : float 507 | current threshold (bound between normal and abnormal events) 508 | 509 | data : numpy.array 510 | stream 511 | 512 | init_data : numpy.array 513 | initial batch of observations (for the calibration/initialization step) 514 | 515 | init_threshold : float 516 | initial threshold computed during the calibration step 517 | 518 | peaks : numpy.array 519 | array of peaks (excesses above the initial threshold) 520 | 521 | n : int 522 | number of observed values 523 | 524 | Nt : int 525 | number of observed peaks 526 | """ 527 | def __init__(self, q = 1e-4): 528 | """ 529 | Constructor 530 | Parameters 531 | ---------- 532 | q 533 | Detection level (risk) 534 | 535 | Returns 536 | ---------- 537 | biSPOT object 538 | """ 539 | self.proba = q 540 | self.data = None 541 | self.init_data = None 542 | self.n = 0 543 | nonedict = {'up':None,'down':None} 544 | 545 | self.extreme_quantile = dict.copy(nonedict) 546 | self.init_threshold = dict.copy(nonedict) 547 | self.peaks = dict.copy(nonedict) 548 | self.gamma = dict.copy(nonedict) 549 | self.sigma = dict.copy(nonedict) 550 | self.Nt = {'up':0,'down':0} 551 | 552 | 553 | def __str__(self): 554 | s = '' 555 | s += 'Streaming Peaks-Over-Threshold Object\n' 556 | s += 'Detection level q = %s\n' % self.proba 557 | if self.data is not None: 558 | s += 'Data imported : Yes\n' 559 | s += '\t initialization : %s values\n' % self.init_data.size 560 | s += '\t stream : %s values\n' % self.data.size 561 | else: 562 | s += 'Data imported : No\n' 563 | return s 564 | 565 | if self.n == 0: 566 | s += 'Algorithm initialized : No\n' 567 | else: 568 | s += 'Algorithm initialized : Yes\n' 569 | s += '\t initial threshold : %s\n' % self.init_threshold 570 | 571 | r = self.n-self.init_data.size 572 | if r > 0: 573 | s += 'Algorithm run : Yes\n' 574 | s += '\t number of observations : %s (%.2f %%)\n' % (r,100*r/self.n) 575 | s += '\t triggered alarms : %s (%.2f %%)\n' % (len(self.alarm),100*len(self.alarm)/self.n) 576 | else: 577 | s += '\t number of peaks : %s\n' % self.Nt 578 | s += '\t upper extreme quantile : %s\n' % self.extreme_quantile['up'] 579 | s += '\t lower extreme quantile : %s\n' % self.extreme_quantile['down'] 580 | s += 'Algorithm run : No\n' 581 | return s 582 | 583 | 584 | def fit(self,init_data,data): 585 | """ 586 | Import data to biSPOT object 587 | 588 | Parameters 589 | ---------- 590 | init_data : list, numpy.array or pandas.Series 591 | initial batch to calibrate the algorithm () 592 | 593 | data : numpy.array 594 | data for the run (list, np.array or pd.series) 595 | 596 | """ 597 | if isinstance(data,list): 598 | self.data = np.array(data) 599 | elif isinstance(data,np.ndarray): 600 | self.data = data 601 | elif isinstance(data,pd.Series): 602 | self.data = data.values 603 | else: 604 | print('This data format (%s) is not supported' % type(data)) 605 | return 606 | 607 | if isinstance(init_data,list): 608 | self.init_data = np.array(init_data) 609 | elif isinstance(init_data,np.ndarray): 610 | self.init_data = init_data 611 | elif isinstance(init_data,pd.Series): 612 | self.init_data = init_data.values 613 | elif isinstance(init_data,int): 614 | self.init_data = self.data[:init_data] 615 | self.data = self.data[init_data:] 616 | elif isinstance(init_data,float) & (init_data<1) & (init_data>0): 617 | r = int(init_data*data.size) 618 | self.init_data = self.data[:r] 619 | self.data = self.data[r:] 620 | else: 621 | print('The initial data cannot be set') 622 | return 623 | 624 | def add(self,data): 625 | """ 626 | This function allows to append data to the already fitted data 627 | 628 | Parameters 629 | ---------- 630 | data : list, numpy.array, pandas.Series 631 | data to append 632 | """ 633 | if isinstance(data,list): 634 | data = np.array(data) 635 | elif isinstance(data,np.ndarray): 636 | data = data 637 | elif isinstance(data,pd.Series): 638 | data = data.values 639 | else: 640 | print('This data format (%s) is not supported' % type(data)) 641 | return 642 | 643 | self.data = np.append(self.data,data) 644 | return 645 | 646 | def initialize(self, verbose = True): 647 | """ 648 | Run the calibration (initialization) step 649 | 650 | Parameters 651 | ---------- 652 | verbose : bool 653 | (default = True) If True, gives details about the batch initialization 654 | """ 655 | n_init = self.init_data.size 656 | 657 | S = np.sort(self.init_data) # we sort X to get the empirical quantile 658 | self.init_threshold['up'] = S[int(0.98*n_init)] # t is fixed for the whole algorithm 659 | self.init_threshold['down'] = S[int(0.02*n_init)] # t is fixed for the whole algorithm 660 | 661 | # initial peaks 662 | self.peaks['up'] = self.init_data[self.init_data>self.init_threshold['up']]-self.init_threshold['up'] 663 | self.peaks['down'] = -(self.init_data[self.init_data0) 755 | Returns 756 | ---------- 757 | float 758 | log-likelihood of the sample Y to be drawn from a GPD(γ,σ,μ=0) 759 | """ 760 | n = Y.size 761 | if gamma != 0: 762 | tau = gamma/sigma 763 | L = -n * log(sigma) - ( 1 + (1/gamma) ) * ( np.log(1+tau*Y) ).sum() 764 | else: 765 | L = n * ( 1 + log(Y.mean()) ) 766 | return L 767 | 768 | 769 | def _grimshaw(self,side,epsilon = 1e-8, n_points = 10): 770 | """ 771 | Compute the GPD parameters estimation with the Grimshaw's trick 772 | 773 | Parameters 774 | ---------- 775 | epsilon : float 776 | numerical parameter to perform (default : 1e-8) 777 | n_points : int 778 | maximum number of candidates for maximum likelihood (default : 10) 779 | Returns 780 | ---------- 781 | gamma_best,sigma_best,ll_best 782 | gamma estimates, sigma estimates and corresponding log-likelihood 783 | """ 784 | def u(s): 785 | return 1 + np.log(s).mean() 786 | 787 | def v(s): 788 | return np.mean(1/s) 789 | 790 | def w(Y,t): 791 | s = 1+t*Y 792 | us = u(s) 793 | vs = v(s) 794 | return us*vs-1 795 | 796 | def jac_w(Y,t): 797 | s = 1+t*Y 798 | us = u(s) 799 | vs = v(s) 800 | jac_us = (1/t)*(1-vs) 801 | jac_vs = (1/t)*(-vs+np.mean(1/s**2)) 802 | return us*jac_vs+vs*jac_us 803 | 804 | 805 | Ym = self.peaks[side].min() 806 | YM = self.peaks[side].max() 807 | Ymean = self.peaks[side].mean() 808 | 809 | 810 | a = -1/YM 811 | if abs(a)<2*epsilon: 812 | epsilon = abs(a)/n_points 813 | 814 | a = a + epsilon 815 | b = 2*(Ymean-Ym)/(Ymean*Ym) 816 | c = 2*(Ymean-Ym)/(Ym**2) 817 | 818 | # We look for possible roots 819 | left_zeros = biSPOT._rootsFinder(lambda t: w(self.peaks[side],t), 820 | lambda t: jac_w(self.peaks[side],t), 821 | (a+epsilon,-epsilon), 822 | n_points,'regular') 823 | 824 | right_zeros = biSPOT._rootsFinder(lambda t: w(self.peaks[side],t), 825 | lambda t: jac_w(self.peaks[side],t), 826 | (b,c), 827 | n_points,'regular') 828 | 829 | # all the possible roots 830 | zeros = np.concatenate((left_zeros,right_zeros)) 831 | 832 | # 0 is always a solution so we initialize with it 833 | gamma_best = 0 834 | sigma_best = Ymean 835 | ll_best = biSPOT._log_likelihood(self.peaks[side],gamma_best,sigma_best) 836 | 837 | # we look for better candidates 838 | for z in zeros: 839 | gamma = u(1+z*self.peaks[side])-1 840 | sigma = gamma/z 841 | ll = biSPOT._log_likelihood(self.peaks[side],gamma,sigma) 842 | if ll>ll_best: 843 | gamma_best = gamma 844 | sigma_best = sigma 845 | ll_best = ll 846 | 847 | return gamma_best,sigma_best,ll_best 848 | 849 | 850 | 851 | def _quantile(self,side,gamma,sigma): 852 | """ 853 | Compute the quantile at level 1-q for a given side 854 | 855 | Parameters 856 | ---------- 857 | side : str 858 | 'up' or 'down' 859 | gamma : float 860 | GPD parameter 861 | sigma : float 862 | GPD parameter 863 | Returns 864 | ---------- 865 | float 866 | quantile at level 1-q for the GPD(γ,σ,μ=0) 867 | """ 868 | if side == 'up': 869 | r = self.n * self.proba / self.Nt[side] 870 | if gamma != 0: 871 | return self.init_threshold['up'] + (sigma/gamma)*(pow(r,-gamma)-1) 872 | else: 873 | return self.init_threshold['up'] - sigma*log(r) 874 | elif side == 'down': 875 | r = self.n * self.proba / self.Nt[side] 876 | if gamma != 0: 877 | return self.init_threshold['down'] - (sigma/gamma)*(pow(r,-gamma)-1) 878 | else: 879 | return self.init_threshold['down'] + sigma*log(r) 880 | else: 881 | print('error : the side is not right') 882 | 883 | 884 | def run(self, with_alarm = True): 885 | """ 886 | Run biSPOT on the stream 887 | 888 | Parameters 889 | ---------- 890 | with_alarm : bool 891 | (default = True) If False, SPOT will adapt the threshold assuming \ 892 | there is no abnormal values 893 | Returns 894 | ---------- 895 | dict 896 | keys : 'upper_thresholds', 'lower_thresholds' and 'alarms' 897 | 898 | '***-thresholds' contains the extreme quantiles and 'alarms' contains \ 899 | the indexes of the values which have triggered alarms 900 | 901 | """ 902 | if (self.n>self.init_data.size): 903 | print('Warning : the algorithm seems to have already been run, you \ 904 | should initialize before running again') 905 | return {} 906 | 907 | # list of the thresholds 908 | thup = [] 909 | thdown = [] 910 | alarm = [] 911 | # Loop over the stream 912 | for i in tqdm.tqdm(range(self.data.size)): 913 | 914 | # If the observed value exceeds the current threshold (alarm case) 915 | if self.data[i]>self.extreme_quantile['up'] : 916 | # if we want to alarm, we put it in the alarm list 917 | if with_alarm: 918 | alarm.append(i) 919 | # otherwise we add it in the peaks 920 | else: 921 | self.peaks['up'] = np.append(self.peaks['up'],self.data[i]-self.init_threshold['up']) 922 | self.Nt['up'] += 1 923 | self.n += 1 924 | # and we update the thresholds 925 | 926 | g,s,l = self._grimshaw('up') 927 | self.extreme_quantile['up'] = self._quantile('up',g,s) 928 | 929 | # case where the value exceeds the initial threshold but not the alarm ones 930 | elif self.data[i]>self.init_threshold['up']: 931 | # we add it in the peaks 932 | self.peaks['up'] = np.append(self.peaks['up'],self.data[i]-self.init_threshold['up']) 933 | self.Nt['up'] += 1 934 | self.n += 1 935 | # and we update the thresholds 936 | 937 | g,s,l = self._grimshaw('up') 938 | self.extreme_quantile['up'] = self._quantile('up',g,s) 939 | 940 | elif self.data[i] 0: 1101 | s += 'Algorithm run : Yes\n' 1102 | s += '\t number of observations : %s (%.2f %%)\n' % (r,100*r/self.n) 1103 | s += '\t triggered alarms : %s (%.2f %%)\n' % (len(self.alarm),100*len(self.alarm)/self.n) 1104 | else: 1105 | s += '\t number of peaks : %s\n' % self.Nt 1106 | s += '\t extreme quantile : %s\n' % self.extreme_quantile 1107 | s += 'Algorithm run : No\n' 1108 | return s 1109 | 1110 | 1111 | def fit(self,init_data,data): 1112 | """ 1113 | Import data to DSPOT object 1114 | 1115 | Parameters 1116 | ---------- 1117 | init_data : list, numpy.array or pandas.Series 1118 | initial batch to calibrate the algorithm 1119 | 1120 | data : numpy.array 1121 | data for the run (list, np.array or pd.series) 1122 | 1123 | """ 1124 | if isinstance(data,list): 1125 | self.data = np.array(data) 1126 | elif isinstance(data,np.ndarray): 1127 | self.data = data 1128 | elif isinstance(data,pd.Series): 1129 | self.data = data.values 1130 | else: 1131 | print('This data format (%s) is not supported' % type(data)) 1132 | return 1133 | 1134 | if isinstance(init_data,list): 1135 | self.init_data = np.array(init_data) 1136 | elif isinstance(init_data,np.ndarray): 1137 | self.init_data = init_data 1138 | elif isinstance(init_data,pd.Series): 1139 | self.init_data = init_data.values 1140 | elif isinstance(init_data,int): 1141 | self.init_data = self.data[:init_data] 1142 | self.data = self.data[init_data:] 1143 | elif isinstance(init_data,float) & (init_data<1) & (init_data>0): 1144 | r = int(init_data*data.size) 1145 | self.init_data = self.data[:r] 1146 | self.data = self.data[r:] 1147 | else: 1148 | print('The initial data cannot be set') 1149 | return 1150 | 1151 | def add(self,data): 1152 | """ 1153 | This function allows to append data to the already fitted data 1154 | 1155 | Parameters 1156 | ---------- 1157 | data : list, numpy.array, pandas.Series 1158 | data to append 1159 | """ 1160 | if isinstance(data,list): 1161 | data = np.array(data) 1162 | elif isinstance(data,np.ndarray): 1163 | data = data 1164 | elif isinstance(data,pd.Series): 1165 | data = data.values 1166 | else: 1167 | print('This data format (%s) is not supported' % type(data)) 1168 | return 1169 | 1170 | self.data = np.append(self.data,data) 1171 | return 1172 | 1173 | def initialize(self, verbose = True): 1174 | """ 1175 | Run the calibration (initialization) step 1176 | 1177 | Parameters 1178 | ---------- 1179 | verbose : bool 1180 | (default = True) If True, gives details about the batch initialization 1181 | """ 1182 | n_init = self.init_data.size - self.depth 1183 | 1184 | M = backMean(self.init_data,self.depth) 1185 | T = self.init_data[self.depth:]-M[:-1] # new variable 1186 | 1187 | S = np.sort(T) # we sort X to get the empirical quantile 1188 | self.init_threshold = S[int(0.98*n_init)] # t is fixed for the whole algorithm 1189 | 1190 | # initial peaks 1191 | self.peaks = T[T>self.init_threshold]-self.init_threshold 1192 | self.Nt = self.peaks.size 1193 | self.n = n_init 1194 | 1195 | if verbose: 1196 | print('Initial threshold : %s' % self.init_threshold) 1197 | print('Number of peaks : %s' % self.Nt) 1198 | print('Grimshaw maximum log-likelihood estimation ... ', end = '') 1199 | 1200 | g,s,l = self._grimshaw() 1201 | self.extreme_quantile = self._quantile(g,s) 1202 | 1203 | if verbose: 1204 | print('[done]') 1205 | print('\t'+chr(0x03B3) + ' = ' + str(g)) 1206 | print('\t'+chr(0x03C3) + ' = ' + str(s)) 1207 | print('\tL = ' + str(l)) 1208 | print('Extreme quantile (probability = %s): %s' % (self.proba,self.extreme_quantile)) 1209 | 1210 | return 1211 | 1212 | 1213 | 1214 | 1215 | def _rootsFinder(fun,jac,bounds,npoints,method): 1216 | """ 1217 | Find possible roots of a scalar function 1218 | 1219 | Parameters 1220 | ---------- 1221 | fun : function 1222 | scalar function 1223 | jac : function 1224 | first order derivative of the function 1225 | bounds : tuple 1226 | (min,max) interval for the roots search 1227 | npoints : int 1228 | maximum number of roots to output 1229 | method : str 1230 | 'regular' : regular sample of the search interval, 'random' : uniform (distribution) sample of the search interval 1231 | 1232 | Returns 1233 | ---------- 1234 | numpy.array 1235 | possible roots of the function 1236 | """ 1237 | if method == 'regular': 1238 | step = (bounds[1]-bounds[0])/(npoints+1) 1239 | X0 = np.arange(bounds[0]+step,bounds[1],step) 1240 | elif method == 'random': 1241 | X0 = np.random.uniform(bounds[0],bounds[1],npoints) 1242 | 1243 | def objFun(X,f,jac): 1244 | g = 0 1245 | j = np.zeros(X.shape) 1246 | i = 0 1247 | for x in X: 1248 | fx = f(x) 1249 | g = g+fx**2 1250 | j[i] = 2*fx*jac(x) 1251 | i = i+1 1252 | return g,j 1253 | 1254 | opt = minimize(lambda X:objFun(X,fun,jac), X0, 1255 | method='L-BFGS-B', 1256 | jac=True, bounds=[bounds]*len(X0)) 1257 | 1258 | X = opt.x 1259 | np.round(X,decimals = 5) 1260 | return np.unique(X) 1261 | 1262 | 1263 | def _log_likelihood(Y,gamma,sigma): 1264 | """ 1265 | Compute the log-likelihood for the Generalized Pareto Distribution (μ=0) 1266 | 1267 | Parameters 1268 | ---------- 1269 | Y : numpy.array 1270 | observations 1271 | gamma : float 1272 | GPD index parameter 1273 | sigma : float 1274 | GPD scale parameter (>0) 1275 | Returns 1276 | ---------- 1277 | float 1278 | log-likelihood of the sample Y to be drawn from a GPD(γ,σ,μ=0) 1279 | """ 1280 | n = Y.size 1281 | if gamma != 0: 1282 | tau = gamma/sigma 1283 | L = -n * log(sigma) - ( 1 + (1/gamma) ) * ( np.log(1+tau*Y) ).sum() 1284 | else: 1285 | L = n * ( 1 + log(Y.mean()) ) 1286 | return L 1287 | 1288 | 1289 | def _grimshaw(self,epsilon = 1e-8, n_points = 10): 1290 | """ 1291 | Compute the GPD parameters estimation with the Grimshaw's trick 1292 | 1293 | Parameters 1294 | ---------- 1295 | epsilon : float 1296 | numerical parameter to perform (default : 1e-8) 1297 | n_points : int 1298 | maximum number of candidates for maximum likelihood (default : 10) 1299 | Returns 1300 | ---------- 1301 | gamma_best,sigma_best,ll_best 1302 | gamma estimates, sigma estimates and corresponding log-likelihood 1303 | """ 1304 | def u(s): 1305 | return 1 + np.log(s).mean() 1306 | 1307 | def v(s): 1308 | return np.mean(1/s) 1309 | 1310 | def w(Y,t): 1311 | s = 1+t*Y 1312 | us = u(s) 1313 | vs = v(s) 1314 | return us*vs-1 1315 | 1316 | def jac_w(Y,t): 1317 | s = 1+t*Y 1318 | us = u(s) 1319 | vs = v(s) 1320 | jac_us = (1/t)*(1-vs) 1321 | jac_vs = (1/t)*(-vs+np.mean(1/s**2)) 1322 | return us*jac_vs+vs*jac_us 1323 | 1324 | 1325 | Ym = self.peaks.min() 1326 | YM = self.peaks.max() 1327 | Ymean = self.peaks.mean() 1328 | 1329 | 1330 | a = -1/YM 1331 | if abs(a)<2*epsilon: 1332 | epsilon = abs(a)/n_points 1333 | 1334 | a = a + epsilon 1335 | b = 2*(Ymean-Ym)/(Ymean*Ym) 1336 | c = 2*(Ymean-Ym)/(Ym**2) 1337 | 1338 | # We look for possible roots 1339 | left_zeros = SPOT._rootsFinder(lambda t: w(self.peaks,t), 1340 | lambda t: jac_w(self.peaks,t), 1341 | (a+epsilon,-epsilon), 1342 | n_points,'regular') 1343 | 1344 | right_zeros = SPOT._rootsFinder(lambda t: w(self.peaks,t), 1345 | lambda t: jac_w(self.peaks,t), 1346 | (b,c), 1347 | n_points,'regular') 1348 | 1349 | # all the possible roots 1350 | zeros = np.concatenate((left_zeros,right_zeros)) 1351 | 1352 | # 0 is always a solution so we initialize with it 1353 | gamma_best = 0 1354 | sigma_best = Ymean 1355 | ll_best = SPOT._log_likelihood(self.peaks,gamma_best,sigma_best) 1356 | 1357 | # we look for better candidates 1358 | for z in zeros: 1359 | gamma = u(1+z*self.peaks)-1 1360 | sigma = gamma/z 1361 | ll = dSPOT._log_likelihood(self.peaks,gamma,sigma) 1362 | if ll>ll_best: 1363 | gamma_best = gamma 1364 | sigma_best = sigma 1365 | ll_best = ll 1366 | 1367 | return gamma_best,sigma_best,ll_best 1368 | 1369 | 1370 | 1371 | def _quantile(self,gamma,sigma): 1372 | """ 1373 | Compute the quantile at level 1-q 1374 | 1375 | Parameters 1376 | ---------- 1377 | gamma : float 1378 | GPD parameter 1379 | sigma : float 1380 | GPD parameter 1381 | Returns 1382 | ---------- 1383 | float 1384 | quantile at level 1-q for the GPD(γ,σ,μ=0) 1385 | """ 1386 | r = self.n * self.proba / self.Nt 1387 | if gamma != 0: 1388 | return self.init_threshold + (sigma/gamma)*(pow(r,-gamma)-1) 1389 | else: 1390 | return self.init_threshold - sigma*log(r) 1391 | 1392 | 1393 | def run(self, with_alarm = True): 1394 | """ 1395 | Run biSPOT on the stream 1396 | 1397 | Parameters 1398 | ---------- 1399 | with_alarm : bool 1400 | (default = True) If False, SPOT will adapt the threshold assuming \ 1401 | there is no abnormal values 1402 | Returns 1403 | ---------- 1404 | dict 1405 | keys : 'upper_thresholds', 'lower_thresholds' and 'alarms' 1406 | 1407 | '***-thresholds' contains the extreme quantiles and 'alarms' contains \ 1408 | the indexes of the values which have triggered alarms 1409 | 1410 | """ 1411 | if (self.n>self.init_data.size): 1412 | print('Warning : the algorithm seems to have already been run, you \ 1413 | should initialize before running again') 1414 | return {} 1415 | 1416 | # actual normal window 1417 | W = self.init_data[-self.depth:] 1418 | 1419 | # list of the thresholds 1420 | th = [] 1421 | alarm = [] 1422 | # Loop over the stream 1423 | for i in tqdm.tqdm(range(self.data.size)): 1424 | Mi = W.mean() 1425 | # If the observed value exceeds the current threshold (alarm case) 1426 | if (self.data[i]-Mi)>self.extreme_quantile: 1427 | # if we want to alarm, we put it in the alarm list 1428 | if with_alarm: 1429 | alarm.append(i) 1430 | # otherwise we add it in the peaks 1431 | else: 1432 | self.peaks = np.append(self.peaks,self.data[i]-Mi-self.init_threshold) 1433 | self.Nt += 1 1434 | self.n += 1 1435 | # and we update the thresholds 1436 | 1437 | g,s,l = self._grimshaw() 1438 | self.extreme_quantile = self._quantile(g,s) #+ Mi 1439 | W = np.append(W[1:],self.data[i]) 1440 | 1441 | # case where the value exceeds the initial threshold but not the alarm ones 1442 | elif (self.data[i]-Mi)>self.init_threshold: 1443 | # we add it in the peaks 1444 | self.peaks = np.append(self.peaks,self.data[i]-Mi-self.init_threshold) 1445 | self.Nt += 1 1446 | self.n += 1 1447 | # and we update the thresholds 1448 | 1449 | g,s,l = self._grimshaw() 1450 | self.extreme_quantile = self._quantile(g,s) #+ Mi 1451 | W = np.append(W[1:],self.data[i]) 1452 | else: 1453 | self.n += 1 1454 | W = np.append(W[1:],self.data[i]) 1455 | 1456 | 1457 | th.append(self.extreme_quantile+Mi) # thresholds record 1458 | 1459 | return {'thresholds' : th, 'alarms': alarm} 1460 | 1461 | 1462 | def plot(self,run_results, with_alarm = True): 1463 | """ 1464 | Plot the results given by the run 1465 | 1466 | Parameters 1467 | ---------- 1468 | run_results : dict 1469 | results given by the 'run' method 1470 | with_alarm : bool 1471 | (default = True) If True, alarms are plotted. 1472 | Returns 1473 | ---------- 1474 | list 1475 | list of the plots 1476 | 1477 | """ 1478 | x = range(self.data.size) 1479 | K = run_results.keys() 1480 | 1481 | ts_fig, = plt.plot(x,self.data,color=air_force_blue,alpha=0.5) 1482 | fig = [ts_fig] 1483 | 1484 | # if 'upper_thresholds' in K: 1485 | # thup = run_results['upper_thresholds'] 1486 | # uth_fig, = plt.plot(x,thup,color=deep_saffron,lw=2,ls='dashed') 1487 | # fig.append(uth_fig) 1488 | # 1489 | # if 'lower_thresholds' in K: 1490 | # thdown = run_results['lower_thresholds'] 1491 | # lth_fig, = plt.plot(x,thdown,color=deep_saffron,lw=2,ls='dashed') 1492 | # fig.append(lth_fig) 1493 | 1494 | if 'thresholds' in K: 1495 | th = run_results['thresholds'] 1496 | th_fig, = plt.plot(x,th,color=deep_saffron,lw=2,ls='dashed',alpha=0.5) 1497 | fig.append(th_fig) 1498 | 1499 | if with_alarm and ('alarms' in K): 1500 | alarm = run_results['alarms'] 1501 | if len(alarm)>0: 1502 | plt.scatter(alarm,self.data[alarm],color='red') 1503 | 1504 | plt.xlim((0,self.data.size)) 1505 | 1506 | 1507 | return fig 1508 | 1509 | 1510 | 1511 | 1512 | 1513 | 1514 | 1515 | """ 1516 | =========================== DRIFT & DOUBLE BOUNDS ============================= 1517 | """ 1518 | 1519 | 1520 | 1521 | class bidSPOT: 1522 | """ 1523 | This class allows to run DSPOT algorithm on univariate dataset (upper and lower bounds) 1524 | 1525 | Attributes 1526 | ---------- 1527 | proba : float 1528 | Detection level (risk), chosen by the user 1529 | 1530 | depth : int 1531 | Number of observations to compute the moving average 1532 | 1533 | extreme_quantile : float 1534 | current threshold (bound between normal and abnormal events) 1535 | 1536 | data : numpy.array 1537 | stream 1538 | 1539 | init_data : numpy.array 1540 | initial batch of observations (for the calibration/initialization step) 1541 | 1542 | init_threshold : float 1543 | initial threshold computed during the calibration step 1544 | 1545 | peaks : numpy.array 1546 | array of peaks (excesses above the initial threshold) 1547 | 1548 | n : int 1549 | number of observed values 1550 | 1551 | Nt : int 1552 | number of observed peaks 1553 | """ 1554 | def __init__(self, q = 1e-4, depth = 10): 1555 | self.proba = q 1556 | self.data = None 1557 | self.init_data = None 1558 | self.n = 0 1559 | self.depth = depth 1560 | 1561 | nonedict = {'up':None,'down':None} 1562 | 1563 | self.extreme_quantile = dict.copy(nonedict) 1564 | self.init_threshold = dict.copy(nonedict) 1565 | self.peaks = dict.copy(nonedict) 1566 | self.gamma = dict.copy(nonedict) 1567 | self.sigma = dict.copy(nonedict) 1568 | self.Nt = {'up':0,'down':0} 1569 | 1570 | 1571 | def __str__(self): 1572 | s = '' 1573 | s += 'Streaming Peaks-Over-Threshold Object\n' 1574 | s += 'Detection level q = %s\n' % self.proba 1575 | if self.data is not None: 1576 | s += 'Data imported : Yes\n' 1577 | s += '\t initialization : %s values\n' % self.init_data.size 1578 | s += '\t stream : %s values\n' % self.data.size 1579 | else: 1580 | s += 'Data imported : No\n' 1581 | return s 1582 | 1583 | if self.n == 0: 1584 | s += 'Algorithm initialized : No\n' 1585 | else: 1586 | s += 'Algorithm initialized : Yes\n' 1587 | s += '\t initial threshold : %s\n' % self.init_threshold 1588 | 1589 | r = self.n-self.init_data.size 1590 | if r > 0: 1591 | s += 'Algorithm run : Yes\n' 1592 | s += '\t number of observations : %s (%.2f %%)\n' % (r,100*r/self.n) 1593 | s += '\t triggered alarms : %s (%.2f %%)\n' % (len(self.alarm),100*len(self.alarm)/self.n) 1594 | else: 1595 | s += '\t number of peaks : %s\n' % self.Nt 1596 | s += '\t upper extreme quantile : %s\n' % self.extreme_quantile['up'] 1597 | s += '\t lower extreme quantile : %s\n' % self.extreme_quantile['down'] 1598 | s += 'Algorithm run : No\n' 1599 | return s 1600 | 1601 | 1602 | def fit(self,init_data,data): 1603 | """ 1604 | Import data to biDSPOT object 1605 | 1606 | Parameters 1607 | ---------- 1608 | init_data : list, numpy.array or pandas.Series 1609 | initial batch to calibrate the algorithm 1610 | 1611 | data : numpy.array 1612 | data for the run (list, np.array or pd.series) 1613 | 1614 | """ 1615 | if isinstance(data,list): 1616 | self.data = np.array(data) 1617 | elif isinstance(data,np.ndarray): 1618 | self.data = data 1619 | elif isinstance(data,pd.Series): 1620 | self.data = data.values 1621 | else: 1622 | print('This data format (%s) is not supported' % type(data)) 1623 | return 1624 | 1625 | if isinstance(init_data,list): 1626 | self.init_data = np.array(init_data) 1627 | elif isinstance(init_data,np.ndarray): 1628 | self.init_data = init_data 1629 | elif isinstance(init_data,pd.Series): 1630 | self.init_data = init_data.values 1631 | elif isinstance(init_data,int): 1632 | self.init_data = self.data[:init_data] 1633 | self.data = self.data[init_data:] 1634 | elif isinstance(init_data,float) & (init_data<1) & (init_data>0): 1635 | r = int(init_data*data.size) 1636 | self.init_data = self.data[:r] 1637 | self.data = self.data[r:] 1638 | else: 1639 | print('The initial data cannot be set') 1640 | return 1641 | 1642 | def add(self,data): 1643 | """ 1644 | This function allows to append data to the already fitted data 1645 | 1646 | Parameters 1647 | ---------- 1648 | data : list, numpy.array, pandas.Series 1649 | data to append 1650 | """ 1651 | if isinstance(data,list): 1652 | data = np.array(data) 1653 | elif isinstance(data,np.ndarray): 1654 | data = data 1655 | elif isinstance(data,pd.Series): 1656 | data = data.values 1657 | else: 1658 | print('This data format (%s) is not supported' % type(data)) 1659 | return 1660 | 1661 | self.data = np.append(self.data,data) 1662 | return 1663 | 1664 | def initialize(self, verbose = True): 1665 | """ 1666 | Run the calibration (initialization) step 1667 | 1668 | Parameters 1669 | ---------- 1670 | verbose : bool 1671 | (default = True) If True, gives details about the batch initialization 1672 | """ 1673 | n_init = self.init_data.size - self.depth 1674 | 1675 | M = backMean(self.init_data,self.depth) 1676 | T = self.init_data[self.depth:]-M[:-1] # new variable 1677 | 1678 | S = np.sort(T) # we sort T to get the empirical quantile 1679 | self.init_threshold['up'] = S[int(0.98*n_init)] # t is fixed for the whole algorithm 1680 | self.init_threshold['down'] = S[int(0.02*n_init)] # t is fixed for the whole algorithm 1681 | 1682 | # initial peaks 1683 | self.peaks['up'] = T[T>self.init_threshold['up']]-self.init_threshold['up'] 1684 | self.peaks['down'] = -( T[ T0) 1777 | Returns 1778 | ---------- 1779 | float 1780 | log-likelihood of the sample Y to be drawn from a GPD(γ,σ,μ=0) 1781 | """ 1782 | n = Y.size 1783 | if gamma != 0: 1784 | tau = gamma/sigma 1785 | L = -n * log(sigma) - ( 1 + (1/gamma) ) * ( np.log(1+tau*Y) ).sum() 1786 | else: 1787 | L = n * ( 1 + log(Y.mean()) ) 1788 | return L 1789 | 1790 | 1791 | def _grimshaw(self,side,epsilon = 1e-8, n_points = 8): 1792 | """ 1793 | Compute the GPD parameters estimation with the Grimshaw's trick 1794 | 1795 | Parameters 1796 | ---------- 1797 | epsilon : float 1798 | numerical parameter to perform (default : 1e-8) 1799 | n_points : int 1800 | maximum number of candidates for maximum likelihood (default : 10) 1801 | Returns 1802 | ---------- 1803 | gamma_best,sigma_best,ll_best 1804 | gamma estimates, sigma estimates and corresponding log-likelihood 1805 | """ 1806 | def u(s): 1807 | return 1 + np.log(s).mean() 1808 | 1809 | def v(s): 1810 | return np.mean(1/s) 1811 | 1812 | def w(Y,t): 1813 | s = 1+t*Y 1814 | us = u(s) 1815 | vs = v(s) 1816 | return us*vs-1 1817 | 1818 | def jac_w(Y,t): 1819 | s = 1+t*Y 1820 | us = u(s) 1821 | vs = v(s) 1822 | jac_us = (1/t)*(1-vs) 1823 | jac_vs = (1/t)*(-vs+np.mean(1/s**2)) 1824 | return us*jac_vs+vs*jac_us 1825 | 1826 | 1827 | Ym = self.peaks[side].min() 1828 | YM = self.peaks[side].max() 1829 | Ymean = self.peaks[side].mean() 1830 | 1831 | 1832 | a = -1/YM 1833 | if abs(a)<2*epsilon: 1834 | epsilon = abs(a)/n_points 1835 | 1836 | a = a + epsilon 1837 | b = 2*(Ymean-Ym)/(Ymean*Ym) 1838 | c = 2*(Ymean-Ym)/(Ym**2) 1839 | 1840 | # We look for possible roots 1841 | left_zeros = bidSPOT._rootsFinder(lambda t: w(self.peaks[side],t), 1842 | lambda t: jac_w(self.peaks[side],t), 1843 | (a+epsilon,-epsilon), 1844 | n_points,'regular') 1845 | 1846 | right_zeros = bidSPOT._rootsFinder(lambda t: w(self.peaks[side],t), 1847 | lambda t: jac_w(self.peaks[side],t), 1848 | (b,c), 1849 | n_points,'regular') 1850 | 1851 | # all the possible roots 1852 | zeros = np.concatenate((left_zeros,right_zeros)) 1853 | 1854 | # 0 is always a solution so we initialize with it 1855 | gamma_best = 0 1856 | sigma_best = Ymean 1857 | ll_best = bidSPOT._log_likelihood(self.peaks[side],gamma_best,sigma_best) 1858 | 1859 | # we look for better candidates 1860 | for z in zeros: 1861 | gamma = u(1+z*self.peaks[side])-1 1862 | sigma = gamma/z 1863 | ll = bidSPOT._log_likelihood(self.peaks[side],gamma,sigma) 1864 | if ll>ll_best: 1865 | gamma_best = gamma 1866 | sigma_best = sigma 1867 | ll_best = ll 1868 | 1869 | return gamma_best,sigma_best,ll_best 1870 | 1871 | 1872 | 1873 | def _quantile(self,side,gamma,sigma): 1874 | """ 1875 | Compute the quantile at level 1-q for a given side 1876 | 1877 | Parameters 1878 | ---------- 1879 | side : str 1880 | 'up' or 'down' 1881 | gamma : float 1882 | GPD parameter 1883 | sigma : float 1884 | GPD parameter 1885 | Returns 1886 | ---------- 1887 | float 1888 | quantile at level 1-q for the GPD(γ,σ,μ=0) 1889 | """ 1890 | if side == 'up': 1891 | r = self.n * self.proba / self.Nt[side] 1892 | if gamma != 0: 1893 | return self.init_threshold['up'] + (sigma/gamma)*(pow(r,-gamma)-1) 1894 | else: 1895 | return self.init_threshold['up'] - sigma*log(r) 1896 | elif side == 'down': 1897 | r = self.n * self.proba / self.Nt[side] 1898 | if gamma != 0: 1899 | return self.init_threshold['down'] - (sigma/gamma)*(pow(r,-gamma)-1) 1900 | else: 1901 | return self.init_threshold['down'] + sigma*log(r) 1902 | else: 1903 | print('error : the side is not right') 1904 | 1905 | 1906 | def run(self, with_alarm = True, plot = True): 1907 | """ 1908 | Run biDSPOT on the stream 1909 | 1910 | Parameters 1911 | ---------- 1912 | with_alarm : bool 1913 | (default = True) If False, SPOT will adapt the threshold assuming \ 1914 | there is no abnormal values 1915 | Returns 1916 | ---------- 1917 | dict 1918 | keys : 'upper_thresholds', 'lower_thresholds' and 'alarms' 1919 | 1920 | '***-thresholds' contains the extreme quantiles and 'alarms' contains \ 1921 | the indexes of the values which have triggered alarms 1922 | 1923 | """ 1924 | if (self.n>self.init_data.size): 1925 | print('Warning : the algorithm seems to have already been run, you \ 1926 | should initialize before running again') 1927 | return {} 1928 | 1929 | # actual normal window 1930 | W = self.init_data[-self.depth:] 1931 | 1932 | # list of the thresholds 1933 | thup = [] 1934 | thdown = [] 1935 | alarm = [] 1936 | # Loop over the stream 1937 | for i in tqdm.tqdm(range(self.data.size)): 1938 | Mi = W.mean() 1939 | Ni = self.data[i]-Mi 1940 | # If the observed value exceeds the current threshold (alarm case) 1941 | if Ni>self.extreme_quantile['up'] : 1942 | # if we want to alarm, we put it in the alarm list 1943 | if with_alarm: 1944 | alarm.append(i) 1945 | # otherwise we add it in the peaks 1946 | else: 1947 | self.peaks['up'] = np.append(self.peaks['up'],Ni-self.init_threshold['up']) 1948 | self.Nt['up'] += 1 1949 | self.n += 1 1950 | # and we update the thresholds 1951 | 1952 | g,s,l = self._grimshaw('up') 1953 | self.extreme_quantile['up'] = self._quantile('up',g,s) 1954 | W = np.append(W[1:],self.data[i]) 1955 | 1956 | # case where the value exceeds the initial threshold but not the alarm ones 1957 | elif Ni>self.init_threshold['up']: 1958 | # we add it in the peaks 1959 | self.peaks['up'] = np.append(self.peaks['up'],Ni-self.init_threshold['up']) 1960 | self.Nt['up'] += 1 1961 | self.n += 1 1962 | # and we update the thresholds 1963 | g,s,l = self._grimshaw('up') 1964 | self.extreme_quantile['up'] = self._quantile('up',g,s) 1965 | W = np.append(W[1:],self.data[i]) 1966 | 1967 | elif Ni0: 2039 | al_fig = plt.scatter(alarm,self.data[alarm],color='red',alpha=0.5) 2040 | fig.append(al_fig) 2041 | 2042 | plt.xlim((0,self.data.size)) 2043 | 2044 | 2045 | return fig 2046 | --------------------------------------------------------------------------------