├── .idea
    ├── encodings.xml
    ├── misc.xml
    ├── modules.xml
    └── pattern_recognize.iml
├── README.md
├── ele.sh
├── main.py
├── models
    ├── CNN.py
    ├── LSTNet.py
    ├── MHA_Net.py
    ├── MultiHeadAttention.py
    ├── RNN.py
    └── __init__.py
├── solar.sh
├── stock.sh
├── traffic.sh
├── train_eval.py
└── utils.py


/.idea/encodings.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="Encoding" addBOMForNewFiles="with NO BOM" />
4 | </project>


--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="JavaScriptSettings">
4 |     <option name="languageLevel" value="ES6" />
5 |   </component>
6 |   <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6" project-jdk-type="Python SDK" />
7 | </project>


--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="ProjectModuleManager">
4 |     <modules>
5 |       <module fileurl="file://$PROJECT_DIR$/.idea/pattern_recognize.iml" filepath="$PROJECT_DIR$/.idea/pattern_recognize.iml" />
6 |     </modules>
7 |   </component>
8 | </project>


--------------------------------------------------------------------------------
/.idea/pattern_recognize.iml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <module type="PYTHON_MODULE" version="4">
 3 |   <component name="NewModuleRootManager">
 4 |     <content url="file://$MODULE_DIR$" />
 5 |     <orderEntry type="inheritedJdk" />
 6 |     <orderEntry type="sourceFolder" forTests="false" />
 7 |   </component>
 8 |   <component name="TestRunnerService">
 9 |     <option name="PROJECT_TEST_RUNNER" value="Unittests" />
10 |   </component>
11 | </module>


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | 
 2 | ## Deep Learning for Multivariate Time Series Forecasting
 3 | 
 4 | ### Mainly referenced paper
 5 | 
 6 | Modeling Long- and Short-Term Temporal Patterns with Deep Neural Networks.(https://arxiv.org/abs/1703.07015)
 7 | 
 8 | ### Environment 
 9 | 
10 | * Python 3.6+
11 | * Pytorch 1.0+
12 | * numpy
13 | 
14 | ### Example
15 | 
16 | 1. `Exchange Rate dataset: `stock.sh
17 | 2. `Traffic dataset: ` traffic.sh
18 | 3. `Solar-Energy dataset:`solar.sh
19 | 4. ` Electricity usage dataset:` ele.sh 
20 | 
21 | ### Instruction
22 | 
23 | main.py  
24 | * --data DATA                     `location of the data file `
25 | * -h --help                       ` show this help message and exit `
26 | * --model DATA                    ` select the model： LSTNet, CNN, RNN or MHA_Net `
27 | * --window WINDOW                 ` window size (history size) `
28 | * --horizon HORIZON               `forecasting horizon(step) `
29 | * --hidRNN HIDRNN                 `number of RNN hidden units each layer`
30 | * --rnn_layers RNN_LAYERS         ` number of RNN hidden layers`
31 | * --hidCNN HIDCNN                 ` number of CNN hidden units (channels)`
32 | * --CNN_kernel CNN_KERNEL         `the kernel size of the CNN layers`
33 | * --highway_window HIGHWAY_WINDOW `The window size of the highway component `  
34 | * -n_head N_HEAD                  `num of self attention heads                     `
35 | * -d_k D_K                        `self attention key dimension`
36 | * -d_v D_V                        `self attention value dimension`
37 | * --clip CLIP                     `gradient clipping limit `
38 | * --epochs EPOCHS                 `upper epoch limit `
39 | * --batch_size N                  `batch_size`
40 | * --dropout DROPOUT               `dropout applied to layers (0 = no dropout)`
41 | * --seed SEED                     `random seed`
42 | * --log_interval N                `report interval`
43 | * --save SAVE                     `path to save the final model'`
44 | * --cuda CUDA                     `whether to use cuda device`
45 | * --optim OPTIM                   `optimizer method ,default 'adam'`
46 | * --amsgrad AMSGRAD               `whether to use amsgrad`
47 | * --lr LR                         `learning rate`
48 | * --skip SKIP                     `autoregression window size`
49 | * --hidSkip HIDSKIP               `skiphidden states dimension`
50 | * --L1Loss L1LOSS                 `whether  to use l1 loss function`
51 | * --normalize NORMALIZE           `whether  to normalize the data`
52 | * --output_fun OUTPUT_FUN         `relu, tanh or sigmoid `
53 | 
54 | 
55 | 
56 | 
57 | 


--------------------------------------------------------------------------------
/ele.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | python main.py --horizon 6 --data data/electricity.txt --save save/elec.pt --output_fun Linear --model LSTNet --skip 0 --batch_size 32 --hidCNN 50 --hidRNN 50
3 | 


--------------------------------------------------------------------------------
/main.py:
--------------------------------------------------------------------------------
  1 | __author__ = "Guan Song Wang"
  2 | 
  3 | import argparse
  4 | import math
  5 | import time
  6 | import torch
  7 | from torch.utils.data import Dataset, DataLoader
  8 | import torch.nn as nn
  9 | from models import LSTNet,MHA_Net,CNN,RNN
 10 | import importlib
 11 | 
 12 | from utils import *
 13 | from train_eval import train, evaluate, makeOptimizer
 14 | 
 15 | parser = argparse.ArgumentParser(description='PyTorch Time series forecasting')
 16 | parser.add_argument('--data', type=str, required=True,help='location of the data file')
 17 | parser.add_argument('--model', type=str, default='LSTNet', help='')
 18 | parser.add_argument('--window', type=int, default=24 * 7,help='window size')
 19 | parser.add_argument('--horizon', type=int, default=12)
 20 | 
 21 | parser.add_argument('--hidRNN', type=int, default=100, help='number of RNN hidden units each layer')
 22 | parser.add_argument('--rnn_layers', type=int, default=1, help='number of RNN hidden layers')
 23 | 
 24 | parser.add_argument('--hidCNN', type=int, default=100, help='number of CNN hidden units (channels)')
 25 | parser.add_argument('--CNN_kernel', type=int, default=6, help='the kernel size of the CNN layers')
 26 | parser.add_argument('--highway_window', type=int, default=24, help='The window size of the highway component')
 27 | 
 28 | parser.add_argument('-n_head', type=int, default=8)
 29 | parser.add_argument('-d_k', type=int, default=64)
 30 | parser.add_argument('-d_v', type=int, default=64)
 31 | 
 32 | parser.add_argument('--clip', type=float, default=10.,help='gradient clipping')
 33 | parser.add_argument('--epochs', type=int, default=50, help='upper epoch limit')
 34 | parser.add_argument('--batch_size', type=int, default=128, metavar='N', help='batch size')
 35 | parser.add_argument('--dropout', type=float, default=0.2, help='dropout applied to layers (0 = no dropout)')
 36 | parser.add_argument('--seed', type=int, default=54321,help='random seed')
 37 | parser.add_argument('--log_interval', type=int, default=2000, metavar='N',help='report interval')
 38 | parser.add_argument('--save', type=str, default='model/model.pt', help='path to save the final model')
 39 | 
 40 | parser.add_argument('--cuda', type=str, default=True)
 41 | parser.add_argument('--optim', type=str, default='adam')
 42 | parser.add_argument('--amsgrad', type=str, default=True)
 43 | parser.add_argument('--lr', type=float, default=0.001)
 44 | parser.add_argument('--skip', type=float, default=24)
 45 | parser.add_argument('--hidSkip', type=int, default=5)
 46 | parser.add_argument('--L1Loss', type=bool, default=True)
 47 | parser.add_argument('--normalize', type=int, default=2)
 48 | parser.add_argument('--output_fun', type=str, default='sigmoid')
 49 | args = parser.parse_args()
 50 | 
 51 | 
 52 | 
 53 | # Choose device: cpu or gpu
 54 | args.cuda = torch.cuda.is_available()
 55 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 56 | 
 57 | # Reproducibility.
 58 | torch.manual_seed(args.seed)
 59 | if args.cuda:
 60 |     torch.cuda.manual_seed(args.seed)
 61 | 
 62 | # Load data
 63 | Data = Data_utility(args.data, 0.6, 0.2, device, args)
 64 | # loss function
 65 | if args.L1Loss:
 66 |     criterion = nn.L1Loss(size_average=False)
 67 | else:
 68 |     criterion = nn.MSELoss(size_average=False)
 69 | evaluateL2 = nn.MSELoss(size_average=False)
 70 | evaluateL1 = nn.L1Loss(size_average=False)
 71 | if args.cuda:
 72 |     criterion = criterion.cuda()
 73 |     evaluateL1 = evaluateL1.cuda()
 74 |     evaluateL2 = evaluateL2.cuda()
 75 | 
 76 | # Select model
 77 | model = eval(args.model).Model(args, Data)
 78 | train_method = train
 79 | eval_method = evaluate
 80 | nParams = sum([p.nelement() for p in model.parameters()])
 81 | print('number of parameters: %d' % nParams)
 82 | if args.cuda:
 83 |     model = nn.DataParallel(model)
 84 | 
 85 | 
 86 | best_val = 10000000
 87 | 
 88 | optim = makeOptimizer(model.parameters(), args)
 89 | 
 90 | # While training you can press Ctrl + C to stop it.
 91 | try:
 92 |     print('Training start')
 93 |     for epoch in range(1, args.epochs + 1):
 94 |         epoch_start_time = time.time()
 95 | 
 96 |         train_loss = train_method(Data, Data.train[0], Data.train[1], model, criterion, optim, args)
 97 | 
 98 |         val_loss, val_rae, val_corr = eval_method(Data, Data.valid[0], Data.valid[1], model, evaluateL2, evaluateL1, args)
 99 |         print('| end of epoch {:3d} | time used: {:5.2f}s | train_loss {:5.4f} | valid rse {:5.4f} | valid rae {:5.4f} | valid corr  {:5.4f}'.
100 |                 format( epoch, (time.time() - epoch_start_time), train_loss, val_loss, val_rae, val_corr))
101 | 
102 |         if val_loss < best_val:
103 |             with open(args.save, 'wb') as f:
104 |                 torch.save(model, f)
105 |             best_val = val_loss
106 |         if epoch % 10 == 0:
107 |             test_acc, test_rae, test_corr = eval_method(Data, Data.test[0], Data.test[1], model, evaluateL2, evaluateL1,args)
108 |             print("| test rse {:5.4f} | test rae {:5.4f} | test corr {:5.4f}\n".format(test_acc, test_rae, test_corr))
109 | 
110 | except KeyboardInterrupt:
111 |     print('-' * 89)
112 |     print('Exiting from training early')
113 | 
114 | # Load the best saved model.
115 | with open(args.save, 'rb') as f:
116 |     model = torch.load(f)
117 | test_acc, test_rae, test_corr = evaluate(Data, Data.test[0], Data.test[1], model, evaluateL2, evaluateL1,args)
118 | print('Best model performance：')
119 | print("| test rse {:5.4f} | test rae {:5.4f} | test corr {:5.4f}".format(test_acc, test_rae, test_corr))
120 | 


--------------------------------------------------------------------------------
/models/CNN.py:
--------------------------------------------------------------------------------
 1 | __author__ = "Guan Song Wang"
 2 | 
 3 | import torch
 4 | import torch.nn as nn
 5 | import torch.nn.functional as F
 6 | 
 7 | class Model(nn.Module):
 8 |     def __init__(self, args, data):
 9 |         super(Model, self).__init__()
10 |         self.window = args.window
11 |         self.variables = data.m
12 |         self.hw=args.highway_window
13 |         self.conv1 = nn.Conv1d(self.variables, 32, kernel_size=3)
14 |         self.activate1=F.relu
15 |         self.conv2=nn.Conv1d(32,32,kernel_size=3)
16 |         self.maxpool1=nn.MaxPool1d(kernel_size=2)
17 |         self.conv3=nn.Conv1d(32,16,kernel_size=3)
18 |         self.maxpool2=nn.MaxPool1d(kernel_size=2)
19 |         self.linear1=nn.Linear(1280,100)
20 |         self.out=nn.Linear(100,self.variables)
21 |         if (self.hw > 0):
22 |             self.highway = nn.Linear(self.hw, 1)
23 | 
24 |         self.dropout = nn.Dropout(p=args.dropout)
25 |         self.output = None
26 |         if (args.output_fun == 'sigmoid'):
27 |             self.output = F.sigmoid
28 |         if (args.output_fun == 'tanh'):
29 |             self.output = F.tanh
30 | 
31 |     def forward(self, x):
32 |         c = x.permute(0,2,1).contiguous()
33 |         c=self.conv1(c)
34 |         c=self.activate1(c)
35 |         c=self.conv2(c)
36 |         c=self.activate1(c)
37 |         c=self.maxpool1(c)
38 |         c=self.conv3(c)
39 |         c=self.activate1(c)
40 |         c=c.view(c.size(0),c.size(1)*c.size(2))
41 |         c=self.dropout(c)
42 |         c=self.linear1(c)
43 |         c=self.dropout(c)
44 |         out=self.out(c)
45 | 
46 |         if (self.hw > 0):
47 | 
48 |             z = x[:, -self.hw:, :]
49 |             z = z.permute(0, 2, 1).contiguous().view(-1, self.hw)
50 |             z = self.highway(z)
51 |             z = z.view(-1, self.variables)
52 |             out = out + z
53 |         if self.output is not None:
54 |             out=self.output(out)
55 |         return out
56 | 


--------------------------------------------------------------------------------
/models/LSTNet.py:
--------------------------------------------------------------------------------
 1 | __author__ = "Guan Song Wang"
 2 | 
 3 | import torch
 4 | import torch.nn as nn
 5 | import torch.nn.functional as F
 6 | 
 7 | 
 8 | class Model(nn.Module):
 9 |     def __init__(self, args, data):
10 |         super(Model, self).__init__()
11 |         self.P = args.window
12 |         self.m = data.m
13 |         self.hidR = args.hidRNN
14 |         self.hidC = args.hidCNN
15 |         self.hidS = args.hidSkip
16 |         self.Ck = args.CNN_kernel
17 |         self.skip = args.skip
18 | 
19 |         self.hw = args.highway_window
20 |         self.conv1 = nn.Conv2d(1, self.hidC, kernel_size=(self.Ck, self.m))
21 |         self.GRU1 = nn.GRU(self.hidC, self.hidR)
22 |         self.dropout = nn.Dropout(p=args.dropout)
23 |         if (self.skip > 0):
24 |             self.pt = (self.P - self.Ck) / self.skip
25 |             self.GRUskip = nn.GRU(self.hidC, self.hidS)
26 |             self.linear1 = nn.Linear(self.hidR + self.skip * self.hidS, self.m)
27 |         else:
28 |             self.linear1 = nn.Linear(self.hidR, self.m)
29 |         if (self.hw > 0):
30 |             self.highway = nn.Linear(self.hw, 1)
31 |         self.output = None
32 |         if (args.output_fun == 'sigmoid'):
33 |             self.output = F.sigmoid
34 |         if (args.output_fun == 'tanh'):
35 |             self.output = F.tanh
36 | 
37 |     def forward(self, x):
38 |         batch_size = x.size(0)
39 | 
40 |         # CNN
41 |         c = x.view(-1, 1, self.P, self.m)
42 |         c = F.relu(self.conv1(c))
43 |         c = self.dropout(c)
44 |         c = torch.squeeze(c, 3)
45 | 
46 |         # RNN 
47 |         r = c.permute(2, 0, 1).contiguous()
48 |         _, r = self.GRU1(r)
49 | 
50 |         r = self.dropout(torch.squeeze(r, 0))
51 | 
52 | 
53 | 
54 |         # skip-rnn
55 | 
56 |         if (self.skip > 0):
57 |             self.pt=int(self.pt)
58 |             s = c[:, :, int(-self.pt * self.skip):].contiguous()
59 | 
60 |             s = s.view(batch_size, self.hidC, self.pt, self.skip)
61 |             s = s.permute(2, 0, 3, 1).contiguous()
62 |             s = s.view(self.pt, batch_size * self.skip, self.hidC)
63 |             _, s = self.GRUskip(s)
64 |             s = s.view(batch_size, self.skip * self.hidS)
65 |             s = self.dropout(s)
66 |             r = torch.cat((r, s), 1)
67 | 
68 |         res = self.linear1(r)
69 | 
70 |         # highway
71 |         if (self.hw > 0):
72 | 
73 |             z = x[:, -self.hw:, :]
74 |             z = z.permute(0, 2, 1).contiguous().view(-1, self.hw)
75 |             z = self.highway(z)
76 |             z = z.view(-1, self.m)
77 |             res = res + z
78 | 
79 |         if (self.output):
80 |             res = self.output(res)
81 |         return res
82 | 


--------------------------------------------------------------------------------
/models/MHA_Net.py:
--------------------------------------------------------------------------------
 1 | __author__ = "Guan Song Wang"
 2 | 
 3 | import torch
 4 | import torch.nn as nn
 5 | import torch.nn.functional as F
 6 | from models.MultiHeadAttention import MultiHeadAttention
 7 | 
 8 | class Model(nn.Module):
 9 |     def __init__(self, args, data):
10 |         super(Model, self).__init__()
11 |         self.window = args.window
12 |         self.variables = data.m
13 |         self.hidC = args.hidCNN
14 |         self.hidR = args.hidRNN
15 |         self.hw=args.highway_window
16 | 
17 |         self.d_v=args.d_v
18 |         self.d_k=args.d_k
19 |         self.Ck = args.CNN_kernel
20 |         self.GRU = nn.GRU(self.variables, self.hidR, num_layers=args.rnn_layers)
21 |         # self.Conv1 = nn.Conv2d(1, self.hidC, kernel_size=(self.Ck, self.variables))
22 | 
23 |         self.slf_attn = MultiHeadAttention(args.n_head, self.variables, self.d_k,self.d_v , dropout=args.dropout)
24 | 
25 |         self.dropout = nn.Dropout(p=args.dropout)
26 |         self.linear_out=nn.Linear(self.hidR,self.variables)
27 | 
28 |         if (self.hw > 0):
29 |             self.highway = nn.Linear(self.hw, 1)
30 |         self.output = None
31 |         if (args.output_fun == 'sigmoid'):
32 |             self.output = F.sigmoid
33 |         if (args.output_fun == 'tanh'):
34 |             self.output = F.tanh
35 | 
36 | 
37 |     def forward(self, x):
38 |         # r = x.permute(1, 0, 2).contiguous()
39 |         # out, _ = self.GRU1(r)
40 |         # c = out.permute(1, 0, 2).contiguous()
41 |         # c = x.view(-1,1, self.window, self.variables)
42 |         # c = F.relu(self.Conv1(c))
43 |         # c = self.dropout(c)
44 |         # c = torch.squeeze(c, 3)
45 |         # c=c.permute(0,2,1).contiguous()
46 | 
47 |         attn_output, slf_attn=self.slf_attn(x,x,x,mask=None)
48 | 
49 |         r=attn_output.permute(1,0,2).contiguous()
50 |         _,r=self.GRU(r)
51 |         r = self.dropout(torch.squeeze(r[-1:, :, :], 0))
52 |         out = self.linear_out(r)
53 | 
54 |         if (self.hw > 0):
55 | 
56 |             z = x[:, -self.hw:, :]
57 |             z = z.permute(0, 2, 1).contiguous().view(-1, self.hw)
58 |             z = self.highway(z)
59 |             z = z.view(-1, self.variables)
60 |             out = out + z
61 |         if self.output is not None:
62 |             out=self.output(out)
63 |         return out


--------------------------------------------------------------------------------
/models/MultiHeadAttention.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | import torch.nn as nn
  3 | import numpy as np
  4 | import torch.nn.functional as F
  5 | 
  6 | # https://github.com/jadore801120/attention-is-all-you-need-pytorch/blob/master/transformer/SubLayers.py
  7 | 
  8 | class ScaledDotProductAttention(nn.Module):
  9 | 
 10 |     # Scaled Dot-Product Attention
 11 | 
 12 |     def __init__(self, temperature, attn_dropout=0.1):
 13 |         super().__init__()
 14 |         self.temperature = temperature
 15 |         self.dropout = nn.Dropout(attn_dropout)
 16 |         self.softmax = nn.Softmax(dim=2)
 17 | 
 18 |     def forward(self, q, k, v, mask=None):
 19 | 
 20 |         attn = torch.bmm(q, k.transpose(1, 2))
 21 |         attn = attn / self.temperature
 22 | 
 23 |         if mask is not None:
 24 |             attn = attn.masked_fill(mask, -np.inf)
 25 | 
 26 |         attn = self.softmax(attn)
 27 |         attn = self.dropout(attn)
 28 |         output = torch.bmm(attn, v)
 29 | 
 30 |         return output, attn
 31 | 
 32 | 
 33 | class MultiHeadAttention(nn.Module):
 34 |     ''' Multi-Head Attention module '''
 35 | 
 36 |     def __init__(self, n_head, d_model, d_k, d_v, dropout=0.1):
 37 |         super().__init__()
 38 | 
 39 |         self.n_head = n_head
 40 |         self.d_k = d_k
 41 |         self.d_v = d_v
 42 | 
 43 |         self.w_qs = nn.Linear(d_model, n_head * d_k)
 44 |         self.w_ks = nn.Linear(d_model, n_head * d_k)
 45 |         self.w_vs = nn.Linear(d_model, n_head * d_v)
 46 |         nn.init.normal_(self.w_qs.weight, mean=0, std=np.sqrt(2.0 / (d_model + d_k)))
 47 |         nn.init.normal_(self.w_ks.weight, mean=0, std=np.sqrt(2.0 / (d_model + d_k)))
 48 |         nn.init.normal_(self.w_vs.weight, mean=0, std=np.sqrt(2.0 / (d_model + d_v)))
 49 | 
 50 |         self.attention = ScaledDotProductAttention(temperature=np.power(d_k, 0.2))
 51 |         self.layer_norm = nn.LayerNorm(d_model)
 52 | 
 53 |         self.fc = nn.Linear(n_head * d_v, d_model)
 54 |         nn.init.xavier_normal_(self.fc.weight)
 55 | 
 56 |         self.dropout = nn.Dropout(dropout)
 57 | 
 58 | 
 59 |     def forward(self, q, k, v, mask=None):
 60 | 
 61 |         d_k, d_v, n_head = self.d_k, self.d_v, self.n_head
 62 | 
 63 |         sz_b, len_q, _ = q.size()
 64 |         sz_b, len_k, _ = k.size()
 65 |         sz_b, len_v, _ = v.size()
 66 | 
 67 |         residual = q
 68 | 
 69 |         q = self.w_qs(q).view(sz_b, len_q, n_head, d_k)
 70 |         k = self.w_ks(k).view(sz_b, len_k, n_head, d_k)
 71 |         v = self.w_vs(v).view(sz_b, len_v, n_head, d_v)
 72 | 
 73 |         q = q.permute(2, 0, 1, 3).contiguous().view(-1, len_q, d_k) # (n*b) x lq x dk
 74 |         k = k.permute(2, 0, 1, 3).contiguous().view(-1, len_k, d_k) # (n*b) x lk x dk
 75 |         v = v.permute(2, 0, 1, 3).contiguous().view(-1, len_v, d_v) # (n*b) x lv x dv
 76 |         if mask is not None:
 77 |             mask = mask.repeat(n_head, 1, 1) # (n*b) x .. x ..
 78 |         output, attn = self.attention(q, k, v, mask=mask)
 79 | 
 80 |         output = output.view(n_head, sz_b, len_q, d_v)
 81 |         output = output.permute(1, 2, 0, 3).contiguous().view(sz_b, len_q, -1) # b x lq x (n*dv)
 82 | 
 83 |         output = self.dropout(self.fc(output))
 84 |         output = self.layer_norm(output + residual)
 85 | 
 86 |         return output, attn
 87 | 
 88 | class PositionwiseFeedForward(nn.Module):
 89 |     ''' A two-feed-forward-layer module '''
 90 | 
 91 |     def __init__(self, d_in, d_hid, dropout=0.1):
 92 |         super().__init__()
 93 |         self.w_1 = nn.Conv1d(d_in, d_hid, 1) # position-wise
 94 |         self.w_2 = nn.Conv1d(d_hid, d_in, 1) # position-wise
 95 |         self.layer_norm = nn.LayerNorm(d_in)
 96 |         self.dropout = nn.Dropout(dropout)
 97 | 
 98 |     def forward(self, x):
 99 |         residual = x
100 |         output = x.transpose(1, 2)
101 |         output = self.w_2(F.relu(self.w_1(output)))
102 |         output = output.transpose(1, 2)
103 |         output = self.dropout(output)
104 |         output = self.layer_norm(output + residual)
105 |         return output
106 | 


--------------------------------------------------------------------------------
/models/RNN.py:
--------------------------------------------------------------------------------
 1 | __author__ = "Guan Song Wang"
 2 | 
 3 | import torch
 4 | import torch.nn as nn
 5 | import torch.nn.functional as F
 6 | 
 7 | class Model(nn.Module):
 8 |     def __init__(self, args, data):
 9 |         super(Model, self).__init__()
10 |         self.window = args.window
11 |         self.variables = data.m
12 |         self.hw=args.highway_window
13 |         self.activate1=F.relu
14 |         self.hidR=args.hidRNN
15 |         self.rnn1=nn.GRU(self.variables,self.hidR,num_layers=args.rnn_layers)
16 |         self.linear1 = nn.Linear(self.hidR, self.variables)
17 |         # self.linear1=nn.Linear(1280,100)
18 |         # self.out=nn.Linear(100,self.variables)
19 |         if (self.hw > 0):
20 |             self.highway = nn.Linear(self.hw, 1)
21 | 
22 |         self.dropout = nn.Dropout(p=args.dropout)
23 |         self.output = None
24 |         if (args.output_fun == 'sigmoid'):
25 |             self.output = F.sigmoid
26 |         if (args.output_fun == 'tanh'):
27 |             self.output = F.tanh
28 | 
29 |     def forward(self, x):
30 |         r= x.permute(1,0,2).contiguous()
31 |         _,r=self.rnn1(r)
32 |         r=self.dropout(torch.squeeze(r[-1:,:,:], 0))
33 |         out = self.linear1(r)
34 | 
35 | 
36 |         if (self.hw > 0):
37 | 
38 |             z = x[:, -self.hw:, :]
39 |             z = z.permute(0, 2, 1).contiguous().view(-1, self.hw)
40 |             z = self.highway(z)
41 |             z = z.view(-1, self.variables)
42 |             out = out + z
43 |         if self.output is not None:
44 |             out=self.output(out)
45 |         return out
46 | 


--------------------------------------------------------------------------------
/models/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Vsooong/multivariate-prediction/e10cb51dcaa996b71b63f19f679e042741d3807e/models/__init__.py


--------------------------------------------------------------------------------
/solar.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | python main.py --data data/solar_AL.txt --save save/solar_AL.pt --skip 0 --output_fun Linear
3 | 


--------------------------------------------------------------------------------
/stock.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | python main.py --model LSTNet --data data/exchange_rate.txt --save save/exchange_rate.pt --hidCNN 50 --hidRNN 50 --L1Loss False --output_fun None
3 | 


--------------------------------------------------------------------------------
/traffic.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | python main.py --data data/traffic.txt --save save/traffic.pt --hidSkip 10 --batch_size 32
3 | 


--------------------------------------------------------------------------------
/train_eval.py:
--------------------------------------------------------------------------------
 1 | import math
 2 | import torch.optim as optim
 3 | import torch
 4 | import numpy as np
 5 | np.seterr(divide='ignore', invalid='ignore')
 6 | 
 7 | def evaluate(data, X, Y, model, evaluateL2, evaluateL1, args):
 8 |     model.eval()
 9 |     total_loss = 0
10 |     total_loss_l1 = 0
11 |     n_samples = 0
12 |     predict = None
13 |     test = None
14 | 
15 |     for X, Y in data.get_batches(X, Y, args.batch_size, False):
16 |         output = model(X)
17 |         if predict is None:
18 |             predict = output.clone().detach()
19 |             test = Y
20 |         else:
21 |             predict = torch.cat((predict, output.clone().detach()))
22 |             test = torch.cat((test, Y))
23 | 
24 |         scale = data.scale.expand(output.size(0), data.m)
25 |         total_loss += float(evaluateL2(output * scale, Y * scale).data.item())
26 |         total_loss_l1 +=float( evaluateL1(output * scale, Y * scale).data.item())
27 | 
28 |         n_samples += int((output.size(0) * data.m))
29 | 
30 |     rse = math.sqrt(total_loss / n_samples) / data.rse
31 |     rae = (total_loss_l1 / n_samples) / data.rae
32 | 
33 |     predict = predict.data.cpu().numpy()
34 |     Ytest = test.data.cpu().numpy()
35 |     sigma_p = (predict).std(axis=0)
36 |     sigma_g = (Ytest).std(axis=0)
37 |     mean_p = predict.mean(axis=0)
38 |     mean_g = Ytest.mean(axis=0)
39 |     index = (sigma_g != 0)
40 |     correlation = ((predict - mean_p) * (Ytest - mean_g)).mean(axis=0) / (sigma_p * sigma_g)
41 |     correlation = (correlation[index]).mean()
42 |     return rse, rae, correlation
43 | 
44 | 
45 | def train(data, X, Y, model, criterion, optim, args):
46 |     model.train()
47 |     total_loss = 0
48 |     n_samples = 0
49 |     for X, Y in data.get_batches(X, Y, args.batch_size, True):
50 |         optim.zero_grad()
51 |         output = model(X)
52 |         scale = data.scale.expand(output.size(0), data.m)
53 |         loss = criterion(output * scale, Y * scale)
54 |         loss.backward()
55 | 
56 |         torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip)
57 |         optim.step()
58 |         total_loss += loss.data.item()
59 |         n_samples +=int( (output.size(0) * data.m))
60 |     return total_loss / n_samples
61 | 
62 | def makeOptimizer(params, args):
63 |     if args.optim == 'sgd':
64 |         optimizer = optim.SGD(params, lr=args.lr, )
65 |     elif args.optim == 'adagrad':
66 |         optimizer = optim.Adagrad(params, lr=args.lr, )
67 |     elif args.optim == 'adadelta':
68 |         optimizer = optim.Adadelta(params, lr=args.lr, )
69 |     elif args.optim == 'adam':
70 |         optimizer = optim.Adam(params, lr=args.lr, )
71 |     else:
72 |         raise RuntimeError("Invalid optim method: " + args.method)
73 |     return optimizer
74 | 
75 | 


--------------------------------------------------------------------------------
/utils.py:
--------------------------------------------------------------------------------
 1 | import torch
 2 | import numpy as np
 3 | from torch.autograd import Variable
 4 | 
 5 | 
 6 | def normal_std(x):
 7 |     return x.std() * np.sqrt((len(x) - 1.) / (len(x)))
 8 | 
 9 | 
10 | class Data_utility(object):
11 |     # train and valid is the ratio of training set and validation set. test = 1 - train - valid
12 |     def __init__(self, file_name, train, valid, device, args):
13 |         self.device = device
14 |         self.P = args.window
15 |         self.h = args.horizon
16 |         fin = open(file_name)
17 |         self.rawdat = np.loadtxt(fin, delimiter=',')
18 |         self.dat = np.zeros(self.rawdat.shape)
19 |         self.n, self.m = self.dat.shape
20 |         self.scale = np.ones(self.m)
21 |         self._normalized(args.normalize)
22 |         self._split(int(train * self.n), int((train + valid) * self.n), self.n)
23 | 
24 |         self.scale = torch.as_tensor(self.scale, device=device, dtype=torch.float)
25 | 
26 |         tmp = self.test[1] * self.scale.expand(self.test[1].size(0), self.m)
27 | 
28 |         self.scale = Variable(self.scale)
29 | 
30 |         self.rse = normal_std(tmp)
31 |         self.rae = torch.mean(torch.abs(tmp - torch.mean(tmp)))
32 |         fin.close()
33 | 
34 |     def _normalized(self, normalize):
35 |         # normalized by the maximum value of entire matrix.
36 | 
37 |         if (normalize == 0):
38 |             self.dat = self.rawdat
39 | 
40 |         if (normalize == 1):
41 |             self.dat = self.rawdat / np.max(self.rawdat)
42 | 
43 |         # normlized by the maximum value of each row(sensor).
44 |         if (normalize == 2):
45 |             for i in range(self.m):
46 |                 self.scale[i] = np.max(np.abs(self.rawdat[:, i]))
47 |                 self.dat[:, i] = self.rawdat[:, i] / np.max(np.abs(self.rawdat[:, i]))
48 | 
49 |     def _split(self, train, valid, test):
50 | 
51 |         train_set = range(self.P + self.h - 1, train)
52 |         valid_set = range(train, valid)
53 |         test_set = range(valid, self.n)
54 |         self.train = self._batchify(train_set, self.h)
55 |         self.valid = self._batchify(valid_set, self.h)
56 |         self.test = self._batchify(test_set, self.h)
57 | 
58 |     def _batchify(self, idx_set, horizon):
59 | 
60 |         n = len(idx_set)
61 |         X = torch.zeros((n, self.P, self.m), device=self.device)
62 |         Y = torch.zeros((n, self.m), device=self.device)
63 | 
64 |         for i in range(n):
65 |             end = idx_set[i] - self.h + 1
66 |             start = end - self.P
67 |             X[i, :, :] = torch.as_tensor(self.dat[start:end, :], device=self.device)
68 |             Y[i, :] = torch.as_tensor(self.dat[idx_set[i], :], device=self.device)
69 | 
70 |         return [X, Y]
71 | 
72 |     def get_batches(self, inputs, targets, batch_size, shuffle=True):
73 |         length = len(inputs)
74 |         if shuffle:
75 |             index = torch.randperm(length,device=self.device)
76 |         else:
77 |             index = torch.as_tensor(range(length),device=self.device,dtype=torch.long)
78 |         start_idx = 0
79 |         while (start_idx < length):
80 |             end_idx = min(length, start_idx + batch_size)
81 |             excerpt = index[start_idx:end_idx]
82 |             X = inputs[excerpt]
83 |             Y = targets[excerpt]
84 | 
85 |             yield Variable(X), Variable(Y)
86 |             start_idx += batch_size
87 | 


--------------------------------------------------------------------------------