├── DFDGCN ├── basicts │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-39.pyc │ │ └── launcher.cpython-39.pyc │ ├── archs │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ └── __init__.cpython-39.pyc │ │ └── arch_zoo │ │ │ └── dfdgcn_arch │ │ │ ├── __init__.py │ │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-39.pyc │ │ │ ├── dfdgcn_arch.cpython-39.pyc │ │ │ └── gwnet_arch.cpython-39.pyc │ │ │ └── dfdgcn_arch.py │ ├── data │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-39.pyc │ │ │ ├── dataset.cpython-39.pyc │ │ │ ├── registry.cpython-39.pyc │ │ │ └── transform.cpython-39.pyc │ │ ├── dataset.py │ │ ├── registry.py │ │ └── transform.py │ ├── launcher.py │ ├── losses │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-39.pyc │ │ │ └── losses.cpython-39.pyc │ │ └── losses.py │ ├── metrics │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-39.pyc │ │ │ ├── mae.cpython-39.pyc │ │ │ ├── mape.cpython-39.pyc │ │ │ └── rmse.cpython-39.pyc │ │ ├── mae.py │ │ ├── mape.py │ │ └── rmse.py │ ├── runners │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-39.pyc │ │ │ ├── base_runner.cpython-39.pyc │ │ │ └── base_tsf_runner.cpython-39.pyc │ │ ├── base_runner.py │ │ ├── base_tsf_runner.py │ │ └── runner_zoo │ │ │ ├── __pycache__ │ │ │ ├── dcrnn_runner.cpython-39.pyc │ │ │ ├── gts_runner.cpython-39.pyc │ │ │ ├── hi_runner.cpython-39.pyc │ │ │ ├── megacrn_runner.cpython-39.pyc │ │ │ ├── mtgnn_runner.cpython-39.pyc │ │ │ └── simple_tsf_runner.cpython-39.pyc │ │ │ ├── dcrnn_runner.py │ │ │ ├── gts_runner.py │ │ │ ├── hi_runner.py │ │ │ ├── megacrn_runner.py │ │ │ ├── mtgnn_runner.py │ │ │ └── simple_tsf_runner.py │ └── utils │ │ ├── __init__.py │ │ ├── __pycache__ │ │ ├── __init__.cpython-39.pyc │ │ ├── adjacent_matrix_norm.cpython-39.pyc │ │ ├── misc.cpython-39.pyc │ │ └── serialization.cpython-39.pyc │ │ ├── adjacent_matrix_norm.py │ │ ├── misc.py │ │ └── serialization.py ├── examples │ ├── DFDGCN │ │ ├── DFDGCN_METR-LA.py │ │ ├── DFDGCN_PEMS-BAY.py │ │ ├── DFDGCN_PEMS03.py │ │ ├── DFDGCN_PEMS04.py │ │ ├── DFDGCN_PEMS07.py │ │ ├── DFDGCN_PEMS08.py │ │ └── __pycache__ │ │ │ ├── DFDGCN_PEMS08.cpython-39.pyc │ │ │ ├── GWNet_METR-LA.cpython-39.pyc │ │ │ ├── GWNet_PEMS-BAY.cpython-39.pyc │ │ │ ├── GWNet_PEMS03.cpython-39.pyc │ │ │ ├── GWNet_PEMS04.cpython-39.pyc │ │ │ ├── GWNet_PEMS07.cpython-39.pyc │ │ │ └── GWNet_PEMS08.cpython-39.pyc │ ├── datasets │ │ └── readme.txt │ └── run.py └── scripts │ ├── data_preparation │ ├── METR-LA │ │ └── generate_training_data.py │ ├── PEMS-BAY │ │ └── generate_training_data.py │ ├── PEMS03 │ │ ├── __pycache__ │ │ │ └── generate_adj_mx.cpython-39.pyc │ │ ├── generate_adj_mx.py │ │ └── generate_training_data.py │ ├── PEMS04 │ │ ├── __pycache__ │ │ │ └── generate_adj_mx.cpython-39.pyc │ │ ├── generate_adj_mx.py │ │ └── generate_training_data.py │ ├── PEMS07 │ │ ├── generate_adj_mx.py │ │ └── generate_training_data.py │ └── PEMS08 │ │ ├── __pycache__ │ │ └── generate_adj_mx.cpython-39.pyc │ │ ├── generate_adj_mx.py │ │ └── generate_training_data.py │ └── data_visualization │ ├── data_visualization.ipynb │ └── stid_example.ipynb └── README.md /DFDGCN/basicts/__init__.py: -------------------------------------------------------------------------------- 1 | from .launcher import launch_training 2 | 3 | __version__ = "0.1.6" 4 | 5 | __all__ = ["__version__", "launch_training"] 6 | -------------------------------------------------------------------------------- /DFDGCN/basicts/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/__pycache__/launcher.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/__pycache__/launcher.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/archs/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from .arch_zoo.dfdgcn_arch import DFDGCN 3 | 4 | __all__ = ["DFDGCN"] 5 | -------------------------------------------------------------------------------- /DFDGCN/basicts/archs/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/archs/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/archs/arch_zoo/dfdgcn_arch/__init__.py: -------------------------------------------------------------------------------- 1 | from .dfdgcn_arch import DFDGCN 2 | 3 | __all__ = ["DFDGCN"] 4 | -------------------------------------------------------------------------------- /DFDGCN/basicts/archs/arch_zoo/dfdgcn_arch/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/archs/arch_zoo/dfdgcn_arch/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/archs/arch_zoo/dfdgcn_arch/__pycache__/dfdgcn_arch.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/archs/arch_zoo/dfdgcn_arch/__pycache__/dfdgcn_arch.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/archs/arch_zoo/dfdgcn_arch/__pycache__/gwnet_arch.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/archs/arch_zoo/dfdgcn_arch/__pycache__/gwnet_arch.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/archs/arch_zoo/dfdgcn_arch/dfdgcn_arch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | import numpy as np 5 | 6 | 7 | class convt(nn.Module): 8 | def __init__(self): 9 | super(convt, self).__init__() 10 | 11 | def forward(self, x, w): 12 | x = torch.einsum('bne, ek->bnk', (x, w)) 13 | return x.contiguous() 14 | class nconv(nn.Module): 15 | def __init__(self): 16 | super(nconv, self).__init__() 17 | 18 | def forward(self, x, A, dims): 19 | if dims == 2: 20 | x = torch.einsum('ncvl,vw->ncwl', (x, A)) 21 | elif dims == 3: 22 | x = torch.einsum('ncvl,nvw->ncwl', (x, A)) 23 | else: 24 | raise NotImplementedError('DFDGCN not implemented for A of dimension ' + str(dims)) 25 | return x.contiguous() 26 | 27 | class linear(nn.Module): 28 | """Linear layer.""" 29 | 30 | def __init__(self, c_in, c_out): 31 | super(linear, self).__init__() 32 | self.mlp = torch.nn.Conv2d(c_in, c_out, kernel_size=( 33 | 1, 1), padding=(0, 0), stride=(1, 1), bias=True) 34 | 35 | def forward(self, x): 36 | return self.mlp(x) 37 | 38 | 39 | class gcn(nn.Module): 40 | """Graph convolution network.""" 41 | 42 | def __init__(self, c_in, c_out, dropout, support_len=3, order=2): 43 | super(gcn, self).__init__() 44 | self.nconv = nconv() 45 | 46 | self.c_in = c_in 47 | c_in = (order * (support_len + 1) + 1) * self.c_in 48 | self.mlp = linear(c_in, c_out) 49 | self.dropout = dropout 50 | self.order = order 51 | 52 | def forward(self, x, support): 53 | 54 | out = [x] 55 | for a in support: 56 | x1 = self.nconv(x, a.to(x.device), a.dim()) 57 | out.append(x1) 58 | 59 | for k in range(2, self.order + 1): 60 | x2 = self.nconv(x1, a.to(x1.device), a.dim()) 61 | out.append(x2) 62 | x1 = x2 63 | h = torch.cat(out, dim=1) 64 | h = self.mlp(h) 65 | h = F.dropout(h, self.dropout, training=self.training) 66 | return h 67 | 68 | 69 | 70 | def dy_mask_graph(adj, k): 71 | M = [] 72 | for i in range(adj.size(0)): 73 | adp = adj[i] 74 | mask = torch.zeros( adj.size(1),adj.size(2)).to(adj.device) 75 | mask = mask.fill_(float("0")) 76 | s1, t1 = (adp + torch.rand_like(adp) * 0.01).topk(k, 1) 77 | mask = mask.scatter_(1, t1, s1.fill_(1)) 78 | M.append(mask) 79 | mask = torch.stack(M,dim=0) 80 | adj = adj * mask 81 | return adj 82 | 83 | 84 | 85 | def cat(x1,x2): 86 | M = [] 87 | for i in range(x1.size(0)): 88 | x = x1[i] 89 | new_x = torch.cat([x,x2],dim=1) 90 | M.append(new_x) 91 | result = torch.stack(M,dim=0) 92 | return result 93 | 94 | 95 | class DFDGCN(nn.Module): 96 | 97 | def __init__(self, num_nodes, dropout=0.3, supports=None, 98 | gcn_bool=True, addaptadj=True, aptinit=None, 99 | in_dim=2, out_dim=12, residual_channels=32, 100 | dilation_channels=32, skip_channels=256, end_channels=512, 101 | kernel_size=2, blocks=4, layers=2, a=1, seq_len=12, affine=True, fft_emb=10, identity_emb=10, hidden_emb=30, subgraph=20): 102 | super(DFDGCN, self).__init__() 103 | self.dropout = dropout 104 | self.blocks = blocks 105 | self.layers = layers 106 | self.gcn_bool = gcn_bool 107 | self.addaptadj = addaptadj 108 | self.filter_convs = nn.ModuleList() 109 | self.gate_convs = nn.ModuleList() 110 | self.residual_convs = nn.ModuleList() 111 | self.skip_convs = nn.ModuleList() 112 | self.bn = nn.ModuleList() 113 | self.gconv = nn.ModuleList() 114 | self.seq_len = seq_len 115 | self.a = a 116 | 117 | self.start_conv = nn.Conv2d(in_channels=in_dim, 118 | out_channels=residual_channels, 119 | kernel_size=(1, 1)) 120 | 121 | self.supports = supports 122 | self.emb = fft_emb 123 | self.subgraph_size = subgraph 124 | self.identity_emb = identity_emb 125 | self.hidden_emb = hidden_emb 126 | self.fft_len = round(seq_len//2) + 1 127 | self.Ex1 = nn.Parameter(torch.randn(self.fft_len, self.emb), requires_grad=True) 128 | self.Wd = nn.Parameter(torch.randn(num_nodes,self.emb + self.identity_emb + self.seq_len * 2, self.hidden_emb), requires_grad=True) 129 | self.Wxabs = nn.Parameter(torch.randn(self.hidden_emb, self.hidden_emb), requires_grad=True) 130 | 131 | self.mlp = linear(residual_channels * 4,residual_channels) 132 | self.layersnorm = torch.nn.LayerNorm(normalized_shape=[num_nodes,self.hidden_emb], eps=1e-08,elementwise_affine=affine) 133 | self.convt = convt() 134 | 135 | self.node1 = nn.Parameter( 136 | torch.randn(num_nodes, self.identity_emb), requires_grad=True) 137 | self.drop = nn.Dropout(p=dropout) 138 | 139 | self.T_i_D_emb = nn.Parameter( 140 | torch.empty(288, self.seq_len)) 141 | self.D_i_W_emb = nn.Parameter( 142 | torch.empty(7, self.seq_len)) 143 | 144 | receptive_field = 1 145 | self.reset_parameter() 146 | self.supports_len = 0 147 | if not addaptadj: 148 | self.supports_len -= 1 149 | if supports is not None: 150 | self.supports_len += len(supports) 151 | if gcn_bool and addaptadj: 152 | if aptinit is None: 153 | if supports is None: 154 | self.supports = [] 155 | self.nodevec1 = nn.Parameter( 156 | torch.randn(num_nodes, self.emb), requires_grad=True) 157 | self.nodevec2 = nn.Parameter( 158 | torch.randn(self.emb, num_nodes), requires_grad=True) 159 | self.supports_len += 1 160 | else: 161 | if supports is None: 162 | self.supports = [] 163 | m, p, n = torch.svd(aptinit) 164 | initemb1 = torch.mm(m[:, :10], torch.diag(p[:10] ** 0.5)) 165 | initemb2 = torch.mm(torch.diag(p[:10] ** 0.5), n[:, :10].t()) 166 | self.nodevec1 = nn.Parameter(initemb1, requires_grad=True) 167 | self.nodevec2 = nn.Parameter(initemb2, requires_grad=True) 168 | self.supports_len += 1 169 | 170 | for b in range(blocks): 171 | additional_scope = kernel_size - 1 172 | new_dilation = 1 173 | for i in range(layers): 174 | # dilated convolutions 175 | self.filter_convs.append(nn.Conv2d(in_channels=residual_channels, 176 | out_channels=dilation_channels, 177 | kernel_size=(1, kernel_size), dilation=new_dilation)) 178 | 179 | self.gate_convs.append(nn.Conv2d(in_channels=residual_channels, 180 | out_channels=dilation_channels, 181 | kernel_size=(1, kernel_size), dilation=new_dilation)) 182 | 183 | # 1x1 convolution for residual connection 184 | self.residual_convs.append(nn.Conv2d(in_channels=dilation_channels, 185 | out_channels=residual_channels, 186 | kernel_size=(1, 1))) 187 | 188 | # 1x1 convolution for skip connection 189 | self.skip_convs.append(nn.Conv2d(in_channels=dilation_channels, 190 | out_channels=skip_channels, 191 | kernel_size=(1, 1))) 192 | self.bn.append(nn.BatchNorm2d(residual_channels)) 193 | new_dilation *= 2 194 | receptive_field += additional_scope 195 | additional_scope *= 2 196 | if self.gcn_bool: 197 | self.gconv.append( 198 | gcn(dilation_channels, residual_channels, dropout, support_len=self.supports_len)) 199 | self.end_conv_1 = nn.Conv2d(in_channels=skip_channels, 200 | out_channels=end_channels, 201 | kernel_size=(1, 1), 202 | bias=True) 203 | 204 | self.end_conv_2 = nn.Conv2d(in_channels=end_channels, 205 | out_channels=out_dim, 206 | kernel_size=(1, 1), 207 | bias=True) 208 | 209 | self.receptive_field = receptive_field 210 | 211 | def reset_parameter(self): 212 | nn.init.xavier_uniform_(self.T_i_D_emb) 213 | nn.init.xavier_uniform_(self.D_i_W_emb) 214 | 215 | 216 | def forward(self, history_data: torch.Tensor, future_data: torch.Tensor, batch_seen: int, epoch: int, train: bool, **kwargs) -> torch.Tensor: 217 | """Feedforward function of DFDGCN; Based on Graph WaveNet 218 | 219 | Args: 220 | history_data (torch.Tensor): shape [B, L, N, C] 221 | 222 | Graphs: 223 | predefined graphs: two graphs; [2, N, N] : Pre-given graph structure, including in-degree and out-degree graphs 224 | 225 | self-adaptive graph: [N, N] : Self-Adaptively constructed graphs with two learnable parameters 226 | torch.mm(self.nodevec1, self.nodevec2) 227 | nodevec: [N, Emb] 228 | 229 | dynamic frequency domain graph: [B, N, N] : Data-driven graphs constructed with frequency domain information from traffic data 230 | traffic_data : [B, N, L] 231 | frequency domain information : [B, N, L/2.round + 1] ------Embedding ------[B, N, Emb2] 232 | Identity embedding : learnable parameter [N, Emb3] 233 | Time embedding : Week and Day : [N, 7] [N, 24(hour) * 12 (60min / 5min due to sampling)] ------Embedding ------ [N, 2 * Emb4] 234 | Concat frequency domain information + Identity embedding + Time embedding ------Embedding , Activating, Normalization and Dropout 235 | Conv1d to get adjacency matrix 236 | 237 | Returns: 238 | torch.Tensor: [B, L, N, 1] 239 | """ 240 | #num_feat = model_args["num_feat"] 241 | input = history_data.transpose(1, 3).contiguous()[:,0:2,:,:] 242 | data = history_data 243 | 244 | in_len = input.size(3) 245 | if in_len < self.receptive_field: 246 | x = nn.functional.pad( 247 | input, (self.receptive_field-in_len, 0, 0, 0)) 248 | else: 249 | x = input 250 | x = self.start_conv(x) 251 | 252 | skip = 0 253 | if self.gcn_bool and self.addaptadj and self.supports is not None: 254 | 255 | 256 | gwadp = F.softmax( 257 | F.relu(torch.mm(self.nodevec1, self.nodevec2)), dim=1) 258 | 259 | new_supports = self.supports + [gwadp] # pretrained graph in DCRNN and self-adaptive graph in GWNet 260 | 261 | # Construction of dynamic frequency domain graph 262 | xn1 = input[:, 0, :, -self.seq_len:] 263 | 264 | T_D = self.T_i_D_emb[(data[:, :, :, 1] * 288).type(torch.LongTensor)][:, -1, :, :] 265 | D_W = self.D_i_W_emb[(data[:, :, :, 1 + 1]).type(torch.LongTensor)][:, -1, :, :] 266 | 267 | xn1 = torch.fft.rfft(xn1, dim=-1) 268 | xn1 = torch.abs(xn1) 269 | 270 | xn1 = torch.nn.functional.normalize(xn1, p=2.0, dim=1, eps=1e-12, out=None) 271 | xn1 = torch.nn.functional.normalize(xn1, p=2.0, dim=2, eps=1e-12, out=None) * self.a 272 | 273 | 274 | xn1 = torch.matmul(xn1, self.Ex1) 275 | xn1k = cat(xn1, self.node1) 276 | x_n1 = torch.cat([xn1k, T_D, D_W], dim=2) 277 | x1 = torch.bmm(x_n1.permute(1,0,2),self.Wd).permute(1,0,2) 278 | x1 = torch.relu(x1) 279 | x1k = self.layersnorm(x1) 280 | x1k = self.drop(x1k) 281 | adp = self.convt(x1k, self.Wxabs) 282 | adj = torch.bmm(adp, x1.permute(0, 2, 1)) 283 | adp = torch.relu(adj) 284 | adp = dy_mask_graph(adp, self.subgraph_size) 285 | adp = F.softmax(adp, dim=2) 286 | new_supports = new_supports + [adp] 287 | 288 | 289 | 290 | # WaveNet layers 291 | for i in range(self.blocks * self.layers): 292 | 293 | # |----------------------------------------| *residual* 294 | # | | 295 | # | |-- conv -- tanh --| | 296 | # -> dilate -|----| * ----|-- 1x1 -- + --> *input* 297 | # |-- conv -- sigm --| | 298 | # 1x1 299 | # | 300 | # ---------------------------------------> + -------------> *skip* 301 | 302 | 303 | # dilated convolution 304 | residual = x 305 | filter = self.filter_convs[i](residual) 306 | filter = torch.tanh(filter) 307 | gate = self.gate_convs[i](residual) 308 | gate = torch.sigmoid(gate) 309 | x = filter * gate 310 | 311 | # parametrized skip connection 312 | 313 | s = x 314 | 315 | s = self.skip_convs[i](s) 316 | try: 317 | skip = skip[:, :, :, -s.size(3):] 318 | 319 | except: 320 | skip = 0 321 | skip = s + skip 322 | 323 | if self.gcn_bool and self.supports is not None: 324 | if self.addaptadj: 325 | x = self.gconv[i](x, new_supports) 326 | 327 | else: 328 | x = self.gconv[i](x, self.supports) 329 | else: 330 | x = self.residual_convs[i](x) 331 | x = x + residual[:, :, :, -x.size(3):] 332 | 333 | x = self.bn[i](x) 334 | 335 | x = F.relu(skip) 336 | x = F.relu(self.end_conv_1(x)) 337 | x = self.end_conv_2(x) 338 | return x 339 | -------------------------------------------------------------------------------- /DFDGCN/basicts/data/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from easytorch.utils.registry import scan_modules 4 | 5 | from .registry import SCALER_REGISTRY 6 | from .dataset import TimeSeriesForecastingDataset 7 | 8 | __all__ = ["SCALER_REGISTRY", "TimeSeriesForecastingDataset"] 9 | 10 | # fix bugs on Windows systems and on jupyter 11 | project_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 12 | scan_modules(project_dir, __file__, ["__init__.py", "registry.py"]) 13 | -------------------------------------------------------------------------------- /DFDGCN/basicts/data/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/data/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/data/__pycache__/dataset.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/data/__pycache__/dataset.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/data/__pycache__/registry.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/data/__pycache__/registry.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/data/__pycache__/transform.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/data/__pycache__/transform.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/data/dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import torch 4 | from torch.utils.data import Dataset 5 | 6 | from ..utils import load_pkl 7 | 8 | 9 | class TimeSeriesForecastingDataset(Dataset): 10 | """Time series forecasting dataset.""" 11 | 12 | def __init__(self, data_file_path: str, index_file_path: str, mode: str) -> None: 13 | super().__init__() 14 | assert mode in ["train", "valid", "test"], "error mode" 15 | self._check_if_file_exists(data_file_path, index_file_path) 16 | # read raw data (normalized) 17 | data = load_pkl(data_file_path) 18 | processed_data = data["processed_data"] 19 | self.data = torch.from_numpy(processed_data).float() 20 | # read index 21 | self.index = load_pkl(index_file_path)[mode] 22 | 23 | def _check_if_file_exists(self, data_file_path: str, index_file_path: str): 24 | """Check if data file and index file exist. 25 | 26 | Args: 27 | data_file_path (str): data file path 28 | index_file_path (str): index file path 29 | 30 | Raises: 31 | FileNotFoundError: no data file 32 | FileNotFoundError: no index file 33 | """ 34 | 35 | if not os.path.isfile(data_file_path): 36 | raise FileNotFoundError("BasicTS can not find data file {0}".format(data_file_path)) 37 | if not os.path.isfile(index_file_path): 38 | raise FileNotFoundError("BasicTS can not find index file {0}".format(index_file_path)) 39 | 40 | def __getitem__(self, index: int) -> tuple: 41 | """Get a sample. 42 | 43 | Args: 44 | index (int): the iteration index (not the self.index) 45 | 46 | Returns: 47 | tuple: (future_data, history_data), where the shape of each is L x N x C. 48 | """ 49 | 50 | idx = list(self.index[index]) 51 | if isinstance(idx[0], int): 52 | # continuous index 53 | history_data = self.data[idx[0]:idx[1]] 54 | future_data = self.data[idx[1]:idx[2]] 55 | else: 56 | # discontinuous index or custom index 57 | # NOTE: current time $t$ should not included in the index[0] 58 | history_index = idx[0] # list 59 | assert idx[1] not in history_index, "current time t should not included in the idx[0]" 60 | history_index.append(idx[1]) 61 | history_data = self.data[history_index] 62 | future_data = self.data[idx[1], idx[2]] 63 | 64 | return future_data, history_data 65 | 66 | def __len__(self): 67 | """Dataset length 68 | 69 | Returns: 70 | int: dataset length 71 | """ 72 | 73 | return len(self.index) 74 | -------------------------------------------------------------------------------- /DFDGCN/basicts/data/registry.py: -------------------------------------------------------------------------------- 1 | from easytorch.utils.registry import Registry 2 | 3 | SCALER_REGISTRY = Registry("Scaler") 4 | -------------------------------------------------------------------------------- /DFDGCN/basicts/data/transform.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | 3 | import torch 4 | import numpy as np 5 | 6 | from .registry import SCALER_REGISTRY 7 | 8 | 9 | @SCALER_REGISTRY.register() 10 | def standard_transform(data: np.array, output_dir: str, train_index: list, history_seq_len: int, future_seq_len: int, norm_each_channel: int = False) -> np.array: 11 | """Standard normalization. 12 | 13 | Args: 14 | data (np.array): raw time series data. 15 | output_dir (str): output dir path. 16 | train_index (list): train index. 17 | history_seq_len (int): historical sequence length. 18 | future_seq_len (int): future sequence length. 19 | norm_each_channel (bool): whether to normalization each channel. 20 | 21 | Returns: 22 | np.array: normalized raw time series data. 23 | """ 24 | 25 | # data: L, N, C, C=1 26 | data_train = data[:train_index[-1][1], ...] 27 | if norm_each_channel: 28 | mean, std = data_train.mean(axis=0, keepdims=True), data_train.std(axis=0, keepdims=True) 29 | else: 30 | mean, std = data_train[..., 0].mean(), data_train[..., 0].std() 31 | 32 | print("mean (training data):", mean) 33 | print("std (training data):", std) 34 | scaler = {} 35 | scaler["func"] = re_standard_transform.__name__ 36 | scaler["args"] = {"mean": mean, "std": std} 37 | # label to identify the scaler for different settings. 38 | with open(output_dir + "/scaler_in{0}_out{1}.pkl".format(history_seq_len, future_seq_len), "wb") as f: 39 | pickle.dump(scaler, f) 40 | 41 | def normalize(x): 42 | return (x - mean) / std 43 | 44 | data_norm = normalize(data) 45 | return data_norm 46 | 47 | 48 | @SCALER_REGISTRY.register() 49 | def re_standard_transform(data: torch.Tensor, **kwargs) -> torch.Tensor: 50 | """Standard re-transformation. 51 | 52 | Args: 53 | data (torch.Tensor): input data. 54 | 55 | Returns: 56 | torch.Tensor: re-scaled data. 57 | """ 58 | 59 | mean, std = kwargs["mean"], kwargs["std"] 60 | if isinstance(mean, np.ndarray): 61 | mean = torch.from_numpy(mean).type_as(data).to(data.device).unsqueeze(0) 62 | std = torch.from_numpy(std).type_as(data).to(data.device).unsqueeze(0) 63 | data = data * std 64 | data = data + mean 65 | return data 66 | 67 | 68 | @SCALER_REGISTRY.register() 69 | def min_max_transform(data: np.array, output_dir: str, train_index: list, history_seq_len: int, future_seq_len: int) -> np.array: 70 | """Min-max normalization. 71 | 72 | Args: 73 | data (np.array): raw time series data. 74 | output_dir (str): output dir path. 75 | train_index (list): train index. 76 | history_seq_len (int): historical sequence length. 77 | future_seq_len (int): future sequence length. 78 | 79 | Returns: 80 | np.array: normalized raw time series data. 81 | """ 82 | 83 | # L, N, C, C=1 84 | data_train = data[:train_index[-1][1], ...] 85 | 86 | min_value = data_train.min(axis=(0, 1), keepdims=False)[0] 87 | max_value = data_train.max(axis=(0, 1), keepdims=False)[0] 88 | 89 | print("min: (training data)", min_value) 90 | print("max: (training data)", max_value) 91 | scaler = {} 92 | scaler["func"] = re_min_max_transform.__name__ 93 | scaler["args"] = {"min_value": min_value, "max_value": max_value} 94 | # label to identify the scaler for different settings. 95 | # To be fair, only one transformation can be implemented per dataset. 96 | # TODO: Therefore we (for now) do not distinguish between the data produced by the different transformation methods. 97 | with open(output_dir + "/scaler_in{0}_out{1}.pkl".format(history_seq_len, future_seq_len), "wb") as f: 98 | pickle.dump(scaler, f) 99 | 100 | def normalize(x): 101 | # ref: 102 | # https://github.com/guoshnBJTU/ASTGNN/blob/f0f8c2f42f76cc3a03ea26f233de5961c79c9037/lib/utils.py#L17 103 | x = 1. * (x - min_value) / (max_value - min_value) 104 | x = 2. * x - 1. 105 | return x 106 | 107 | data_norm = normalize(data) 108 | return data_norm 109 | 110 | 111 | @SCALER_REGISTRY.register() 112 | def re_min_max_transform(data: torch.Tensor, **kwargs) -> torch.Tensor: 113 | """Standard re-min-max transform. 114 | 115 | Args: 116 | data (torch.Tensor): input data. 117 | 118 | Returns: 119 | torch.Tensor: re-scaled data. 120 | """ 121 | 122 | min_value, max_value = kwargs["min_value"], kwargs["max_value"] 123 | # ref: 124 | # https://github.com/guoshnBJTU/ASTGNN/blob/f0f8c2f42f76cc3a03ea26f233de5961c79c9037/lib/utils.py#L23 125 | data = (data + 1.) / 2. 126 | data = 1. * data * (max_value - min_value) + min_value 127 | return data 128 | -------------------------------------------------------------------------------- /DFDGCN/basicts/launcher.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Union 2 | from packaging import version 3 | 4 | import easytorch 5 | 6 | 7 | def launch_training(cfg: Union[Dict, str], gpus: str = None, node_rank: int = 0): 8 | """Extended easytorch launch_training. 9 | 10 | Args: 11 | cfg (Union[Dict, str]): Easytorch config. 12 | gpus (str): set ``CUDA_VISIBLE_DEVICES`` environment variable. 13 | node_rank (int): Rank of the current node. 14 | """ 15 | 16 | # pre-processing of some possible future features, such as: 17 | # registering model, runners. 18 | # config checking 19 | pass 20 | # launch training based on easytorch 21 | easytorch_version = easytorch.__version__ 22 | if version.parse(easytorch_version) >= version.parse("1.3"): 23 | easytorch.launch_training(cfg=cfg, devices=gpus, node_rank=node_rank) 24 | else: 25 | easytorch.launch_training(cfg=cfg, gpus=gpus, node_rank=node_rank) 26 | -------------------------------------------------------------------------------- /DFDGCN/basicts/losses/__init__.py: -------------------------------------------------------------------------------- 1 | from .losses import l1_loss, l2_loss 2 | from ..metrics import masked_mae, masked_mape, masked_rmse, masked_mse 3 | 4 | __all__ = ["l1_loss", "l2_loss", "masked_mae", "masked_mape", "masked_rmse", "masked_mse"] 5 | -------------------------------------------------------------------------------- /DFDGCN/basicts/losses/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/losses/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/losses/__pycache__/losses.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/losses/__pycache__/losses.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/losses/losses.py: -------------------------------------------------------------------------------- 1 | import torch.nn.functional as F 2 | 3 | from ..utils import check_nan_inf 4 | 5 | 6 | def l1_loss(input_data, target_data, **kwargs): 7 | """unmasked mae.""" 8 | 9 | return F.l1_loss(input_data, target_data) 10 | 11 | 12 | def l2_loss(input_data, target_data, **kwargs): 13 | """unmasked mse""" 14 | 15 | check_nan_inf(input_data) 16 | check_nan_inf(target_data) 17 | return F.mse_loss(input_data, target_data) 18 | -------------------------------------------------------------------------------- /DFDGCN/basicts/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | from .mae import masked_mae 2 | from .mape import masked_mape 3 | from .rmse import masked_rmse, masked_mse 4 | 5 | __all__ = ["masked_mae", "masked_mape", "masked_rmse", "masked_mse"] 6 | -------------------------------------------------------------------------------- /DFDGCN/basicts/metrics/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/metrics/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/metrics/__pycache__/mae.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/metrics/__pycache__/mae.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/metrics/__pycache__/mape.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/metrics/__pycache__/mape.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/metrics/__pycache__/rmse.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/metrics/__pycache__/rmse.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/metrics/mae.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | 4 | 5 | def masked_mae(preds: torch.Tensor, labels: torch.Tensor, null_val: float = np.nan) -> torch.Tensor: 6 | """Masked mean absolute error. 7 | 8 | Args: 9 | preds (torch.Tensor): predicted values 10 | labels (torch.Tensor): labels 11 | null_val (float, optional): null value. Defaults to np.nan. 12 | 13 | Returns: 14 | torch.Tensor: masked mean absolute error 15 | """ 16 | 17 | if np.isnan(null_val): 18 | mask = ~torch.isnan(labels) 19 | else: 20 | eps = 5e-5 21 | mask = ~torch.isclose(labels, torch.tensor(null_val).expand_as(labels).to(labels.device), atol=eps, rtol=0.) 22 | mask = mask.float() 23 | mask /= torch.mean((mask)) 24 | mask = torch.where(torch.isnan(mask), torch.zeros_like(mask), mask) 25 | loss = torch.abs(preds-labels) 26 | loss = loss * mask 27 | loss = torch.where(torch.isnan(loss), torch.zeros_like(loss), loss) 28 | return torch.mean(loss) 29 | -------------------------------------------------------------------------------- /DFDGCN/basicts/metrics/mape.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | 4 | 5 | def masked_mape(preds: torch.Tensor, labels: torch.Tensor, null_val: float = 0.0) -> torch.Tensor: 6 | """Masked mean absolute percentage error. 7 | 8 | Args: 9 | preds (torch.Tensor): predicted values 10 | labels (torch.Tensor): labels 11 | null_val (float, optional): null value. 12 | In the mape metric, null_val is set to 0.0 by all default. 13 | We keep this parameter for consistency, but we do not allow it to be changed. 14 | Zeros in labels will lead to inf in mape. Therefore, null_val is set to 0.0 by default. 15 | 16 | Returns: 17 | torch.Tensor: masked mean absolute percentage error 18 | """ 19 | # we do not allow null_val to be changed 20 | null_val = 0.0 21 | # delete small values to avoid abnormal results 22 | # TODO: support multiple null values 23 | labels = torch.where(torch.abs(labels) < 1e-4, torch.zeros_like(labels), labels) 24 | if np.isnan(null_val): 25 | mask = ~torch.isnan(labels) 26 | else: 27 | eps = 5e-5 28 | mask = ~torch.isclose(labels, torch.tensor(null_val).expand_as(labels).to(labels.device), atol=eps, rtol=0.) 29 | mask = mask.float() 30 | mask /= torch.mean((mask)) 31 | mask = torch.where(torch.isnan(mask), torch.zeros_like(mask), mask) 32 | loss = torch.abs(torch.abs(preds-labels)/labels) 33 | loss = loss * mask 34 | loss = torch.where(torch.isnan(loss), torch.zeros_like(loss), loss) 35 | return torch.mean(loss) 36 | -------------------------------------------------------------------------------- /DFDGCN/basicts/metrics/rmse.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | 4 | 5 | def masked_mse(preds: torch.Tensor, labels: torch.Tensor, null_val: float = np.nan) -> torch.Tensor: 6 | """Masked mean squared error. 7 | 8 | Args: 9 | preds (torch.Tensor): predicted values 10 | labels (torch.Tensor): labels 11 | null_val (float, optional): null value. Defaults to np.nan. 12 | 13 | Returns: 14 | torch.Tensor: masked mean squared error 15 | """ 16 | 17 | if np.isnan(null_val): 18 | mask = ~torch.isnan(labels) 19 | else: 20 | eps = 5e-5 21 | mask = ~torch.isclose(labels, torch.tensor(null_val).expand_as(labels).to(labels.device), atol=eps, rtol=0.) 22 | mask = mask.float() 23 | mask /= torch.mean((mask)) 24 | mask = torch.where(torch.isnan(mask), torch.zeros_like(mask), mask) 25 | loss = (preds-labels)**2 26 | loss = loss * mask 27 | loss = torch.where(torch.isnan(loss), torch.zeros_like(loss), loss) 28 | return torch.mean(loss) 29 | 30 | 31 | def masked_rmse(preds: torch.Tensor, labels: torch.Tensor, null_val: float = np.nan) -> torch.Tensor: 32 | """root mean squared error. 33 | 34 | Args: 35 | preds (torch.Tensor): predicted values 36 | labels (torch.Tensor): labels 37 | null_val (float, optional): null value . Defaults to np.nan. 38 | 39 | Returns: 40 | torch.Tensor: root mean squared error 41 | """ 42 | 43 | return torch.sqrt(masked_mse(preds=preds, labels=labels, null_val=null_val)) 44 | -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/__init__.py: -------------------------------------------------------------------------------- 1 | from .base_tsf_runner import BaseTimeSeriesForecastingRunner 2 | from .runner_zoo.simple_tsf_runner import SimpleTimeSeriesForecastingRunner 3 | from .runner_zoo.dcrnn_runner import DCRNNRunner 4 | from .runner_zoo.mtgnn_runner import MTGNNRunner 5 | from .runner_zoo.gts_runner import GTSRunner 6 | from .runner_zoo.hi_runner import HIRunner 7 | from .runner_zoo.megacrn_runner import MegaCRNRunner 8 | 9 | __all__ = ["BaseTimeSeriesForecastingRunner", 10 | "SimpleTimeSeriesForecastingRunner", 11 | "DCRNNRunner","MTGNNRunner", "GTSRunner", 12 | "HIRunner", "MegaCRNRunner"] 13 | -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/runners/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/__pycache__/base_runner.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/runners/__pycache__/base_runner.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/__pycache__/base_tsf_runner.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/runners/__pycache__/base_tsf_runner.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/base_runner.py: -------------------------------------------------------------------------------- 1 | import time 2 | from typing import Dict 3 | 4 | import setproctitle 5 | import torch 6 | from torch import nn 7 | from torch.utils.data import DataLoader 8 | from easytorch import Runner 9 | from easytorch.utils import master_only 10 | from easytorch.core.data_loader import build_data_loader 11 | 12 | 13 | class BaseRunner(Runner): 14 | """ 15 | An expanded easytorch runner for benchmarking time series models. 16 | - Support test loader and test process. 17 | - Support setup_graph for the models acting like tensorflow. 18 | """ 19 | 20 | def __init__(self, cfg: dict): 21 | """Init 22 | 23 | Args: 24 | cfg (dict): all in one configurations 25 | """ 26 | 27 | super().__init__(cfg) 28 | 29 | # validate every `val_interval` epoch 30 | self.val_interval = cfg["VAL"].get("INTERVAL", 1) 31 | # test every `test_interval` epoch 32 | self.test_interval = cfg["TEST"].get("INTERVAL", 1) 33 | 34 | # declare data loader 35 | self.train_data_loader = None 36 | self.val_data_loader = None 37 | self.test_data_loader = None 38 | 39 | # fit higher easy-torch version 40 | if not hasattr(self,"to_running_device"): 41 | from easytorch.device import to_device 42 | self.to_running_device = to_device 43 | 44 | # set proctitle 45 | proctitle_name = "{0}({1})".format(cfg["MODEL"].get( 46 | "NAME", " "), cfg.get("DATASET_NAME", " ")) 47 | setproctitle.setproctitle("{0}@BasicTS".format(proctitle_name)) 48 | 49 | @staticmethod 50 | def define_model(cfg: Dict) -> nn.Module: 51 | return cfg["MODEL"]["ARCH"](**cfg.MODEL.PARAM) 52 | 53 | def build_train_data_loader(self, cfg: dict) -> DataLoader: 54 | """Support "setup_graph" for the models acting like tensorflow. 55 | 56 | Args: 57 | cfg (dict): all in one configurations 58 | 59 | Returns: 60 | DataLoader: train dataloader 61 | """ 62 | 63 | train_data_loader = super().build_train_data_loader(cfg) 64 | if cfg["TRAIN"].get("SETUP_GRAPH", False): 65 | for data in train_data_loader: 66 | self.setup_graph(data) 67 | break 68 | return train_data_loader 69 | 70 | def setup_graph(self, data: torch.Tensor): 71 | """Setup all parameters and the computation graph. 72 | 73 | Args: 74 | data (torch.Tensor): data necessary for a forward pass 75 | """ 76 | 77 | pass 78 | 79 | def init_training(self, cfg: dict): 80 | """Initialize training and support test dataloader. 81 | 82 | Args: 83 | cfg (dict): config 84 | """ 85 | 86 | super().init_training(cfg) 87 | # init test 88 | if hasattr(cfg, "TEST"): 89 | self.init_test(cfg) 90 | 91 | @master_only 92 | def init_test(self, cfg: dict): 93 | """Initialize test. 94 | 95 | Args: 96 | cfg (dict): config 97 | """ 98 | 99 | self.test_interval = cfg["TEST"].get("INTERVAL", 1) 100 | self.test_data_loader = self.build_test_data_loader(cfg) 101 | self.register_epoch_meter("test_time", "test", "{:.2f} (s)", plt=False) 102 | 103 | def build_test_data_loader(self, cfg: dict) -> DataLoader: 104 | """Build val dataset and dataloader. 105 | Build dataset by calling ```self.build_train_dataset```, 106 | build dataloader by calling ```build_data_loader```. 107 | 108 | Args: 109 | cfg (dict): config 110 | 111 | Returns: 112 | val data loader (DataLoader) 113 | """ 114 | 115 | dataset = self.build_test_dataset(cfg) 116 | return build_data_loader(dataset, cfg["TEST"]["DATA"]) 117 | 118 | @staticmethod 119 | def build_test_dataset(cfg: dict): 120 | """It can be implemented to a build dataset for test. 121 | 122 | Args: 123 | cfg (dict): config 124 | 125 | Returns: 126 | val dataset (Dataset) 127 | """ 128 | 129 | raise NotImplementedError() 130 | 131 | # support test process 132 | def on_epoch_end(self, epoch: int): 133 | """Callback at the end of an epoch. 134 | 135 | Args: 136 | epoch (int): current epoch. 137 | """ 138 | 139 | # print train meters 140 | self.print_epoch_meters("train") 141 | # tensorboard plt meters 142 | self.plt_epoch_meters("train", epoch) 143 | # validate 144 | if self.val_data_loader is not None and epoch % self.val_interval == 0: 145 | self.validate(train_epoch=epoch) 146 | # test 147 | if self.test_data_loader is not None and epoch % self.test_interval == 0: 148 | self.test_process(train_epoch=epoch) 149 | # save model 150 | self.save_model(epoch) 151 | # reset meters 152 | self.reset_epoch_meters() 153 | 154 | @torch.no_grad() 155 | @master_only 156 | def test_process(self, cfg: dict = None, train_epoch: int = None): 157 | """The whole test process. 158 | 159 | Args: 160 | cfg (dict, optional): config 161 | train_epoch (int, optional): current epoch if in training process. 162 | """ 163 | 164 | # init test if not in training process 165 | if train_epoch is None: 166 | self.init_test(cfg) 167 | 168 | self.on_test_start() 169 | 170 | test_start_time = time.time() 171 | self.model.eval() 172 | 173 | # test 174 | self.test() 175 | 176 | test_end_time = time.time() 177 | self.update_epoch_meter("test_time", test_end_time - test_start_time) 178 | # print test meters 179 | self.print_epoch_meters("test") 180 | if train_epoch is not None: 181 | # tensorboard plt meters 182 | self.plt_epoch_meters("test", train_epoch // self.test_interval) 183 | 184 | self.on_test_end() 185 | 186 | @master_only 187 | def on_test_start(self): 188 | """Callback at the start of testing. 189 | """ 190 | 191 | pass 192 | 193 | @master_only 194 | def on_test_end(self): 195 | """Callback at the end of testing. 196 | """ 197 | 198 | pass 199 | 200 | def test(self, train_epoch: int = None): 201 | """It can be implemented to define testing details. 202 | 203 | Args: 204 | train_epoch (int, optional): current epoch if in training process. 205 | """ 206 | 207 | raise NotImplementedError() 208 | -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/base_tsf_runner.py: -------------------------------------------------------------------------------- 1 | import math 2 | import functools 3 | from typing import Tuple, Union, Optional 4 | 5 | import torch 6 | import numpy as np 7 | from easytorch.utils.dist import master_only 8 | 9 | from .base_runner import BaseRunner 10 | from ..data import SCALER_REGISTRY 11 | from ..utils import load_pkl 12 | from ..metrics import masked_mae, masked_mape, masked_rmse 13 | 14 | 15 | class BaseTimeSeriesForecastingRunner(BaseRunner): 16 | """ 17 | Runner for short term multivariate time series forecasting datasets. 18 | Typically, models predict the future 12 time steps based on historical time series. 19 | Features: 20 | - Evaluate at horizon 3, 6, 12, and overall. 21 | - Metrics: MAE, RMSE, MAPE. The best model is the one with the smallest mae at validation. 22 | - Loss: MAE (masked_mae). Allow customization. 23 | - Support curriculum learning. 24 | - Users only need to implement the `forward` function. 25 | """ 26 | 27 | def __init__(self, cfg: dict): 28 | super().__init__(cfg) 29 | self.dataset_name = cfg["DATASET_NAME"] 30 | # different datasets have different null_values, e.g., 0.0 or np.nan. 31 | self.null_val = cfg["TRAIN"].get("NULL_VAL", np.nan) # consist with metric functions 32 | self.dataset_type = cfg["DATASET_TYPE"] 33 | self.evaluate_on_gpu = cfg["TEST"].get("USE_GPU", True) # evaluate on gpu or cpu (gpu is faster but may cause OOM) 34 | 35 | # read scaler for re-normalization 36 | self.scaler = load_pkl("{0}/scaler_in{1}_out{2}.pkl".format(cfg["TRAIN"]["DATA"]["DIR"], cfg["DATASET_INPUT_LEN"], cfg["DATASET_OUTPUT_LEN"])) 37 | # define loss 38 | self.loss = cfg["TRAIN"]["LOSS"] 39 | # define metric 40 | self.metrics = {"MAE": masked_mae, "RMSE": masked_rmse, "MAPE": masked_mape} 41 | # curriculum learning for output. Note that this is different from the CL in Seq2Seq archs. 42 | self.cl_param = cfg.TRAIN.get("CL", None) 43 | if self.cl_param is not None: 44 | self.warm_up_epochs = cfg.TRAIN.CL.get("WARM_EPOCHS", 0) 45 | self.cl_epochs = cfg.TRAIN.CL.get("CL_EPOCHS") 46 | self.prediction_length = cfg.TRAIN.CL.get("PREDICTION_LENGTH") 47 | self.cl_step_size = cfg.TRAIN.CL.get("STEP_SIZE", 1) 48 | # evaluation horizon 49 | self.evaluation_horizons = [_ - 1 for _ in cfg["TEST"].get("EVALUATION_HORIZONS", range(1, 13))] 50 | assert min(self.evaluation_horizons) >= 0, "The horizon should start counting from 0." 51 | 52 | def init_training(self, cfg: dict): 53 | """Initialize training. 54 | 55 | Including loss, training meters, etc. 56 | 57 | Args: 58 | cfg (dict): config 59 | """ 60 | 61 | super().init_training(cfg) 62 | for key, _ in self.metrics.items(): 63 | self.register_epoch_meter("train_"+key, "train", "{:.4f}") 64 | 65 | def init_validation(self, cfg: dict): 66 | """Initialize validation. 67 | 68 | Including validation meters, etc. 69 | 70 | Args: 71 | cfg (dict): config 72 | """ 73 | 74 | super().init_validation(cfg) 75 | for key, _ in self.metrics.items(): 76 | self.register_epoch_meter("val_"+key, "val", "{:.4f}") 77 | 78 | def init_test(self, cfg: dict): 79 | """Initialize test. 80 | 81 | Including test meters, etc. 82 | 83 | Args: 84 | cfg (dict): config 85 | """ 86 | 87 | super().init_test(cfg) 88 | for key, _ in self.metrics.items(): 89 | self.register_epoch_meter("test_"+key, "test", "{:.4f}") 90 | 91 | def build_train_dataset(self, cfg: dict): 92 | """Build MNIST train dataset 93 | 94 | Args: 95 | cfg (dict): config 96 | 97 | Returns: 98 | train dataset (Dataset) 99 | """ 100 | 101 | data_file_path = "{0}/data_in{1}_out{2}.pkl".format(cfg["TRAIN"]["DATA"]["DIR"], cfg["DATASET_INPUT_LEN"], cfg["DATASET_OUTPUT_LEN"]) 102 | index_file_path = "{0}/index_in{1}_out{2}.pkl".format(cfg["TRAIN"]["DATA"]["DIR"], cfg["DATASET_INPUT_LEN"], cfg["DATASET_OUTPUT_LEN"]) 103 | 104 | # build dataset args 105 | dataset_args = cfg.get("DATASET_ARGS", {}) 106 | # three necessary arguments, data file path, corresponding index file path, and mode (train, valid, or test) 107 | dataset_args["data_file_path"] = data_file_path 108 | dataset_args["index_file_path"] = index_file_path 109 | dataset_args["mode"] = "train" 110 | 111 | dataset = cfg["DATASET_CLS"](**dataset_args) 112 | print("train len: {0}".format(len(dataset))) 113 | 114 | batch_size = cfg["TRAIN"]["DATA"]["BATCH_SIZE"] 115 | self.iter_per_epoch = math.ceil(len(dataset) / batch_size) 116 | 117 | return dataset 118 | 119 | @staticmethod 120 | def build_val_dataset(cfg: dict): 121 | """Build MNIST val dataset 122 | 123 | Args: 124 | cfg (dict): config 125 | 126 | Returns: 127 | validation dataset (Dataset) 128 | """ 129 | data_file_path = "{0}/data_in{1}_out{2}.pkl".format(cfg["VAL"]["DATA"]["DIR"], cfg["DATASET_INPUT_LEN"], cfg["DATASET_OUTPUT_LEN"]) 130 | index_file_path = "{0}/index_in{1}_out{2}.pkl".format(cfg["VAL"]["DATA"]["DIR"], cfg["DATASET_INPUT_LEN"], cfg["DATASET_OUTPUT_LEN"]) 131 | 132 | # build dataset args 133 | dataset_args = cfg.get("DATASET_ARGS", {}) 134 | # three necessary arguments, data file path, corresponding index file path, and mode (train, valid, or test) 135 | dataset_args["data_file_path"] = data_file_path 136 | dataset_args["index_file_path"] = index_file_path 137 | dataset_args["mode"] = "valid" 138 | 139 | dataset = cfg["DATASET_CLS"](**dataset_args) 140 | print("val len: {0}".format(len(dataset))) 141 | 142 | return dataset 143 | 144 | @staticmethod 145 | def build_test_dataset(cfg: dict): 146 | """Build MNIST val dataset 147 | 148 | Args: 149 | cfg (dict): config 150 | 151 | Returns: 152 | train dataset (Dataset) 153 | """ 154 | 155 | data_file_path = "{0}/data_in{1}_out{2}.pkl".format(cfg["TEST"]["DATA"]["DIR"], cfg["DATASET_INPUT_LEN"], cfg["DATASET_OUTPUT_LEN"]) 156 | index_file_path = "{0}/index_in{1}_out{2}.pkl".format(cfg["TEST"]["DATA"]["DIR"], cfg["DATASET_INPUT_LEN"], cfg["DATASET_OUTPUT_LEN"]) 157 | 158 | # build dataset args 159 | dataset_args = cfg.get("DATASET_ARGS", {}) 160 | # three necessary arguments, data file path, corresponding index file path, and mode (train, valid, or test) 161 | dataset_args["data_file_path"] = data_file_path 162 | dataset_args["index_file_path"] = index_file_path 163 | dataset_args["mode"] = "test" 164 | 165 | dataset = cfg["DATASET_CLS"](**dataset_args) 166 | print("test len: {0}".format(len(dataset))) 167 | 168 | return dataset 169 | 170 | def curriculum_learning(self, epoch: int = None) -> int: 171 | """Calculate task level in curriculum learning. 172 | 173 | Args: 174 | epoch (int, optional): current epoch if in training process, else None. Defaults to None. 175 | 176 | Returns: 177 | int: task level 178 | """ 179 | 180 | if epoch is None: 181 | return self.prediction_length 182 | epoch -= 1 183 | # generate curriculum length 184 | if epoch < self.warm_up_epochs: 185 | # still warm up 186 | cl_length = self.prediction_length 187 | else: 188 | _ = ((epoch - self.warm_up_epochs) // self.cl_epochs + 1) * self.cl_step_size 189 | cl_length = min(_, self.prediction_length) 190 | return cl_length 191 | 192 | def forward(self, data: tuple, epoch: int = None, iter_num: int = None, train: bool = True, **kwargs) -> tuple: 193 | """Feed forward process for train, val, and test. Note that the outputs are NOT re-scaled. 194 | 195 | Args: 196 | data (tuple): data (future data, history data). [B, L, N, C] for each of them 197 | epoch (int, optional): epoch number. Defaults to None. 198 | iter_num (int, optional): iteration number. Defaults to None. 199 | train (bool, optional): if in the training process. Defaults to True. 200 | 201 | Returns: 202 | tuple: (prediction, real_value). [B, L, N, C] for each of them. 203 | """ 204 | 205 | raise NotImplementedError() 206 | 207 | def metric_forward(self, metric_func, args): 208 | """Computing metrics. 209 | 210 | Args: 211 | metric_func (function, functools.partial): metric function. 212 | args (list): arguments for metrics computation. 213 | """ 214 | 215 | if isinstance(metric_func, functools.partial) and list(metric_func.keywords.keys()) == ["null_val"]: 216 | # support partial(metric_func, null_val = something) 217 | metric_item = metric_func(*args) 218 | elif callable(metric_func): 219 | # is a function 220 | metric_item = metric_func(*args, null_val=self.null_val) 221 | else: 222 | raise TypeError("Unknown metric type: {0}".format(type(metric_func))) 223 | return metric_item 224 | 225 | def train_iters(self, epoch: int, iter_index: int, data: Union[torch.Tensor, Tuple]) -> torch.Tensor: 226 | """Training details. 227 | 228 | Args: 229 | data (Union[torch.Tensor, Tuple]): Data provided by DataLoader 230 | epoch (int): current epoch. 231 | iter_index (int): current iter. 232 | 233 | Returns: 234 | loss (torch.Tensor) 235 | """ 236 | 237 | iter_num = (epoch-1) * self.iter_per_epoch + iter_index 238 | forward_return = list(self.forward(data=data, epoch=epoch, iter_num=iter_num, train=True)) 239 | # re-scale data 240 | prediction_rescaled = SCALER_REGISTRY.get(self.scaler["func"])(forward_return[0], **self.scaler["args"]) 241 | real_value_rescaled = SCALER_REGISTRY.get(self.scaler["func"])(forward_return[1], **self.scaler["args"]) 242 | # loss 243 | if self.cl_param: 244 | cl_length = self.curriculum_learning(epoch=epoch) 245 | forward_return[0] = prediction_rescaled[:, :cl_length, :, :] 246 | forward_return[1] = real_value_rescaled[:, :cl_length, :, :] 247 | else: 248 | forward_return[0] = prediction_rescaled 249 | forward_return[1] = real_value_rescaled 250 | loss = self.metric_forward(self.loss, forward_return) 251 | # metrics 252 | for metric_name, metric_func in self.metrics.items(): 253 | metric_item = self.metric_forward(metric_func, forward_return[:2]) 254 | self.update_epoch_meter("train_"+metric_name, metric_item.item()) 255 | return loss 256 | 257 | def val_iters(self, iter_index: int, data: Union[torch.Tensor, Tuple]): 258 | """Validation details. 259 | 260 | Args: 261 | data (Union[torch.Tensor, Tuple]): Data provided by DataLoader 262 | train_epoch (int): current epoch if in training process. Else None. 263 | iter_index (int): current iter. 264 | """ 265 | 266 | forward_return = self.forward(data=data, epoch=None, iter_num=None, train=False) 267 | # re-scale data 268 | prediction_rescaled = SCALER_REGISTRY.get(self.scaler["func"])(forward_return[0], **self.scaler["args"]) 269 | real_value_rescaled = SCALER_REGISTRY.get(self.scaler["func"])(forward_return[1], **self.scaler["args"]) 270 | # metrics 271 | for metric_name, metric_func in self.metrics.items(): 272 | metric_item = self.metric_forward(metric_func, [prediction_rescaled, real_value_rescaled]) 273 | self.update_epoch_meter("val_"+metric_name, metric_item.item()) 274 | 275 | @torch.no_grad() 276 | @master_only 277 | def test(self): 278 | """Evaluate the model. 279 | 280 | Args: 281 | train_epoch (int, optional): current epoch if in training process. 282 | """ 283 | 284 | # test loop 285 | prediction = [] 286 | real_value = [] 287 | for _, data in enumerate(self.test_data_loader): 288 | forward_return = self.forward(data, epoch=None, iter_num=None, train=False) 289 | prediction.append(forward_return[0]) # preds = forward_return[0] 290 | real_value.append(forward_return[1]) # testy = forward_return[1] 291 | prediction = torch.cat(prediction, dim=0) 292 | real_value = torch.cat(real_value, dim=0) 293 | # re-scale data 294 | prediction = SCALER_REGISTRY.get(self.scaler["func"])( 295 | prediction, **self.scaler["args"]) 296 | real_value = SCALER_REGISTRY.get(self.scaler["func"])( 297 | real_value, **self.scaler["args"]) 298 | # summarize the results. 299 | # test performance of different horizon 300 | for i in self.evaluation_horizons: 301 | # For horizon i, only calculate the metrics **at that time** slice here. 302 | pred = prediction[:, i, :, :] 303 | real = real_value[:, i, :, :] 304 | # metrics 305 | metric_repr = "" 306 | for metric_name, metric_func in self.metrics.items(): 307 | metric_item = self.metric_forward(metric_func, [pred, real]) 308 | metric_repr += ", Test {0}: {1:.4f}".format(metric_name, metric_item.item()) 309 | log = "Evaluate best model on test data for horizon {:d}" + metric_repr 310 | log = log.format(i+1) 311 | self.logger.info(log) 312 | # test performance overall 313 | for metric_name, metric_func in self.metrics.items(): 314 | if self.evaluate_on_gpu: 315 | metric_item = self.metric_forward(metric_func, [prediction, real_value]) 316 | else: 317 | metric_item = self.metric_forward(metric_func, [prediction.detach().cpu(), real_value.detach().cpu()]) 318 | self.update_epoch_meter("test_"+metric_name, metric_item.item()) 319 | 320 | @master_only 321 | def on_validating_end(self, train_epoch: Optional[int]): 322 | """Callback at the end of validating. 323 | 324 | Args: 325 | train_epoch (Optional[int]): current epoch if in training process. 326 | """ 327 | 328 | if train_epoch is not None: 329 | self.save_best_model(train_epoch, "val_MAE", greater_best=False) 330 | -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/runner_zoo/__pycache__/dcrnn_runner.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/runners/runner_zoo/__pycache__/dcrnn_runner.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/runner_zoo/__pycache__/gts_runner.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/runners/runner_zoo/__pycache__/gts_runner.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/runner_zoo/__pycache__/hi_runner.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/runners/runner_zoo/__pycache__/hi_runner.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/runner_zoo/__pycache__/megacrn_runner.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/runners/runner_zoo/__pycache__/megacrn_runner.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/runner_zoo/__pycache__/mtgnn_runner.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/runners/runner_zoo/__pycache__/mtgnn_runner.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/runner_zoo/__pycache__/simple_tsf_runner.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/runners/runner_zoo/__pycache__/simple_tsf_runner.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/runner_zoo/dcrnn_runner.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from ..base_tsf_runner import BaseTimeSeriesForecastingRunner 4 | 5 | 6 | class DCRNNRunner(BaseTimeSeriesForecastingRunner): 7 | """Runner for DCRNN: add setup_graph and teacher forcing.""" 8 | 9 | def __init__(self, cfg: dict): 10 | super().__init__(cfg) 11 | self.forward_features = cfg["MODEL"].get("FORWARD_FEATURES", None) 12 | self.target_features = cfg["MODEL"].get("TARGET_FEATURES", None) 13 | 14 | def setup_graph(self, data): 15 | """The dcrnn official codes act like tensorflow, which create parameters in the first feedforward process.""" 16 | try: 17 | self.train_iters(1, 0, data) 18 | except AttributeError: 19 | pass 20 | 21 | def select_input_features(self, data: torch.Tensor) -> torch.Tensor: 22 | """Select input features and reshape data to fit the target model. 23 | 24 | Args: 25 | data (torch.Tensor): input history data, shape [B, L, N, C]. 26 | 27 | Returns: 28 | torch.Tensor: reshaped data 29 | """ 30 | 31 | # select feature using self.forward_features 32 | if self.forward_features is not None: 33 | data = data[:, :, :, self.forward_features] 34 | return data 35 | 36 | def select_target_features(self, data: torch.Tensor) -> torch.Tensor: 37 | """Select target features and reshape data back to the BasicTS framework 38 | 39 | Args: 40 | data (torch.Tensor): prediction of the model with arbitrary shape. 41 | 42 | Returns: 43 | torch.Tensor: reshaped data with shape [B, L, N, C] 44 | """ 45 | 46 | # select feature using self.target_features 47 | data = data[:, :, :, self.target_features] 48 | return data 49 | 50 | def forward(self, data: tuple, epoch: int = None, iter_num: int = None, train: bool = True) -> tuple: 51 | """Feed forward process for train, val, and test. Note that the outputs are NOT re-scaled. 52 | 53 | Args: 54 | data (tuple): data (future data, history data). [B, L, N, C] for each of them 55 | epoch (int, optional): epoch number. Defaults to None. 56 | iter_num (int, optional): iteration number. Defaults to None. 57 | train (bool, optional): if in the training process. Defaults to True. 58 | 59 | Returns: 60 | tuple: (prediction, real_value) 61 | """ 62 | 63 | # preprocess 64 | future_data, history_data = data 65 | history_data = self.to_running_device(history_data) # B, L, N, C 66 | future_data = self.to_running_device(future_data) # B, L, N, C 67 | batch_size, length, num_nodes, _ = future_data.shape 68 | 69 | history_data = self.select_input_features(history_data) 70 | if train: 71 | # teacher forcing only use the first dimension. 72 | future_data_4_dec = future_data[..., [0]] 73 | else: 74 | future_data_4_dec = None 75 | 76 | # feed forward 77 | prediction_data = self.model(history_data=history_data, future_data=future_data_4_dec, 78 | batch_seen=iter_num if self.model.training else None, epoch=epoch) 79 | assert list(prediction_data.shape)[:3] == [batch_size, length, num_nodes], \ 80 | "error shape of the output, edit the forward function to reshape it to [B, L, N, C]" 81 | # post process 82 | prediction = self.select_target_features(prediction_data) 83 | real_value = self.select_target_features(future_data) 84 | return prediction, real_value 85 | -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/runner_zoo/gts_runner.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from ..base_tsf_runner import BaseTimeSeriesForecastingRunner 4 | 5 | 6 | class GTSRunner(BaseTimeSeriesForecastingRunner): 7 | def __init__(self, cfg: dict): 8 | super().__init__(cfg) 9 | self.forward_features = cfg["MODEL"].get("FORWARD_FEATURES", None) 10 | self.target_features = cfg["MODEL"].get("TARGET_FEATURES", None) 11 | 12 | def setup_graph(self, data): 13 | try: 14 | self.train_iters(1, 0, data) 15 | except AttributeError: 16 | pass 17 | 18 | def select_input_features(self, data: torch.Tensor) -> torch.Tensor: 19 | """Select input features and reshape data to fit the target model. 20 | 21 | Args: 22 | data (torch.Tensor): input history data, shape [B, L, N, C]. 23 | 24 | Returns: 25 | torch.Tensor: reshaped data 26 | """ 27 | 28 | # select feature using self.forward_features 29 | if self.forward_features is not None: 30 | data = data[:, :, :, self.forward_features] 31 | return data 32 | 33 | def select_target_features(self, data: torch.Tensor) -> torch.Tensor: 34 | """Select target features and reshape data back to the BasicTS framework 35 | 36 | Args: 37 | data (torch.Tensor): prediction of the model with arbitrary shape. 38 | 39 | Returns: 40 | torch.Tensor: reshaped data with shape [B, L, N, C] 41 | """ 42 | 43 | # select feature using self.target_features 44 | data = data[:, :, :, self.target_features] 45 | return data 46 | 47 | def forward(self, data: tuple, epoch:int = None, iter_num: int = None, train:bool = True, **kwargs) -> tuple: 48 | """feed forward process for train, val, and test. Note that the outputs are NOT re-scaled. 49 | 50 | Args: 51 | data (tuple): data (future data, history data). [B, L, N, C] for each of them 52 | epoch (int, optional): epoch number. Defaults to None. 53 | iter_num (int, optional): iteration number. Defaults to None. 54 | train (bool, optional): if in the training process. Defaults to True. 55 | 56 | Returns: 57 | tuple: (prediction, real_value) 58 | """ 59 | 60 | # preprocess 61 | future_data, history_data = data 62 | history_data = self.to_running_device(history_data) # B, L, N, C 63 | future_data = self.to_running_device(future_data) # B, L, N, C 64 | batch_size, length, num_nodes, _ = future_data.shape 65 | 66 | history_data = self.select_input_features(history_data) 67 | if train: 68 | # teacher forcing only use the first dimension. 69 | future_data_4_dec = future_data[..., [0]] 70 | else: 71 | future_data_4_dec = None 72 | 73 | # feed forward 74 | prediction_data, pred_adj, prior_adj = self.model(history_data=history_data, future_data=future_data_4_dec, batch_seen=iter_num, epoch=epoch) 75 | assert list(prediction_data.shape)[:3] == [batch_size, length, num_nodes], \ 76 | "error shape of the output, edit the forward function to reshape it to [B, L, N, C]" 77 | # post process 78 | prediction = self.select_target_features(prediction_data) 79 | real_value = self.select_target_features(future_data) 80 | return prediction, real_value, pred_adj, prior_adj 81 | -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/runner_zoo/hi_runner.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from .simple_tsf_runner import SimpleTimeSeriesForecastingRunner 4 | 5 | 6 | class HIRunner(SimpleTimeSeriesForecastingRunner): 7 | 8 | def backward(self, loss: torch.Tensor): 9 | pass 10 | return 11 | -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/runner_zoo/megacrn_runner.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from ..base_tsf_runner import BaseTimeSeriesForecastingRunner 4 | 5 | 6 | class MegaCRNRunner(BaseTimeSeriesForecastingRunner): 7 | def __init__(self, cfg: dict): 8 | super().__init__(cfg) 9 | self.forward_features = cfg["MODEL"].get("FORWARD_FEATURES", None) 10 | self.target_features = cfg["MODEL"].get("TARGET_FEATURES", None) 11 | 12 | def select_input_features(self, data: torch.Tensor) -> torch.Tensor: 13 | """Select input features and reshape data to fit the target model. 14 | 15 | Args: 16 | data (torch.Tensor): input history data, shape [B, L, N, C]. 17 | 18 | Returns: 19 | torch.Tensor: reshaped data 20 | """ 21 | 22 | # select feature using self.forward_features 23 | if self.forward_features is not None: 24 | data = data[:, :, :, self.forward_features] 25 | return data 26 | 27 | def select_target_features(self, data: torch.Tensor) -> torch.Tensor: 28 | """Select target features and reshape data back to the BasicTS framework 29 | 30 | Args: 31 | data (torch.Tensor): prediction of the model with arbitrary shape. 32 | 33 | Returns: 34 | torch.Tensor: reshaped data with shape [B, L, N, C] 35 | """ 36 | 37 | # select feature using self.target_features 38 | data = data[:, :, :, self.target_features] 39 | return data 40 | 41 | def forward(self, data: tuple, epoch:int = None, iter_num: int = None, train:bool = True, **kwargs) -> tuple: 42 | """feed forward process for train, val, and test. Note that the outputs are NOT re-scaled. 43 | 44 | Args: 45 | data (tuple): data (future data, history data). [B, L, N, C] for each of them 46 | epoch (int, optional): epoch number. Defaults to None. 47 | iter_num (int, optional): iteration number. Defaults to None. 48 | train (bool, optional): if in the training process. Defaults to True. 49 | 50 | Returns: 51 | tuple: (prediction, real_value) 52 | """ 53 | 54 | # preprocess 55 | future_data, history_data = data 56 | history_data = self.to_running_device(history_data) # B, L, N, C 57 | future_data = self.to_running_device(future_data) # B, L, N, C 58 | batch_size, length, num_nodes, _ = future_data.shape 59 | 60 | history_data = self.select_input_features(history_data) 61 | future_data = self.select_input_features(future_data) 62 | 63 | # feed forward 64 | prediction_data, h_att, query, pos, neg = self.model(history_data=history_data, future_data=future_data, batch_seen=iter_num, epoch=epoch) 65 | assert list(prediction_data.shape)[:3] == [batch_size, length, num_nodes], \ 66 | "error shape of the output, edit the forward function to reshape it to [B, L, N, C]" 67 | # post process 68 | prediction = self.select_target_features(prediction_data) 69 | real_value = self.select_target_features(future_data) 70 | return prediction, real_value, query, pos, neg 71 | -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/runner_zoo/mtgnn_runner.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple, Union 2 | 3 | import torch 4 | import numpy as np 5 | 6 | from ..base_tsf_runner import BaseTimeSeriesForecastingRunner 7 | 8 | 9 | class MTGNNRunner(BaseTimeSeriesForecastingRunner): 10 | def __init__(self, cfg: dict): 11 | super().__init__(cfg) 12 | self.forward_features = cfg["MODEL"].get("FORWARD_FEATURES", None) 13 | self.target_features = cfg["MODEL"].get("TARGET_FEATURES", None) 14 | # graph training 15 | self.step_size = cfg.TRAIN.CUSTOM.STEP_SIZE 16 | self.num_nodes = cfg.TRAIN.CUSTOM.NUM_NODES 17 | self.num_split = cfg.TRAIN.CUSTOM.NUM_SPLIT 18 | self.perm = None 19 | 20 | def select_input_features(self, data: torch.Tensor) -> torch.Tensor: 21 | """Select input features. 22 | 23 | Args: 24 | data (torch.Tensor): input history data, shape [B, L, N, C] 25 | 26 | Returns: 27 | torch.Tensor: reshaped data 28 | """ 29 | 30 | # select feature using self.forward_features 31 | if self.forward_features is not None: 32 | data = data[:, :, :, self.forward_features] 33 | return data 34 | 35 | def select_target_features(self, data: torch.Tensor) -> torch.Tensor: 36 | """Select target feature 37 | 38 | Args: 39 | data (torch.Tensor): prediction of the model with arbitrary shape. 40 | 41 | Returns: 42 | torch.Tensor: reshaped data with shape [B, L, N, C] 43 | """ 44 | 45 | # select feature using self.target_features 46 | data = data[:, :, :, self.target_features] 47 | return data 48 | 49 | def forward(self, data: tuple, epoch: int = None, iter_num: int = None, train: bool = True, **kwargs) -> tuple: 50 | """Feed forward process for train, val, and test. Note that the outputs are NOT re-scaled. 51 | 52 | Args: 53 | data (tuple): data (future data, history data). [B, L, N, C] for each of them 54 | epoch (int, optional): epoch number. Defaults to None. 55 | iter_num (int, optional): iteration number. Defaults to None. 56 | train (bool, optional): if in the training process. Defaults to True. 57 | 58 | Returns: 59 | tuple: (prediction, real_value). [B, L, N, C] for each of them. 60 | """ 61 | 62 | if train: 63 | future_data, history_data, idx = data 64 | else: 65 | future_data, history_data = data 66 | idx = None 67 | 68 | history_data = self.to_running_device(history_data) # B, L, N, C 69 | future_data = self.to_running_device(future_data) # B, L, N, C 70 | batch_size, seq_len, num_nodes, _ = future_data.shape 71 | 72 | history_data = self.select_input_features(history_data) 73 | 74 | prediction_data = self.model( 75 | history_data=history_data, idx=idx, batch_seen=iter_num, epoch=epoch) # B, L, N, C 76 | assert list(prediction_data.shape)[:3] == [ 77 | batch_size, seq_len, num_nodes], "error shape of the output, edit the forward function to reshape it to [B, L, N, C]" 78 | # post process 79 | prediction = self.select_target_features(prediction_data) 80 | real_value = self.select_target_features(future_data) 81 | return prediction, real_value 82 | 83 | def train_iters(self, epoch: int, iter_index: int, data: Union[torch.Tensor, Tuple]) -> torch.Tensor: 84 | """It must be implement to define training detail. 85 | 86 | If it returns `loss`, the function ```self.backward``` will be called. 87 | 88 | Args: 89 | epoch (int): current epoch. 90 | iter_index (int): current iter. 91 | data (torch.Tensor or tuple): Data provided by DataLoader 92 | 93 | Returns: 94 | loss (torch.Tensor) 95 | """ 96 | 97 | if iter_index % self.step_size == 0: 98 | self.perm = np.random.permutation(range(self.num_nodes)) 99 | num_sub = int(self.num_nodes/self.num_split) 100 | for j in range(self.num_split): 101 | if j != self.num_split-1: 102 | idx = self.perm[j * num_sub:(j + 1) * num_sub] 103 | raise 104 | else: 105 | idx = self.perm[j * num_sub:] 106 | idx = torch.tensor(idx).type(torch.long) 107 | future_data, history_data = data 108 | data = future_data[:, :, idx, :], history_data[:, :, idx, :], idx 109 | loss = super().train_iters(epoch, iter_index, data) 110 | self.backward(loss) 111 | -------------------------------------------------------------------------------- /DFDGCN/basicts/runners/runner_zoo/simple_tsf_runner.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from ..base_tsf_runner import BaseTimeSeriesForecastingRunner 4 | 5 | 6 | class SimpleTimeSeriesForecastingRunner(BaseTimeSeriesForecastingRunner): 7 | """Simple Runner: select forward features and target features. This runner can cover most cases.""" 8 | 9 | def __init__(self, cfg: dict): 10 | super().__init__(cfg) 11 | self.forward_features = cfg["MODEL"].get("FORWARD_FEATURES", None) 12 | self.target_features = cfg["MODEL"].get("TARGET_FEATURES", None) 13 | 14 | def select_input_features(self, data: torch.Tensor) -> torch.Tensor: 15 | """Select input features. 16 | 17 | Args: 18 | data (torch.Tensor): input history data, shape [B, L, N, C] 19 | 20 | Returns: 21 | torch.Tensor: reshaped data 22 | """ 23 | 24 | # select feature using self.forward_features 25 | if self.forward_features is not None: 26 | data = data[:, :, :, self.forward_features] 27 | return data 28 | 29 | def select_target_features(self, data: torch.Tensor) -> torch.Tensor: 30 | """Select target feature. 31 | 32 | Args: 33 | data (torch.Tensor): prediction of the model with arbitrary shape. 34 | 35 | Returns: 36 | torch.Tensor: reshaped data with shape [B, L, N, C] 37 | """ 38 | 39 | # select feature using self.target_features 40 | data = data[:, :, :, self.target_features] 41 | return data 42 | 43 | def forward(self, data: tuple, epoch: int = None, iter_num: int = None, train: bool = True, **kwargs) -> tuple: 44 | """Feed forward process for train, val, and test. Note that the outputs are NOT re-scaled. 45 | 46 | Args: 47 | data (tuple): data (future data, history ata). 48 | epoch (int, optional): epoch number. Defaults to None. 49 | iter_num (int, optional): iteration number. Defaults to None. 50 | train (bool, optional): if in the training process. Defaults to True. 51 | 52 | Returns: 53 | tuple: (prediction, real_value) 54 | """ 55 | 56 | # preprocess 57 | future_data, history_data = data 58 | history_data = self.to_running_device(history_data) # B, L, N, C 59 | future_data = self.to_running_device(future_data) # B, L, N, C 60 | batch_size, length, num_nodes, _ = future_data.shape 61 | 62 | history_data = self.select_input_features(history_data) 63 | future_data_4_dec = self.select_input_features(future_data) 64 | 65 | # curriculum learning 66 | if self.cl_param is None: 67 | prediction_data = self.model(history_data=history_data, future_data=future_data_4_dec, batch_seen=iter_num, epoch=epoch, train=train) 68 | else: 69 | task_level = self.curriculum_learning(epoch) 70 | prediction_data = self.model(history_data=history_data, future_data=future_data_4_dec, batch_seen=iter_num, epoch=epoch, train=train,\ 71 | task_level=task_level) 72 | # feed forward 73 | assert list(prediction_data.shape)[:3] == [batch_size, length, num_nodes], \ 74 | "error shape of the output, edit the forward function to reshape it to [B, L, N, C]" 75 | # post process 76 | prediction = self.select_target_features(prediction_data) 77 | real_value = self.select_target_features(future_data) 78 | return prediction, real_value 79 | -------------------------------------------------------------------------------- /DFDGCN/basicts/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .serialization import load_adj, load_pkl, dump_pkl, load_node2vec_emb 2 | from .misc import clock, check_nan_inf, remove_nan_inf 3 | 4 | __all__ = ["load_adj", "load_pkl", "dump_pkl", "load_node2vec_emb", "clock", "check_nan_inf", "remove_nan_inf"] 5 | -------------------------------------------------------------------------------- /DFDGCN/basicts/utils/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/utils/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/utils/__pycache__/adjacent_matrix_norm.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/utils/__pycache__/adjacent_matrix_norm.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/utils/__pycache__/misc.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/utils/__pycache__/misc.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/utils/__pycache__/serialization.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/basicts/utils/__pycache__/serialization.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/basicts/utils/adjacent_matrix_norm.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.sparse as sp 3 | from scipy.sparse import linalg 4 | 5 | 6 | def calculate_symmetric_normalized_laplacian(adj: np.ndarray) -> np.matrix: 7 | """Calculate yymmetric normalized laplacian. 8 | Assuming unnormalized laplacian matrix is `L = D - A`, 9 | then symmetric normalized laplacian matrix is: 10 | `L^{Sym} = D^-1/2 L D^-1/2 = D^-1/2 (D-A) D^-1/2 = I - D^-1/2 A D^-1/2` 11 | For node `i` and `j` where `i!=j`, L^{sym}_{ij} <=0. 12 | 13 | Args: 14 | adj (np.ndarray): Adjacent matrix A 15 | 16 | Returns: 17 | np.matrix: Symmetric normalized laplacian L^{Sym} 18 | """ 19 | 20 | adj = sp.coo_matrix(adj) 21 | degree = np.array(adj.sum(1)) 22 | # diagonals of D^{-1/2} 23 | degree_inv_sqrt = np.power(degree, -0.5).flatten() 24 | degree_inv_sqrt[np.isinf(degree_inv_sqrt)] = 0. 25 | matrix_degree_inv_sqrt = sp.diags(degree_inv_sqrt) # D^{-1/2} 26 | symmetric_normalized_laplacian = sp.eye( 27 | adj.shape[0]) - matrix_degree_inv_sqrt.dot(adj).dot(matrix_degree_inv_sqrt).tocoo() 28 | return symmetric_normalized_laplacian 29 | 30 | 31 | def calculate_scaled_laplacian(adj: np.ndarray, lambda_max: int = 2, undirected: bool = True) -> np.matrix: 32 | """Re-scaled the eigenvalue to [-1, 1] by scaled the normalized laplacian matrix for chebyshev pol. 33 | According to `2017 ICLR GCN`, the lambda max is set to 2, and the graph is set to undirected. 34 | Note that rescale the laplacian matrix is equal to rescale the eigenvalue matrix. 35 | `L_{scaled} = (2 / lambda_max * L) - I` 36 | 37 | Args: 38 | adj (np.ndarray): Adjacent matrix A 39 | lambda_max (int, optional): Defaults to 2. 40 | undirected (bool, optional): Defaults to True. 41 | 42 | Returns: 43 | np.matrix: The rescaled laplacian matrix. 44 | """ 45 | 46 | if undirected: 47 | adj = np.maximum.reduce([adj, adj.T]) 48 | laplacian_matrix = calculate_symmetric_normalized_laplacian(adj) 49 | if lambda_max is None: # manually cal the max lambda 50 | lambda_max, _ = linalg.eigsh(laplacian_matrix, 1, which='LM') 51 | lambda_max = lambda_max[0] 52 | laplacian_matrix = sp.csr_matrix(laplacian_matrix) 53 | num_nodes, _ = laplacian_matrix.shape 54 | identity_matrix = sp.identity( 55 | num_nodes, format='csr', dtype=laplacian_matrix.dtype) 56 | laplacian_res = (2 / lambda_max * laplacian_matrix) - identity_matrix 57 | return laplacian_res 58 | 59 | 60 | def calculate_symmetric_message_passing_adj(adj: np.ndarray) -> np.matrix: 61 | """Calculate the renormalized message passing adj in `GCN`. 62 | A = A + I 63 | return D^{-1/2} A D^{-1/2} 64 | 65 | Args: 66 | adj (np.ndarray): Adjacent matrix A 67 | 68 | Returns: 69 | np.matrix: Renormalized message passing adj in `GCN`. 70 | """ 71 | 72 | # add self loop 73 | adj = adj + np.diag(np.ones(adj.shape[0], dtype=np.float32)) 74 | # print("calculating the renormalized message passing adj, please ensure that self-loop has added to adj.") 75 | adj = sp.coo_matrix(adj) 76 | row_sum = np.array(adj.sum(1)) 77 | d_inv_sqrt = np.power(row_sum, -0.5).flatten() 78 | d_inv_sqrt[np.isinf(d_inv_sqrt)] = 0. 79 | d_mat_inv_sqrt = sp.diags(d_inv_sqrt) 80 | mp_adj = d_mat_inv_sqrt.dot(adj).transpose().dot( 81 | d_mat_inv_sqrt).astype(np.float32) 82 | return mp_adj 83 | 84 | 85 | def calculate_transition_matrix(adj: np.ndarray) -> np.matrix: 86 | """Calculate the transition matrix `P` proposed in DCRNN and Graph WaveNet. 87 | P = D^{-1}A = A/rowsum(A) 88 | 89 | Args: 90 | adj (np.ndarray): Adjacent matrix A 91 | 92 | Returns: 93 | np.matrix: Transition matrix P 94 | """ 95 | 96 | adj = sp.coo_matrix(adj) 97 | row_sum = np.array(adj.sum(1)).flatten() 98 | d_inv = np.power(row_sum, -1).flatten() 99 | d_inv[np.isinf(d_inv)] = 0. 100 | d_mat = sp.diags(d_inv) 101 | prob_matrix = d_mat.dot(adj).astype(np.float32).todense() 102 | return prob_matrix 103 | -------------------------------------------------------------------------------- /DFDGCN/basicts/utils/misc.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import torch 4 | 5 | 6 | def clock(func): 7 | """clock decorator""" 8 | def clocked(*args, **kw): 9 | """decorator for clock""" 10 | t0 = time.perf_counter() 11 | result = func(*args, **kw) 12 | elapsed = time.perf_counter() - t0 13 | name = func.__name__ 14 | print("%s: %0.8fs..." % (name, elapsed)) 15 | return result 16 | return clocked 17 | 18 | 19 | def check_nan_inf(tensor: torch.Tensor, raise_ex: bool = True) -> tuple: 20 | """check nan and in in tensor 21 | 22 | Args: 23 | tensor (torch.Tensor): Tensor 24 | raise_ex (bool, optional): If raise exceptions. Defaults to True. 25 | 26 | Raises: 27 | Exception: If raise_ex is True and there are nans or infs in tensor, then raise Exception. 28 | 29 | Returns: 30 | dict: {'nan': bool, 'inf': bool} 31 | bool: if exist nan or if 32 | """ 33 | 34 | # nan 35 | nan = torch.any(torch.isnan(tensor)) 36 | # inf 37 | inf = torch.any(torch.isinf(tensor)) 38 | # raise 39 | if raise_ex and (nan or inf): 40 | raise Exception({"nan": nan, "inf": inf}) 41 | return {"nan": nan, "inf": inf}, nan or inf 42 | 43 | 44 | def remove_nan_inf(tensor: torch.Tensor): 45 | """remove nan and inf in tensor 46 | 47 | Args: 48 | tensor (torch.Tensor): input tensor 49 | 50 | Returns: 51 | torch.Tensor: output tensor 52 | """ 53 | 54 | tensor = torch.where(torch.isnan(tensor), torch.zeros_like(tensor), tensor) 55 | tensor = torch.where(torch.isinf(tensor), torch.zeros_like(tensor), tensor) 56 | return tensor 57 | -------------------------------------------------------------------------------- /DFDGCN/basicts/utils/serialization.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import pickle 3 | 4 | import torch 5 | import numpy as np 6 | 7 | from .adjacent_matrix_norm import calculate_scaled_laplacian, calculate_symmetric_normalized_laplacian, calculate_symmetric_message_passing_adj, calculate_transition_matrix 8 | 9 | 10 | def load_pkl(pickle_file: str) -> object: 11 | """Load pickle data. 12 | 13 | Args: 14 | pickle_file (str): file path 15 | 16 | Returns: 17 | object: loaded objected 18 | """ 19 | 20 | try: 21 | #print(os.path.abspath(pickle_file)) 22 | with open(pickle_file, "rb") as f: 23 | 24 | pickle_data = pickle.load(f) 25 | except UnicodeDecodeError: 26 | with open(pickle_file, "rb") as f: 27 | pickle_data = pickle.load(f, encoding="latin1") 28 | except Exception as e: 29 | print("Unable to load data ", pickle_file, ":", e) 30 | raise 31 | return pickle_data 32 | 33 | 34 | def dump_pkl(obj: object, file_path: str): 35 | """Dumplicate pickle data. 36 | 37 | Args: 38 | obj (object): object 39 | file_path (str): file path 40 | """ 41 | 42 | with open(file_path, "wb") as f: 43 | pickle.dump(obj, f) 44 | 45 | 46 | def load_adj(file_path: str, adj_type: str): 47 | """load adjacency matrix. 48 | 49 | Args: 50 | file_path (str): file path 51 | adj_type (str): adjacency matrix type 52 | 53 | Returns: 54 | list of numpy.matrix: list of preproceesed adjacency matrices 55 | np.ndarray: raw adjacency matrix 56 | """ 57 | 58 | try: 59 | # METR and PEMS_BAY 60 | _, _, adj_mx = load_pkl(file_path) 61 | except ValueError: 62 | # PEMS04 63 | adj_mx = load_pkl(file_path) 64 | if adj_type == "scalap": 65 | adj = [calculate_scaled_laplacian(adj_mx).astype(np.float32).todense()] 66 | elif adj_type == "normlap": 67 | adj = [calculate_symmetric_normalized_laplacian( 68 | adj_mx).astype(np.float32).todense()] 69 | elif adj_type == "symnadj": 70 | adj = [calculate_symmetric_message_passing_adj( 71 | adj_mx).astype(np.float32).todense()] 72 | elif adj_type == "transition": 73 | adj = [calculate_transition_matrix(adj_mx).T] 74 | elif adj_type == "doubletransition": 75 | adj = [calculate_transition_matrix(adj_mx).T, calculate_transition_matrix(adj_mx.T).T] 76 | elif adj_type == "identity": 77 | adj = [np.diag(np.ones(adj_mx.shape[0])).astype(np.float32)] 78 | elif adj_type == "original": 79 | adj = [adj_mx] 80 | else: 81 | error = 0 82 | assert error, "adj type not defined" 83 | return adj, adj_mx 84 | 85 | 86 | def load_node2vec_emb(file_path: str) -> torch.Tensor: 87 | """load node2vec embedding 88 | 89 | Args: 90 | file_path (str): file path 91 | 92 | Returns: 93 | torch.Tensor: node2vec embedding 94 | """ 95 | 96 | # spatial embedding 97 | with open(file_path, mode="r") as f: 98 | lines = f.readlines() 99 | temp = lines[0].split(" ") 100 | num_vertex, dims = int(temp[0]), int(temp[1]) 101 | spatial_embeddings = torch.zeros((num_vertex, dims), dtype=torch.float32) 102 | for line in lines[1:]: 103 | temp = line.split(" ") 104 | index = int(temp[0]) 105 | spatial_embeddings[index] = torch.Tensor([float(ch) for ch in temp[1:]]) 106 | return spatial_embeddings 107 | -------------------------------------------------------------------------------- /DFDGCN/examples/DFDGCN/DFDGCN_METR-LA.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | # TODO: remove it when basicts can be installed by pip 5 | sys.path.append(os.path.abspath(__file__ + "/../../..")) 6 | import torch 7 | from easydict import EasyDict 8 | from basicts.archs import DFDGCN 9 | from basicts.runners import SimpleTimeSeriesForecastingRunner 10 | from basicts.data import TimeSeriesForecastingDataset 11 | from basicts.losses import masked_mae 12 | from basicts.utils import load_adj 13 | 14 | 15 | CFG = EasyDict() 16 | 17 | # ================= general ================= # 18 | CFG.DESCRIPTION = "DFDGCN model configuration" 19 | CFG.RUNNER = SimpleTimeSeriesForecastingRunner 20 | CFG.DATASET_CLS = TimeSeriesForecastingDataset 21 | CFG.DATASET_NAME = "METR-LA" 22 | CFG.DATASET_TYPE = "Traffic speed" 23 | CFG.DATASET_INPUT_LEN = 12 24 | CFG.DATASET_OUTPUT_LEN = 12 25 | CFG.GPU_NUM = 1 26 | 27 | # ================= environment ================= # 28 | CFG.ENV = EasyDict() 29 | CFG.ENV.SEED = 1 30 | CFG.ENV.CUDNN = EasyDict() 31 | CFG.ENV.CUDNN.ENABLED = True 32 | 33 | # ================= model ================= # 34 | CFG.MODEL = EasyDict() 35 | CFG.MODEL.NAME = "DFDGCN" 36 | CFG.MODEL.ARCH = GraphWaveNet 37 | adj_mx, _ = load_adj("datasets/" + CFG.DATASET_NAME + 38 | "/adj_mx.pkl", "doubletransition") 39 | CFG.MODEL.PARAM = { 40 | 41 | "num_nodes": 207, 42 | "supports": [torch.tensor(i) for i in adj_mx], 43 | "dropout": 0.3, 44 | "gcn_bool": True, 45 | "addaptadj": True, 46 | "aptinit": None, 47 | "in_dim": 2, 48 | "out_dim": 12, 49 | "residual_channels": 32 , 50 | "dilation_channels": 32, 51 | "skip_channels": 256, 52 | "end_channels": 512, 53 | "kernel_size": 2, 54 | "blocks": 4, 55 | "layers": 2, 56 | "a": 10, 57 | "seq_len": 12, 58 | "affine": False, 59 | "fft_emb": 10, 60 | "identity_emb": 10, 61 | "hidden_emb": 30, 62 | "subgraph": 20 63 | } 64 | CFG.MODEL.FORWARD_FEATURES = [0, 1,2] 65 | CFG.MODEL.TARGET_FEATURES = [0] 66 | 67 | # ================= optim ================= # 68 | CFG.TRAIN = EasyDict() 69 | CFG.TRAIN.LOSS = masked_mae 70 | CFG.TRAIN.OPTIM = EasyDict() 71 | CFG.TRAIN.OPTIM.TYPE = "Adam" 72 | CFG.TRAIN.OPTIM.PARAM = { 73 | "lr": 0.002, 74 | "weight_decay": 1.0e-5, 75 | 'eps':1.0e-8 76 | } 77 | CFG.TRAIN.LR_SCHEDULER = EasyDict() 78 | CFG.TRAIN.LR_SCHEDULER.TYPE = "MultiStepLR" 79 | CFG.TRAIN.LR_SCHEDULER.PARAM = { 80 | "milestones": [1, 50, 75], 81 | "gamma": 0.5 82 | } 83 | 84 | # ================= train ================= # 85 | CFG.TRAIN.CLIP_GRAD_PARAM = { 86 | "max_norm": 5.0 87 | } 88 | CFG.TRAIN.NUM_EPOCHS = 100 89 | CFG.TRAIN.CKPT_SAVE_DIR = os.path.join( 90 | "checkpoints", 91 | "_".join([CFG.MODEL.NAME, str(CFG.TRAIN.NUM_EPOCHS)]) 92 | ) 93 | # train data 94 | CFG.TRAIN.DATA = EasyDict() 95 | CFG.TRAIN.NULL_VAL = 0.0 96 | # read data 97 | CFG.TRAIN.DATA.DIR = "datasets/" + CFG.DATASET_NAME 98 | # dataloader args, optional 99 | CFG.TRAIN.DATA.BATCH_SIZE = 64 100 | CFG.TRAIN.DATA.PREFETCH = False 101 | CFG.TRAIN.DATA.SHUFFLE = True 102 | CFG.TRAIN.DATA.NUM_WORKERS = 2 103 | CFG.TRAIN.DATA.PIN_MEMORY = False 104 | 105 | # ================= validate ================= # 106 | CFG.VAL = EasyDict() 107 | CFG.VAL.INTERVAL = 1 108 | # validating data 109 | CFG.VAL.DATA = EasyDict() 110 | # read data 111 | CFG.VAL.DATA.DIR = "datasets/" + CFG.DATASET_NAME 112 | # dataloader args, optional 113 | CFG.VAL.DATA.BATCH_SIZE = 64 114 | CFG.VAL.DATA.PREFETCH = False 115 | CFG.VAL.DATA.SHUFFLE = False 116 | CFG.VAL.DATA.NUM_WORKERS = 2 117 | CFG.VAL.DATA.PIN_MEMORY = False 118 | 119 | # ================= test ================= # 120 | CFG.TEST = EasyDict() 121 | CFG.TEST.INTERVAL = 1 122 | # test data 123 | CFG.TEST.DATA = EasyDict() 124 | # read data 125 | CFG.TEST.DATA.DIR = "datasets/" + CFG.DATASET_NAME 126 | # dataloader args, optional 127 | CFG.TEST.DATA.BATCH_SIZE = 64 128 | CFG.TEST.DATA.PREFETCH = False 129 | CFG.TEST.DATA.SHUFFLE = False 130 | CFG.TEST.DATA.NUM_WORKERS = 2 131 | CFG.TEST.DATA.PIN_MEMORY = False 132 | -------------------------------------------------------------------------------- /DFDGCN/examples/DFDGCN/DFDGCN_PEMS-BAY.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | # TODO: remove it when basicts can be installed by pip 5 | sys.path.append(os.path.abspath(__file__ + "/../../..")) 6 | import torch 7 | from easydict import EasyDict 8 | from basicts.archs import DFDGCN 9 | from basicts.runners import SimpleTimeSeriesForecastingRunner 10 | from basicts.data import TimeSeriesForecastingDataset 11 | from basicts.losses import masked_mae 12 | from basicts.utils import load_adj 13 | 14 | 15 | CFG = EasyDict() 16 | 17 | # ================= general ================= # 18 | CFG.DESCRIPTION = "DFDGCN model configuration" 19 | CFG.RUNNER = SimpleTimeSeriesForecastingRunner 20 | CFG.DATASET_CLS = TimeSeriesForecastingDataset 21 | CFG.DATASET_NAME = "PEMS-BAY" 22 | CFG.DATASET_TYPE = "Traffic speed" 23 | CFG.DATASET_INPUT_LEN = 12 24 | CFG.DATASET_OUTPUT_LEN = 12 25 | CFG.GPU_NUM = 1 26 | 27 | # ================= environment ================= # 28 | CFG.ENV = EasyDict() 29 | CFG.ENV.SEED = 1 30 | CFG.ENV.CUDNN = EasyDict() 31 | CFG.ENV.CUDNN.ENABLED = True 32 | 33 | # ================= model ================= # 34 | CFG.MODEL = EasyDict() 35 | CFG.MODEL.NAME = "DFDGCN" 36 | CFG.MODEL.ARCH = DFDGCN 37 | adj_mx, _ = load_adj("datasets/" + CFG.DATASET_NAME + 38 | "/adj_mx.pkl", "doubletransition") 39 | CFG.MODEL.PARAM = { 40 | 41 | 42 | "num_nodes": 325, 43 | "supports": [torch.tensor(i) for i in adj_mx], 44 | "dropout": 0.3, 45 | "gcn_bool": True, 46 | "addaptadj": True, 47 | "aptinit": None, 48 | "in_dim": 2, 49 | "out_dim": 12, 50 | "residual_channels": 32, 51 | "dilation_channels": 32, 52 | "skip_channels": 256, 53 | "end_channels": 512, 54 | "kernel_size": 2, 55 | "blocks": 4, 56 | "layers": 2, 57 | "a": 1, 58 | "seq_len": 12, 59 | "affine": True, 60 | "fft_emb": 10, 61 | "identity_emb": 10, 62 | "hidden_emb": 30, 63 | "subgraph": 20 64 | } 65 | CFG.MODEL.FORWARD_FEATURES = [0, 1,2] 66 | CFG.MODEL.TARGET_FEATURES = [0] 67 | 68 | # ================= optim ================= # 69 | CFG.TRAIN = EasyDict() 70 | CFG.TRAIN.LOSS = masked_mae 71 | CFG.TRAIN.OPTIM = EasyDict() 72 | CFG.TRAIN.OPTIM.TYPE = "Adam" 73 | CFG.TRAIN.OPTIM.PARAM = { 74 | "lr": 0.002, 75 | "weight_decay": 1.0e-5, 76 | 'eps':1.0e-8 77 | } 78 | CFG.TRAIN.LR_SCHEDULER = EasyDict() 79 | CFG.TRAIN.LR_SCHEDULER.TYPE = "MultiStepLR" 80 | CFG.TRAIN.LR_SCHEDULER.PARAM = { 81 | "milestones": [1, 50, 75], 82 | "gamma": 0.5 83 | } 84 | 85 | # ================= train ================= # 86 | CFG.TRAIN.CLIP_GRAD_PARAM = { 87 | "max_norm": 5.0 88 | } 89 | CFG.TRAIN.NUM_EPOCHS = 100 90 | CFG.TRAIN.CKPT_SAVE_DIR = os.path.join( 91 | "checkpoints", 92 | "_".join([CFG.MODEL.NAME, str(CFG.TRAIN.NUM_EPOCHS)]) 93 | ) 94 | # train data 95 | CFG.TRAIN.DATA = EasyDict() 96 | CFG.TRAIN.NULL_VAL = 0.0 97 | # read data 98 | CFG.TRAIN.DATA.DIR = "datasets/" + CFG.DATASET_NAME 99 | # dataloader args, optional 100 | CFG.TRAIN.DATA.BATCH_SIZE = 64 101 | CFG.TRAIN.DATA.PREFETCH = False 102 | CFG.TRAIN.DATA.SHUFFLE = True 103 | CFG.TRAIN.DATA.NUM_WORKERS = 2 104 | CFG.TRAIN.DATA.PIN_MEMORY = False 105 | 106 | # ================= validate ================= # 107 | CFG.VAL = EasyDict() 108 | CFG.VAL.INTERVAL = 1 109 | # validating data 110 | CFG.VAL.DATA = EasyDict() 111 | # read data 112 | CFG.VAL.DATA.DIR = "datasets/" + CFG.DATASET_NAME 113 | # dataloader args, optional 114 | CFG.VAL.DATA.BATCH_SIZE = 64 115 | CFG.VAL.DATA.PREFETCH = False 116 | CFG.VAL.DATA.SHUFFLE = False 117 | CFG.VAL.DATA.NUM_WORKERS = 2 118 | CFG.VAL.DATA.PIN_MEMORY = False 119 | 120 | # ================= test ================= # 121 | CFG.TEST = EasyDict() 122 | CFG.TEST.INTERVAL = 1 123 | # test data 124 | CFG.TEST.DATA = EasyDict() 125 | # read data 126 | CFG.TEST.DATA.DIR = "datasets/" + CFG.DATASET_NAME 127 | # dataloader args, optional 128 | CFG.TEST.DATA.BATCH_SIZE = 64 129 | CFG.TEST.DATA.PREFETCH = False 130 | CFG.TEST.DATA.SHUFFLE = False 131 | CFG.TEST.DATA.NUM_WORKERS = 2 132 | CFG.TEST.DATA.PIN_MEMORY = False 133 | -------------------------------------------------------------------------------- /DFDGCN/examples/DFDGCN/DFDGCN_PEMS03.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | # TODO: remove it when basicts can be installed by pip 5 | sys.path.append(os.path.abspath(__file__ + "/../../..")) 6 | import torch 7 | from easydict import EasyDict 8 | from basicts.archs import DFDGCN 9 | from basicts.runners import SimpleTimeSeriesForecastingRunner 10 | from basicts.data import TimeSeriesForecastingDataset 11 | from basicts.losses import masked_mae 12 | from basicts.utils import load_adj 13 | 14 | 15 | CFG = EasyDict() 16 | 17 | # ================= general ================= # 18 | CFG.DESCRIPTION = "DFDGCN model configuration" 19 | CFG.RUNNER = SimpleTimeSeriesForecastingRunner 20 | CFG.DATASET_CLS = TimeSeriesForecastingDataset 21 | CFG.DATASET_NAME = "PEMS03" 22 | CFG.DATASET_TYPE = "Traffic flow" 23 | CFG.DATASET_INPUT_LEN = 12 24 | CFG.DATASET_OUTPUT_LEN = 12 25 | CFG.GPU_NUM = 1 26 | 27 | # ================= environment ================= # 28 | CFG.ENV = EasyDict() 29 | CFG.ENV.SEED = 1 30 | CFG.ENV.CUDNN = EasyDict() 31 | CFG.ENV.CUDNN.ENABLED = True 32 | 33 | # ================= model ================= # 34 | CFG.MODEL = EasyDict() 35 | CFG.MODEL.NAME = "DFDGCN" 36 | CFG.MODEL.ARCH = DFDGCN 37 | adj_mx, _ = load_adj("datasets/" + CFG.DATASET_NAME + 38 | "/adj_mx.pkl", "doubletransition") 39 | CFG.MODEL.PARAM = { 40 | 41 | "num_nodes": 358, 42 | "supports": [torch.tensor(i) for i in adj_mx], 43 | "dropout": 0.3, 44 | "gcn_bool": True, 45 | "addaptadj": True, 46 | "aptinit": None, 47 | "in_dim": 2, 48 | "out_dim": 12, 49 | "residual_channels": 32, 50 | "dilation_channels": 32, 51 | "skip_channels": 256, 52 | "end_channels": 512, 53 | "kernel_size": 2, 54 | "blocks": 4, 55 | "layers": 2, 56 | "a": 10, 57 | "seq_len": 12, 58 | "affine": False, 59 | "fft_emb": 10, 60 | "identity_emb": 10, 61 | "hidden_emb": 30, 62 | "subgraph": 20 63 | } 64 | CFG.MODEL.FORWARD_FEATURES = [0, 1,2] 65 | CFG.MODEL.TARGET_FEATURES = [0] 66 | 67 | # ================= optim ================= # 68 | CFG.TRAIN = EasyDict() 69 | CFG.TRAIN.LOSS = masked_mae 70 | CFG.TRAIN.OPTIM = EasyDict() 71 | CFG.TRAIN.OPTIM.TYPE = "Adam" 72 | CFG.TRAIN.OPTIM.PARAM = { 73 | "lr": 0.002, 74 | "weight_decay": 0.0001, 75 | } 76 | CFG.TRAIN.LR_SCHEDULER = EasyDict() 77 | CFG.TRAIN.LR_SCHEDULER.TYPE = "MultiStepLR" 78 | CFG.TRAIN.LR_SCHEDULER.PARAM = { 79 | "milestones": [1, 50, 100, 150, 180], 80 | "gamma": 0.5 81 | } 82 | 83 | # ================= train ================= # 84 | CFG.TRAIN.CLIP_GRAD_PARAM = { 85 | "max_norm": 5.0 86 | } 87 | CFG.TRAIN.NUM_EPOCHS = 200 88 | CFG.TRAIN.CKPT_SAVE_DIR = os.path.join( 89 | "checkpoints", 90 | "_".join([CFG.MODEL.NAME, str(CFG.TRAIN.NUM_EPOCHS)]) 91 | ) 92 | # train data 93 | CFG.TRAIN.DATA = EasyDict() 94 | CFG.TRAIN.NULL_VAL = 0.0 95 | # read data 96 | CFG.TRAIN.DATA.DIR = "datasets/" + CFG.DATASET_NAME 97 | # dataloader args, optional 98 | CFG.TRAIN.DATA.BATCH_SIZE = 64 99 | CFG.TRAIN.DATA.PREFETCH = False 100 | CFG.TRAIN.DATA.SHUFFLE = True 101 | CFG.TRAIN.DATA.NUM_WORKERS = 2 102 | CFG.TRAIN.DATA.PIN_MEMORY = False 103 | 104 | # ================= validate ================= # 105 | CFG.VAL = EasyDict() 106 | CFG.VAL.INTERVAL = 1 107 | # validating data 108 | CFG.VAL.DATA = EasyDict() 109 | # read data 110 | CFG.VAL.DATA.DIR = "datasets/" + CFG.DATASET_NAME 111 | # dataloader args, optional 112 | CFG.VAL.DATA.BATCH_SIZE = 64 113 | CFG.VAL.DATA.PREFETCH = False 114 | CFG.VAL.DATA.SHUFFLE = False 115 | CFG.VAL.DATA.NUM_WORKERS = 2 116 | CFG.VAL.DATA.PIN_MEMORY = False 117 | 118 | # ================= test ================= # 119 | CFG.TEST = EasyDict() 120 | CFG.TEST.INTERVAL = 1 121 | # test data 122 | CFG.TEST.DATA = EasyDict() 123 | # read data 124 | CFG.TEST.DATA.DIR = "datasets/" + CFG.DATASET_NAME 125 | # dataloader args, optional 126 | CFG.TEST.DATA.BATCH_SIZE = 64 127 | CFG.TEST.DATA.PREFETCH = False 128 | CFG.TEST.DATA.SHUFFLE = False 129 | CFG.TEST.DATA.NUM_WORKERS = 2 130 | CFG.TEST.DATA.PIN_MEMORY = False 131 | -------------------------------------------------------------------------------- /DFDGCN/examples/DFDGCN/DFDGCN_PEMS04.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | # TODO: remove it when basicts can be installed by pip 5 | sys.path.append(os.path.abspath(__file__ + "/../../..")) 6 | import torch 7 | from easydict import EasyDict 8 | from basicts.archs import DFDGCN 9 | from basicts.runners import SimpleTimeSeriesForecastingRunner 10 | from basicts.data import TimeSeriesForecastingDataset 11 | from basicts.losses import masked_mae 12 | from basicts.utils import load_adj 13 | 14 | 15 | CFG = EasyDict() 16 | 17 | # ================= general ================= # 18 | CFG.DESCRIPTION = "DFDGCN model configuration" 19 | CFG.RUNNER = SimpleTimeSeriesForecastingRunner 20 | CFG.DATASET_CLS = TimeSeriesForecastingDataset 21 | CFG.DATASET_NAME = "PEMS04" 22 | CFG.DATASET_TYPE = "Traffic flow" 23 | CFG.DATASET_INPUT_LEN = 12 24 | CFG.DATASET_OUTPUT_LEN = 12 25 | CFG.GPU_NUM = 1 26 | 27 | # ================= environment ================= # 28 | CFG.ENV = EasyDict() 29 | CFG.ENV.SEED = 1 30 | CFG.ENV.CUDNN = EasyDict() 31 | CFG.ENV.CUDNN.ENABLED = True 32 | 33 | # ================= model ================= # 34 | CFG.MODEL = EasyDict() 35 | CFG.MODEL.NAME = "DFDGCN" 36 | CFG.MODEL.ARCH = DFDGCN 37 | adj_mx, _ = load_adj("datasets/" + CFG.DATASET_NAME + 38 | "/adj_mx.pkl", "doubletransition") 39 | CFG.MODEL.PARAM = { 40 | 41 | "num_nodes": 307, 42 | "supports": [torch.tensor(i) for i in adj_mx], 43 | "dropout": 0.3, 44 | "gcn_bool": True, 45 | "addaptadj": True, 46 | "aptinit": None, 47 | "in_dim": 2, 48 | "out_dim": 12, 49 | "residual_channels": 32, 50 | "dilation_channels": 32, 51 | "skip_channels": 256, 52 | "end_channels": 512, 53 | "kernel_size": 2, 54 | "blocks": 4, 55 | "layers": 2, 56 | "a": 10, 57 | "seq_len": 12, 58 | "affine": False, 59 | "fft_emb": 10, 60 | "identity_emb": 10, 61 | "hidden_emb": 30, 62 | "subgraph": 20 63 | } 64 | CFG.MODEL.FORWARD_FEATURES = [0, 1,2] 65 | CFG.MODEL.TARGET_FEATURES = [0] 66 | 67 | # ================= optim ================= # 68 | CFG.TRAIN = EasyDict() 69 | CFG.TRAIN.LOSS = masked_mae 70 | CFG.TRAIN.OPTIM = EasyDict() 71 | CFG.TRAIN.OPTIM.TYPE = "Adam" 72 | CFG.TRAIN.OPTIM.PARAM = { 73 | "lr": 0.002, 74 | "weight_decay": 1.0e-5, 75 | 'eps':1.0e-8 76 | } 77 | CFG.TRAIN.LR_SCHEDULER = EasyDict() 78 | CFG.TRAIN.LR_SCHEDULER.TYPE = "MultiStepLR" 79 | CFG.TRAIN.LR_SCHEDULER.PARAM = { 80 | "milestones": [1, 50, 100, 150, 180], 81 | "gamma": 0.5 82 | } 83 | 84 | # ================= train ================= # 85 | CFG.TRAIN.CLIP_GRAD_PARAM = { 86 | "max_norm": 5.0 87 | } 88 | CFG.TRAIN.NUM_EPOCHS = 200 89 | CFG.TRAIN.CKPT_SAVE_DIR = os.path.join( 90 | "checkpoints", 91 | "_".join([CFG.MODEL.NAME, str(CFG.TRAIN.NUM_EPOCHS)]) 92 | ) 93 | # train data 94 | CFG.TRAIN.DATA = EasyDict() 95 | CFG.TRAIN.NULL_VAL = 0.0 96 | # read data 97 | CFG.TRAIN.DATA.DIR = "datasets/" + CFG.DATASET_NAME 98 | # dataloader args, optional 99 | CFG.TRAIN.DATA.BATCH_SIZE = 64 100 | CFG.TRAIN.DATA.PREFETCH = False 101 | CFG.TRAIN.DATA.SHUFFLE = True 102 | CFG.TRAIN.DATA.NUM_WORKERS = 2 103 | CFG.TRAIN.DATA.PIN_MEMORY = False 104 | 105 | # ================= validate ================= # 106 | CFG.VAL = EasyDict() 107 | CFG.VAL.INTERVAL = 1 108 | # validating data 109 | CFG.VAL.DATA = EasyDict() 110 | # read data 111 | CFG.VAL.DATA.DIR = "datasets/" + CFG.DATASET_NAME 112 | # dataloader args, optional 113 | CFG.VAL.DATA.BATCH_SIZE = 64 114 | CFG.VAL.DATA.PREFETCH = False 115 | CFG.VAL.DATA.SHUFFLE = False 116 | CFG.VAL.DATA.NUM_WORKERS = 2 117 | CFG.VAL.DATA.PIN_MEMORY = False 118 | 119 | # ================= test ================= # 120 | CFG.TEST = EasyDict() 121 | CFG.TEST.INTERVAL = 1 122 | # test data 123 | CFG.TEST.DATA = EasyDict() 124 | # read data 125 | CFG.TEST.DATA.DIR = "datasets/" + CFG.DATASET_NAME 126 | # dataloader args, optional 127 | CFG.TEST.DATA.BATCH_SIZE = 64 128 | CFG.TEST.DATA.PREFETCH = False 129 | CFG.TEST.DATA.SHUFFLE = False 130 | CFG.TEST.DATA.NUM_WORKERS = 2 131 | CFG.TEST.DATA.PIN_MEMORY = False 132 | -------------------------------------------------------------------------------- /DFDGCN/examples/DFDGCN/DFDGCN_PEMS07.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | # TODO: remove it when basicts can be installed by pip 5 | sys.path.append(os.path.abspath(__file__ + "/../../..")) 6 | import torch 7 | from easydict import EasyDict 8 | from basicts.archs import DFDGCN 9 | from basicts.runners import SimpleTimeSeriesForecastingRunner 10 | from basicts.data import TimeSeriesForecastingDataset 11 | from basicts.losses import masked_mae 12 | from basicts.utils import load_adj 13 | 14 | 15 | CFG = EasyDict() 16 | 17 | # ================= general ================= # 18 | CFG.DESCRIPTION = "DFDGCN model configuration" 19 | CFG.RUNNER = SimpleTimeSeriesForecastingRunner 20 | CFG.DATASET_CLS = TimeSeriesForecastingDataset 21 | CFG.DATASET_NAME = "PEMS07" 22 | CFG.DATASET_TYPE = "Traffic flow" 23 | CFG.DATASET_INPUT_LEN = 12 24 | CFG.DATASET_OUTPUT_LEN = 12 25 | CFG.GPU_NUM = 1 26 | 27 | # ================= environment ================= # 28 | CFG.ENV = EasyDict() 29 | CFG.ENV.SEED = 1 30 | CFG.ENV.CUDNN = EasyDict() 31 | CFG.ENV.CUDNN.ENABLED = True 32 | 33 | # ================= model ================= # 34 | CFG.MODEL = EasyDict() 35 | CFG.MODEL.NAME = "DFDGCN" 36 | CFG.MODEL.ARCH = DFDGCN 37 | adj_mx, _ = load_adj("datasets/" + CFG.DATASET_NAME + 38 | "/adj_mx.pkl", "doubletransition") 39 | CFG.MODEL.PARAM = { 40 | 41 | "num_nodes": 883, 42 | "supports": [torch.tensor(i) for i in adj_mx], 43 | "dropout": 0.3, 44 | "gcn_bool": True, 45 | "addaptadj": True, 46 | "aptinit": None, 47 | "in_dim": 2, 48 | "out_dim": 12, 49 | "residual_channels": 32, 50 | "dilation_channels": 32, 51 | "skip_channels": 256, 52 | "end_channels": 512, 53 | "kernel_size": 2, 54 | "blocks": 4, 55 | "layers": 2, 56 | "a": 1, 57 | "seq_len": 12, 58 | "affine": True, 59 | "fft_emb": 10, 60 | "identity_emb": 10, 61 | "hidden_emb": 30, 62 | "subgraph": 20 63 | } 64 | CFG.MODEL.FORWARD_FEATURES = [0, 1,2] 65 | CFG.MODEL.TARGET_FEATURES = [0] 66 | 67 | # ================= optim ================= # 68 | CFG.TRAIN = EasyDict() 69 | CFG.TRAIN.LOSS = masked_mae 70 | CFG.TRAIN.OPTIM = EasyDict() 71 | CFG.TRAIN.OPTIM.TYPE = "Adam" 72 | CFG.TRAIN.OPTIM.PARAM = { 73 | "lr": 0.002, 74 | "weight_decay": 0.0001, 75 | } 76 | CFG.TRAIN.LR_SCHEDULER = EasyDict() 77 | CFG.TRAIN.LR_SCHEDULER.TYPE = "MultiStepLR" 78 | CFG.TRAIN.LR_SCHEDULER.PARAM = { 79 | "milestones": [1, 50, 100, 150, 170], 80 | "gamma": 0.5 81 | } 82 | 83 | # ================= train ================= # 84 | CFG.TRAIN.CLIP_GRAD_PARAM = { 85 | "max_norm": 5.0 86 | } 87 | CFG.TRAIN.NUM_EPOCHS = 200 88 | CFG.TRAIN.CKPT_SAVE_DIR = os.path.join( 89 | "checkpoints", 90 | "_".join([CFG.MODEL.NAME, str(CFG.TRAIN.NUM_EPOCHS)]) 91 | ) 92 | # train data 93 | CFG.TRAIN.DATA = EasyDict() 94 | CFG.TRAIN.NULL_VAL = 0.0 95 | # read data 96 | CFG.TRAIN.DATA.DIR = "datasets/" + CFG.DATASET_NAME 97 | # dataloader args, optional 98 | CFG.TRAIN.DATA.BATCH_SIZE = 64 99 | CFG.TRAIN.DATA.PREFETCH = False 100 | CFG.TRAIN.DATA.SHUFFLE = True 101 | CFG.TRAIN.DATA.NUM_WORKERS = 2 102 | CFG.TRAIN.DATA.PIN_MEMORY = False 103 | 104 | # ================= validate ================= # 105 | CFG.VAL = EasyDict() 106 | CFG.VAL.INTERVAL = 1 107 | # validating data 108 | CFG.VAL.DATA = EasyDict() 109 | # read data 110 | CFG.VAL.DATA.DIR = "datasets/" + CFG.DATASET_NAME 111 | # dataloader args, optional 112 | CFG.VAL.DATA.BATCH_SIZE = 64 113 | CFG.VAL.DATA.PREFETCH = False 114 | CFG.VAL.DATA.SHUFFLE = False 115 | CFG.VAL.DATA.NUM_WORKERS = 2 116 | CFG.VAL.DATA.PIN_MEMORY = False 117 | 118 | # ================= test ================= # 119 | CFG.TEST = EasyDict() 120 | CFG.TEST.INTERVAL = 1 121 | # validating data 122 | CFG.TEST.DATA = EasyDict() 123 | # read data 124 | CFG.TEST.DATA.DIR = "datasets/" + CFG.DATASET_NAME 125 | # dataloader args, optional 126 | CFG.TEST.DATA.BATCH_SIZE = 64 127 | CFG.TEST.DATA.PREFETCH = False 128 | CFG.TEST.DATA.SHUFFLE = False 129 | CFG.TEST.DATA.NUM_WORKERS = 2 130 | CFG.TEST.DATA.PIN_MEMORY = False 131 | -------------------------------------------------------------------------------- /DFDGCN/examples/DFDGCN/DFDGCN_PEMS08.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | # TODO: remove it when basicts can be installed by pip 5 | sys.path.append(os.path.abspath(__file__ + "/../../..")) 6 | import torch 7 | from easydict import EasyDict 8 | from basicts.archs import DFDGCN 9 | from basicts.runners import SimpleTimeSeriesForecastingRunner 10 | from basicts.data import TimeSeriesForecastingDataset 11 | from basicts.losses import masked_mae 12 | from basicts.utils import load_adj 13 | 14 | 15 | CFG = EasyDict() 16 | 17 | # ================= general ================= # 18 | CFG.DESCRIPTION = "DFDGCN model configuration" 19 | CFG.RUNNER = SimpleTimeSeriesForecastingRunner 20 | CFG.DATASET_CLS = TimeSeriesForecastingDataset 21 | CFG.DATASET_NAME = "PEMS08" 22 | CFG.DATASET_TYPE = "Traffic flow" 23 | CFG.DATASET_INPUT_LEN = 12 24 | CFG.DATASET_OUTPUT_LEN = 12 25 | CFG.GPU_NUM = 1 26 | 27 | # ================= environment ================= # 28 | CFG.ENV = EasyDict() 29 | CFG.ENV.SEED = 1 30 | CFG.ENV.CUDNN = EasyDict() 31 | CFG.ENV.CUDNN.ENABLED = True 32 | 33 | # ================= model ================= # 34 | CFG.MODEL = EasyDict() 35 | CFG.MODEL.NAME = "DFDGCN" 36 | CFG.MODEL.ARCH = DFDGCN 37 | adj_mx, _ = load_adj("datasets/" + CFG.DATASET_NAME + 38 | "/adj_mx.pkl", "doubletransition") 39 | CFG.MODEL.PARAM = { 40 | 41 | 42 | "num_nodes": 170, 43 | "supports": [torch.tensor(i) for i in adj_mx], 44 | "dropout": 0.3, 45 | "gcn_bool": True, 46 | "addaptadj": True, 47 | "aptinit": None, 48 | "in_dim": 2, 49 | "out_dim": 12, 50 | "residual_channels": 32, 51 | "dilation_channels": 32, 52 | "skip_channels": 256, 53 | "end_channels": 512, 54 | "kernel_size": 2, 55 | "blocks": 4, 56 | "layers": 2, 57 | "a": 1, 58 | "seq_len": 12, 59 | "affine": True, 60 | "fft_emb": 10, 61 | "identity_emb": 10, 62 | "hidden_emb": 30, 63 | "subgraph": 20 64 | } 65 | CFG.MODEL.FORWARD_FEATURES = [0, 1,2] 66 | CFG.MODEL.TARGET_FEATURES = [0] 67 | 68 | # ================= optim ================= # 69 | CFG.TRAIN = EasyDict() 70 | CFG.TRAIN.LOSS = masked_mae 71 | CFG.TRAIN.OPTIM = EasyDict() 72 | CFG.TRAIN.OPTIM.TYPE = "Adam" 73 | CFG.TRAIN.OPTIM.PARAM = { 74 | "lr": 0.002, 75 | "weight_decay": 0.0001, 76 | } 77 | CFG.TRAIN.LR_SCHEDULER = EasyDict() 78 | CFG.TRAIN.LR_SCHEDULER.TYPE = "MultiStepLR" 79 | CFG.TRAIN.LR_SCHEDULER.PARAM = { 80 | "milestones": [1, 50, 100, 150, 180], 81 | "gamma": 0.5 82 | } 83 | 84 | # ================= train ================= # 85 | CFG.TRAIN.CLIP_GRAD_PARAM = { 86 | "max_norm": 5.0 87 | } 88 | CFG.TRAIN.NUM_EPOCHS = 200 89 | CFG.TRAIN.CKPT_SAVE_DIR = os.path.join( 90 | "checkpoints", 91 | "_".join([CFG.MODEL.NAME, str(CFG.TRAIN.NUM_EPOCHS)]) 92 | ) 93 | # train data 94 | CFG.TRAIN.DATA = EasyDict() 95 | CFG.TRAIN.NULL_VAL = 0.0 96 | # read data 97 | CFG.TRAIN.DATA.DIR = "datasets/" + CFG.DATASET_NAME 98 | # dataloader args, optional 99 | CFG.TRAIN.DATA.BATCH_SIZE = 64 100 | CFG.TRAIN.DATA.PREFETCH = False 101 | CFG.TRAIN.DATA.SHUFFLE = True 102 | CFG.TRAIN.DATA.NUM_WORKERS = 2 103 | CFG.TRAIN.DATA.PIN_MEMORY = False 104 | 105 | # ================= validate ================= # 106 | CFG.VAL = EasyDict() 107 | CFG.VAL.INTERVAL = 1 108 | # validating data 109 | CFG.VAL.DATA = EasyDict() 110 | # read data 111 | CFG.VAL.DATA.DIR = "datasets/" + CFG.DATASET_NAME 112 | # dataloader args, optional 113 | CFG.VAL.DATA.BATCH_SIZE = 64 114 | CFG.VAL.DATA.PREFETCH = False 115 | CFG.VAL.DATA.SHUFFLE = False 116 | CFG.VAL.DATA.NUM_WORKERS = 2 117 | CFG.VAL.DATA.PIN_MEMORY = False 118 | 119 | # ================= test ================= # 120 | CFG.TEST = EasyDict() 121 | CFG.TEST.INTERVAL = 1 122 | # test data 123 | CFG.TEST.DATA = EasyDict() 124 | # read data 125 | CFG.TEST.DATA.DIR = "datasets/" + CFG.DATASET_NAME 126 | # dataloader args, optional 127 | CFG.TEST.DATA.BATCH_SIZE = 64 128 | CFG.TEST.DATA.PREFETCH = False 129 | CFG.TEST.DATA.SHUFFLE = False 130 | CFG.TEST.DATA.NUM_WORKERS = 2 131 | CFG.TEST.DATA.PIN_MEMORY = False 132 | -------------------------------------------------------------------------------- /DFDGCN/examples/DFDGCN/__pycache__/DFDGCN_PEMS08.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/examples/DFDGCN/__pycache__/DFDGCN_PEMS08.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/examples/DFDGCN/__pycache__/GWNet_METR-LA.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/examples/DFDGCN/__pycache__/GWNet_METR-LA.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/examples/DFDGCN/__pycache__/GWNet_PEMS-BAY.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/examples/DFDGCN/__pycache__/GWNet_PEMS-BAY.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/examples/DFDGCN/__pycache__/GWNet_PEMS03.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/examples/DFDGCN/__pycache__/GWNet_PEMS03.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/examples/DFDGCN/__pycache__/GWNet_PEMS04.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/examples/DFDGCN/__pycache__/GWNet_PEMS04.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/examples/DFDGCN/__pycache__/GWNet_PEMS07.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/examples/DFDGCN/__pycache__/GWNet_PEMS07.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/examples/DFDGCN/__pycache__/GWNet_PEMS08.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/examples/DFDGCN/__pycache__/GWNet_PEMS08.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/examples/datasets/readme.txt: -------------------------------------------------------------------------------- 1 | Download the data and place the pre-processed data here. 2 | How to get the data such as PEMS-BAY/PEMS-03? 3 | https://github.com/zezhishao/BasicTS -------------------------------------------------------------------------------- /DFDGCN/examples/run.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from argparse import ArgumentParser 4 | 5 | # TODO: remove it when basicts can be installed by pip 6 | sys.path.append(os.path.abspath(__file__ + "/../..")) 7 | import torch 8 | from basicts import launch_training 9 | 10 | torch.set_num_threads(1) # aviod high cpu avg usage 11 | 12 | 13 | def parse_args(): 14 | parser = ArgumentParser(description="Run time series forecasting model in BasicTS framework!") 15 | parser.add_argument("-c", "--cfg", default="examples/DFDGCN/DFDGCN_PEMS08.py", help="training config") 16 | parser.add_argument("--gpus", default="0", help="visible gpus") 17 | return parser.parse_args() 18 | 19 | if __name__ == "__main__": 20 | args = parse_args() 21 | 22 | launch_training(args.cfg, args.gpus) 23 | -------------------------------------------------------------------------------- /DFDGCN/scripts/data_preparation/METR-LA/generate_training_data.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import shutil 4 | import pickle 5 | import argparse 6 | import csv 7 | 8 | import numpy as np 9 | import pandas as pd 10 | 11 | # TODO: remove it when basicts can be installed by pip 12 | sys.path.append(os.path.abspath(__file__ + "/../../../..")) 13 | from basicts.data.transform import standard_transform 14 | 15 | def process_location(url): 16 | lst = [] 17 | with open(url, "r") as f: 18 | f.readline() 19 | reader = csv.reader(f) 20 | for row in reader: 21 | if len(row) != 4: 22 | continue 23 | if "i" in str(row[0]): 24 | pass 25 | else: 26 | index, id, lon, lat = int(row[0]), int(row[1]), float(row[2]), float(row[3]) 27 | lst.append([lon,lat]) 28 | return np.array(lst) 29 | 30 | 31 | 32 | def generate_data(args: argparse.Namespace): 33 | """Preprocess and generate train/valid/test datasets. 34 | Default settings of METR-LA dataset: 35 | - Normalization method: standard norm. 36 | - Dataset division: 7:1:2. 37 | - Window size: history 12, future 12. 38 | - Channels (features): three channels [traffic speed, time of day, day of week] 39 | - Target: predict the traffic speed of the future 12 time steps. 40 | 41 | Args: 42 | args (argparse): configurations of preprocessing 43 | """ 44 | 45 | target_channel = args.target_channel 46 | future_seq_len = args.future_seq_len 47 | history_seq_len = args.history_seq_len 48 | add_time_of_day = args.tod 49 | add_day_of_week = args.dow 50 | output_dir = args.output_dir 51 | train_ratio = args.train_ratio 52 | valid_ratio = args.valid_ratio 53 | data_file_path = args.data_file_path 54 | graph_file_path = args.graph_file_path 55 | location_file_path = args.location_file_path 56 | 57 | # read data 58 | print(os.path.abspath(data_file_path)) 59 | df = pd.read_hdf(data_file_path) 60 | data = np.expand_dims(df.values, axis=-1) 61 | 62 | data = data[..., target_channel] 63 | print("raw time series shape: {0}".format(data.shape)) 64 | 65 | l, n, f = data.shape 66 | num_samples = l - (history_seq_len + future_seq_len) + 1 67 | train_num_short = round(num_samples * train_ratio) 68 | valid_num_short = round(num_samples * valid_ratio) 69 | test_num_short = num_samples - train_num_short - valid_num_short 70 | print("number of training samples:{0}".format(train_num_short)) 71 | print("number of validation samples:{0}".format(valid_num_short)) 72 | print("number of test samples:{0}".format(test_num_short)) 73 | 74 | index_list = [] 75 | for t in range(history_seq_len, num_samples + history_seq_len): 76 | index = (t-history_seq_len, t, t+future_seq_len) 77 | index_list.append(index) 78 | 79 | train_index = index_list[:train_num_short] 80 | valid_index = index_list[train_num_short: train_num_short + valid_num_short] 81 | test_index = index_list[train_num_short + 82 | valid_num_short: train_num_short + valid_num_short + test_num_short] 83 | 84 | scaler = standard_transform 85 | data_norm = scaler(data, output_dir, train_index, history_seq_len, future_seq_len) 86 | 87 | # add external feature 88 | feature_list = [data_norm] 89 | if add_time_of_day: 90 | # numerical time_of_day 91 | tod = ( 92 | df.index.values - df.index.values.astype("datetime64[D]")) / np.timedelta64(1, "D") 93 | tod_tiled = np.tile(tod, [1, n, 1]).transpose((2, 1, 0)) 94 | feature_list.append(tod_tiled) 95 | 96 | if add_day_of_week: 97 | # numerical day_of_week 98 | dow = df.index.dayofweek 99 | dow_tiled = np.tile(dow, [1, n, 1]).transpose((2, 1, 0)) 100 | feature_list.append(dow_tiled) 101 | 102 | processed_data = np.concatenate(feature_list, axis=-1) 103 | lon_lat = process_location(location_file_path) 104 | 105 | # dump data 106 | index = {} 107 | index["train"] = train_index 108 | index["valid"] = valid_index 109 | index["test"] = test_index 110 | with open(output_dir + "/index_in{0}_out{1}.pkl".format(history_seq_len, future_seq_len), "wb") as f: 111 | pickle.dump(index, f) 112 | 113 | data = {} 114 | data["processed_data"] = processed_data 115 | with open(output_dir + "/data_in{0}_out{1}.pkl".format(history_seq_len, future_seq_len), "wb") as f: 116 | pickle.dump(data, f) 117 | # copy adj 118 | shutil.copyfile(graph_file_path, output_dir + "/adj_mx.pkl") 119 | 120 | location = {} 121 | location["lon_lat"] = lon_lat 122 | with open(output_dir + "/location.pkl",'wb') as f: 123 | pickle.dump(location, f) 124 | 125 | 126 | if __name__ == "__main__": 127 | # sliding window size for generating history sequence and target sequence 128 | HISTORY_SEQ_LEN = 12 129 | FUTURE_SEQ_LEN = 12 130 | 131 | TRAIN_RATIO = 0.7 132 | VALID_RATIO = 0.1 133 | TARGET_CHANNEL = [0] # target channel(s) 134 | 135 | DATASET_NAME = "METR-LA" 136 | TOD = True # if add time_of_day feature 137 | DOW = True # if add day_of_week feature 138 | OUTPUT_DIR = "datasets/" + DATASET_NAME 139 | DATA_FILE_PATH = "datasets/raw_data//{0}/{0}.h5".format(DATASET_NAME) 140 | GRAPH_FILE_PATH = "datasets/raw_data//{0}/adj_{0}.pkl".format(DATASET_NAME) 141 | 142 | 143 | LOCATION_FILE_PATH = "D:/myfile/BasicTS/datasets/raw_data/{0}/graph_sensor_locations.csv".format(DATASET_NAME) 144 | 145 | parser = argparse.ArgumentParser() 146 | parser.add_argument("--output_dir", type=str, 147 | default=OUTPUT_DIR, help="Output directory.") 148 | parser.add_argument("--data_file_path", type=str, 149 | default=DATA_FILE_PATH, help="Raw traffic readings.") 150 | parser.add_argument("--graph_file_path", type=str, 151 | default=GRAPH_FILE_PATH, help="Raw traffic readings.") 152 | parser.add_argument("--location_file_path", type=str, 153 | default=LOCATION_FILE_PATH, help="Raw traffic readings.") 154 | parser.add_argument("--history_seq_len", type=int, 155 | default=HISTORY_SEQ_LEN, help="Sequence Length.") 156 | 157 | parser.add_argument("--future_seq_len", type=int, 158 | default=FUTURE_SEQ_LEN, help="Sequence Length.") 159 | parser.add_argument("--tod", type=bool, default=TOD, 160 | help="Add feature time_of_day.") 161 | parser.add_argument("--dow", type=bool, default=DOW, 162 | help="Add feature day_of_week.") 163 | parser.add_argument("--target_channel", type=list, 164 | default=TARGET_CHANNEL, help="Selected channels.") 165 | parser.add_argument("--train_ratio", type=float, 166 | default=TRAIN_RATIO, help="Train ratio") 167 | parser.add_argument("--valid_ratio", type=float, 168 | default=VALID_RATIO, help="Validate ratio.") 169 | args_metr = parser.parse_args() 170 | 171 | # print args 172 | print("-"*(20+45+5)) 173 | for key, value in sorted(vars(args_metr).items()): 174 | print("|{0:>20} = {1:<45}|".format(key, str(value))) 175 | print("-"*(20+45+5)) 176 | 177 | if os.path.exists(args_metr.output_dir): 178 | reply = str(input( 179 | f"{args_metr.output_dir} exists. Do you want to overwrite it? (y/n)")).lower().strip() 180 | if reply[0] != "y": 181 | sys.exit(0) 182 | else: 183 | os.makedirs(args_metr.output_dir) 184 | generate_data(args_metr) 185 | -------------------------------------------------------------------------------- /DFDGCN/scripts/data_preparation/PEMS-BAY/generate_training_data.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import shutil 4 | import pickle 5 | import argparse 6 | import csv 7 | import numpy as np 8 | import pandas as pd 9 | 10 | # TODO: remove it when basicts can be installed by pip 11 | sys.path.append(os.path.abspath(__file__ + "/../../../..")) 12 | from basicts.data.transform import standard_transform 13 | def process_location(url): 14 | lst = [] 15 | with open(url, "r") as f: 16 | f.readline() 17 | reader = csv.reader(f) 18 | for row in reader: 19 | if len(row) != 3: 20 | continue 21 | print(row) 22 | index, lon, lat = int(row[0]), float(row[1]), float(row[2]) 23 | lst.append([lon,lat]) 24 | return np.array(lst) 25 | 26 | def generate_data(args: argparse.Namespace): 27 | """Preprocess and generate train/valid/test datasets. 28 | Default settings of PEMS-BAY dataset: 29 | - Normalization method: standard norm. 30 | - Dataset division: 7:1:2. 31 | - Window size: history 12, future 12. 32 | - Channels (features): three channels [traffic speed, time of day, day of week] 33 | - Target: predict the traffic speed of the future 12 time steps. 34 | 35 | Args: 36 | args (argparse): configurations of preprocessing 37 | """ 38 | 39 | target_channel = args.target_channel 40 | future_seq_len = args.future_seq_len 41 | history_seq_len = args.history_seq_len 42 | add_time_of_day = args.tod 43 | add_day_of_week = args.dow 44 | output_dir = args.output_dir 45 | train_ratio = args.train_ratio 46 | valid_ratio = args.valid_ratio 47 | data_file_path = args.data_file_path 48 | graph_file_path = args.graph_file_path 49 | location_file_path = args.location_file_path 50 | 51 | # read data 52 | df = pd.read_hdf(data_file_path) 53 | data = np.expand_dims(df.values, axis=-1) 54 | 55 | data = data[..., target_channel] 56 | print("raw time series shape: {0}".format(data.shape)) 57 | 58 | l, n, f = data.shape 59 | num_samples = l - (history_seq_len + future_seq_len) + 1 60 | train_num_short = round(num_samples * train_ratio) 61 | valid_num_short = round(num_samples * valid_ratio) 62 | test_num_short = num_samples - train_num_short - valid_num_short 63 | print("number of training samples:{0}".format(train_num_short)) 64 | print("number of validation samples:{0}".format(valid_num_short)) 65 | print("number of test samples:{0}".format(test_num_short)) 66 | 67 | index_list = [] 68 | for t in range(history_seq_len, num_samples + history_seq_len): 69 | index = (t-history_seq_len, t, t+future_seq_len) 70 | index_list.append(index) 71 | 72 | train_index = index_list[:train_num_short] 73 | valid_index = index_list[train_num_short: train_num_short + valid_num_short] 74 | test_index = index_list[train_num_short + 75 | valid_num_short: train_num_short + valid_num_short + test_num_short] 76 | 77 | scaler = standard_transform 78 | data_norm = scaler(data, output_dir, train_index, history_seq_len, future_seq_len) 79 | 80 | # add external feature 81 | feature_list = [data_norm] 82 | if add_time_of_day: 83 | # numerical time_of_day 84 | tod = ( 85 | df.index.values - df.index.values.astype("datetime64[D]")) / np.timedelta64(1, "D") 86 | tod_tiled = np.tile(tod, [1, n, 1]).transpose((2, 1, 0)) 87 | feature_list.append(tod_tiled) 88 | 89 | if add_day_of_week: 90 | # numerical day_of_week 91 | dow = df.index.dayofweek 92 | dow_tiled = np.tile(dow, [1, n, 1]).transpose((2, 1, 0)) 93 | feature_list.append(dow_tiled) 94 | 95 | processed_data = np.concatenate(feature_list, axis=-1) 96 | lon_lat = process_location(location_file_path) 97 | print(len(lon_lat), 'location 1111111') 98 | 99 | # dump data 100 | index = {} 101 | index["train"] = train_index 102 | index["valid"] = valid_index 103 | index["test"] = test_index 104 | with open(output_dir + "/index_in{0}_out{1}.pkl".format(history_seq_len, future_seq_len), "wb") as f: 105 | pickle.dump(index, f) 106 | 107 | data = {} 108 | data["processed_data"] = processed_data 109 | with open(output_dir + "/data_in{0}_out{1}.pkl".format(history_seq_len, future_seq_len), "wb") as f: 110 | pickle.dump(data, f) 111 | # copy adj 112 | shutil.copyfile(graph_file_path, output_dir + "/adj_mx.pkl") 113 | location = {} 114 | location["lon_lat"] = lon_lat 115 | with open(output_dir + "/location.pkl",'wb') as f: 116 | pickle.dump(location, f) 117 | 118 | if __name__ == "__main__": 119 | # sliding window size for generating history sequence and target sequence 120 | HISTORY_SEQ_LEN = 12 121 | FUTURE_SEQ_LEN = 12 122 | 123 | TRAIN_RATIO = 0.7 124 | VALID_RATIO = 0.1 125 | TARGET_CHANNEL = [0] # target channel(s) 126 | 127 | DATASET_NAME = "PEMS-BAY" 128 | TOD = True # if add time_of_day feature 129 | DOW = True # if add day_of_week feature 130 | OUTPUT_DIR = "datasets/" + DATASET_NAME 131 | DATA_FILE_PATH = "datasets/raw_data/{0}/{0}.h5".format(DATASET_NAME) 132 | GRAPH_FILE_PATH = "datasets/raw_data/{0}/adj_{0}.pkl".format(DATASET_NAME) 133 | LOCATION_FILE_PATH = "datasets/raw_data/{0}/graph_sensor_locations_bay.csv".format(DATASET_NAME) 134 | 135 | parser = argparse.ArgumentParser() 136 | parser.add_argument("--output_dir", type=str, 137 | default=OUTPUT_DIR, help="Output directory.") 138 | parser.add_argument("--data_file_path", type=str, 139 | default=DATA_FILE_PATH, help="Raw traffic readings.") 140 | parser.add_argument("--graph_file_path", type=str, 141 | default=GRAPH_FILE_PATH, help="Raw traffic readings.") 142 | parser.add_argument("--location_file_path", type=str, 143 | default=LOCATION_FILE_PATH, help="Raw traffic readings.") 144 | parser.add_argument("--history_seq_len", type=int, 145 | default=HISTORY_SEQ_LEN, help="Sequence Length.") 146 | parser.add_argument("--future_seq_len", type=int, 147 | default=FUTURE_SEQ_LEN, help="Sequence Length.") 148 | parser.add_argument("--tod", type=bool, default=TOD, 149 | help="Add feature time_of_day.") 150 | parser.add_argument("--dow", type=bool, default=DOW, 151 | help="Add feature day_of_week.") 152 | parser.add_argument("--target_channel", type=list, 153 | default=TARGET_CHANNEL, help="Selected channels.") 154 | parser.add_argument("--train_ratio", type=float, 155 | default=TRAIN_RATIO, help="Train ratio") 156 | parser.add_argument("--valid_ratio", type=float, 157 | default=VALID_RATIO, help="Validate ratio.") 158 | args_metr = parser.parse_args() 159 | 160 | # print args 161 | print("-"*(20+45+5)) 162 | for key, value in sorted(vars(args_metr).items()): 163 | print("|{0:>20} = {1:<45}|".format(key, str(value))) 164 | print("-"*(20+45+5)) 165 | 166 | if os.path.exists(args_metr.output_dir): 167 | reply = str(input( 168 | f"{args_metr.output_dir} exists. Do you want to overwrite it? (y/n)")).lower().strip() 169 | if reply[0] != "y": 170 | sys.exit(0) 171 | else: 172 | os.makedirs(args_metr.output_dir) 173 | generate_data(args_metr) 174 | -------------------------------------------------------------------------------- /DFDGCN/scripts/data_preparation/PEMS03/__pycache__/generate_adj_mx.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/scripts/data_preparation/PEMS03/__pycache__/generate_adj_mx.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/scripts/data_preparation/PEMS03/generate_adj_mx.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | import pickle 4 | 5 | import numpy as np 6 | 7 | 8 | def get_adjacency_matrix(distance_df_filename: str, num_of_vertices: int, id_filename: str = None) -> tuple: 9 | """Generate adjacency matrix. 10 | 11 | Args: 12 | distance_df_filename (str): path of the csv file contains edges information 13 | num_of_vertices (int): number of vertices 14 | id_filename (str, optional): id filename. Defaults to None. 15 | 16 | Returns: 17 | tuple: two adjacency matrix. 18 | np.array: connectivity-based adjacency matrix A (A[i, j]=0 or A[i, j]=1) 19 | np.array: distance-based adjacency matrix A 20 | """ 21 | 22 | if "npy" in distance_df_filename: 23 | adj_mx = np.load(distance_df_filename) 24 | return adj_mx, None 25 | else: 26 | adjacency_matrix_connectivity = np.zeros((int(num_of_vertices), int( 27 | num_of_vertices)), dtype=np.float32) 28 | adjacency_matrix_distance = np.zeros((int(num_of_vertices), int(num_of_vertices)), 29 | dtype=np.float32) 30 | if id_filename: 31 | # the id in the distance file does not start from 0, so it needs to be remapped 32 | with open(id_filename, "r") as f: 33 | id_dict = {int(i): idx for idx, i in enumerate( 34 | f.read().strip().split("\n"))} # map node idx to 0-based index (start from 0) 35 | with open(distance_df_filename, "r") as f: 36 | f.readline() # omit the first line 37 | reader = csv.reader(f) 38 | for row in reader: 39 | if len(row) != 3: 40 | continue 41 | i, j, distance = int(row[0]), int(row[1]), float(row[2]) 42 | adjacency_matrix_connectivity[id_dict[i], id_dict[j]] = 1 43 | adjacency_matrix_connectivity[id_dict[j], id_dict[i]] = 1 44 | adjacency_matrix_distance[id_dict[i], 45 | id_dict[j]] = distance 46 | adjacency_matrix_distance[id_dict[j], 47 | id_dict[i]] = distance 48 | return adjacency_matrix_connectivity, adjacency_matrix_distance 49 | else: 50 | # ids in distance file start from 0 51 | with open(distance_df_filename, "r") as f: 52 | f.readline() 53 | reader = csv.reader(f) 54 | for row in reader: 55 | if len(row) != 3: 56 | continue 57 | i, j, distance = int(row[0]), int(row[1]), float(row[2]) 58 | adjacency_matrix_connectivity[i, j] = 1 59 | adjacency_matrix_connectivity[j, i] = 1 60 | adjacency_matrix_distance[i, j] = distance 61 | adjacency_matrix_distance[j, i] = distance 62 | return adjacency_matrix_connectivity, adjacency_matrix_distance 63 | 64 | 65 | def generate_adj_pems03(): 66 | distance_df_filename, num_of_vertices = "datasets/raw_data/PEMS03/PEMS03.csv", 358 67 | if os.path.exists(distance_df_filename.split(".", maxsplit=1)[0] + ".txt"): 68 | id_filename = distance_df_filename.split(".", maxsplit=1)[0] + ".txt" 69 | else: 70 | id_filename = None 71 | adj_mx, distance_mx = get_adjacency_matrix( 72 | distance_df_filename, num_of_vertices, id_filename=id_filename) 73 | # the self loop is missing 74 | add_self_loop = False 75 | if add_self_loop: 76 | print("adding self loop to adjacency matrices.") 77 | adj_mx = adj_mx + np.identity(adj_mx.shape[0]) 78 | distance_mx = distance_mx + np.identity(distance_mx.shape[0]) 79 | else: 80 | print("kindly note that there is no self loop in adjacency matrices.") 81 | with open("datasets/raw_data/PEMS03/adj_PEMS03.pkl", "wb") as f: 82 | pickle.dump(adj_mx, f) 83 | with open("datasets/raw_data/PEMS03/adj_PEMS03_distance.pkl", "wb") as f: 84 | pickle.dump(distance_mx, f) 85 | -------------------------------------------------------------------------------- /DFDGCN/scripts/data_preparation/PEMS03/generate_training_data.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import shutil 4 | import pickle 5 | import argparse 6 | 7 | import numpy as np 8 | 9 | from generate_adj_mx import generate_adj_pems03 10 | # TODO: remove it when basicts can be installed by pip 11 | sys.path.append(os.path.abspath(__file__ + "/../../../..")) 12 | from basicts.data.transform import standard_transform 13 | 14 | 15 | def generate_data(args: argparse.Namespace): 16 | """Preprocess and generate train/valid/test datasets. 17 | Default settings of PEMS03 dataset: 18 | - Normalization method: standard norm. 19 | - Dataset division: 6:2:2. 20 | - Window size: history 12, future 12. 21 | - Channels (features): three channels [traffic flow, time of day, day of week] 22 | - Target: predict the traffic flow of the future 12 time steps. 23 | 24 | Args: 25 | args (argparse): configurations of preprocessing 26 | """ 27 | 28 | target_channel = args.target_channel 29 | future_seq_len = args.future_seq_len 30 | history_seq_len = args.history_seq_len 31 | add_time_of_day = args.tod 32 | add_day_of_week = args.dow 33 | output_dir = args.output_dir 34 | train_ratio = args.train_ratio 35 | valid_ratio = args.valid_ratio 36 | data_file_path = args.data_file_path 37 | graph_file_path = args.graph_file_path 38 | steps_per_day = args.steps_per_day 39 | 40 | # read data 41 | data = np.load(data_file_path)["data"] 42 | data = data[..., target_channel] 43 | print("raw time series shape: {0}".format(data.shape)) 44 | 45 | l, n, f = data.shape 46 | num_samples = l - (history_seq_len + future_seq_len) + 1 47 | train_num_short = round(num_samples * train_ratio) 48 | valid_num_short = round(num_samples * valid_ratio) 49 | test_num_short = num_samples - train_num_short - valid_num_short 50 | print("number of training samples:{0}".format(train_num_short)) 51 | print("number of validation samples:{0}".format(valid_num_short)) 52 | print("number of test samples:{0}".format(test_num_short)) 53 | 54 | index_list = [] 55 | for t in range(history_seq_len, num_samples + history_seq_len): 56 | index = (t-history_seq_len, t, t+future_seq_len) 57 | index_list.append(index) 58 | 59 | train_index = index_list[:train_num_short] 60 | valid_index = index_list[train_num_short: train_num_short + valid_num_short] 61 | test_index = index_list[train_num_short + 62 | valid_num_short: train_num_short + valid_num_short + test_num_short] 63 | 64 | scaler = standard_transform 65 | data_norm = scaler(data, output_dir, train_index, history_seq_len, future_seq_len) 66 | 67 | # add external feature 68 | feature_list = [data_norm] 69 | if add_time_of_day: 70 | # numerical time_of_day 71 | tod = [i % steps_per_day / 72 | steps_per_day for i in range(data_norm.shape[0])] 73 | tod = np.array(tod) 74 | tod_tiled = np.tile(tod, [1, n, 1]).transpose((2, 1, 0)) 75 | feature_list.append(tod_tiled) 76 | 77 | if add_day_of_week: 78 | # numerical day_of_week 79 | dow = [(i // steps_per_day) % 7 for i in range(data_norm.shape[0])] 80 | dow = np.array(dow) 81 | dow_tiled = np.tile(dow, [1, n, 1]).transpose((2, 1, 0)) 82 | feature_list.append(dow_tiled) 83 | 84 | processed_data = np.concatenate(feature_list, axis=-1) 85 | 86 | # dump data 87 | index = {} 88 | index["train"] = train_index 89 | index["valid"] = valid_index 90 | index["test"] = test_index 91 | with open(output_dir + "/index_in{0}_out{1}.pkl".format(history_seq_len, future_seq_len), "wb") as f: 92 | pickle.dump(index, f) 93 | 94 | data = {} 95 | data["processed_data"] = processed_data 96 | with open(output_dir + "/data_in{0}_out{1}.pkl".format(history_seq_len, future_seq_len), "wb") as f: 97 | pickle.dump(data, f) 98 | # copy adj 99 | if os.path.exists(graph_file_path): 100 | # copy 101 | shutil.copyfile(graph_file_path, output_dir + "/adj_mx.pkl") 102 | else: 103 | # generate and copy 104 | generate_adj_pems03() 105 | shutil.copyfile(graph_file_path, output_dir + "/adj_mx.pkl") 106 | 107 | 108 | if __name__ == "__main__": 109 | # sliding window size for generating history sequence and target sequence 110 | HISTORY_SEQ_LEN = 12 111 | FUTURE_SEQ_LEN = 12 112 | 113 | TRAIN_RATIO = 0.6 114 | VALID_RATIO = 0.2 115 | TARGET_CHANNEL = [0] # target channel(s) 116 | STEPS_PER_DAY = 288 117 | 118 | DATASET_NAME = "PEMS03" 119 | TOD = True # if add time_of_day feature 120 | DOW = True # if add day_of_week feature 121 | OUTPUT_DIR = "datasets/" + DATASET_NAME 122 | DATA_FILE_PATH = "datasets/raw_data/{0}/{0}.npz".format(DATASET_NAME) 123 | GRAPH_FILE_PATH = "datasets/raw_data/{0}/adj_{0}.pkl".format(DATASET_NAME) 124 | 125 | parser = argparse.ArgumentParser() 126 | parser.add_argument("--output_dir", type=str, 127 | default=OUTPUT_DIR, help="Output directory.") 128 | parser.add_argument("--data_file_path", type=str, 129 | default=DATA_FILE_PATH, help="Raw traffic readings.") 130 | parser.add_argument("--graph_file_path", type=str, 131 | default=GRAPH_FILE_PATH, help="Raw traffic readings.") 132 | parser.add_argument("--history_seq_len", type=int, 133 | default=HISTORY_SEQ_LEN, help="Sequence Length.") 134 | parser.add_argument("--future_seq_len", type=int, 135 | default=FUTURE_SEQ_LEN, help="Sequence Length.") 136 | parser.add_argument("--steps_per_day", type=int, 137 | default=STEPS_PER_DAY, help="Sequence Length.") 138 | parser.add_argument("--tod", type=bool, default=TOD, 139 | help="Add feature time_of_day.") 140 | parser.add_argument("--dow", type=bool, default=DOW, 141 | help="Add feature day_of_week.") 142 | parser.add_argument("--target_channel", type=list, 143 | default=TARGET_CHANNEL, help="Selected channels.") 144 | parser.add_argument("--train_ratio", type=float, 145 | default=TRAIN_RATIO, help="Train ratio") 146 | parser.add_argument("--valid_ratio", type=float, 147 | default=VALID_RATIO, help="Validate ratio.") 148 | args_metr = parser.parse_args() 149 | 150 | # print args 151 | print("-"*(20+45+5)) 152 | for key, value in sorted(vars(args_metr).items()): 153 | print("|{0:>20} = {1:<45}|".format(key, str(value))) 154 | print("-"*(20+45+5)) 155 | 156 | if os.path.exists(args_metr.output_dir): 157 | reply = str(input( 158 | f"{args_metr.output_dir} exists. Do you want to overwrite it? (y/n)")).lower().strip() 159 | if reply[0] != "y": 160 | sys.exit(0) 161 | else: 162 | os.makedirs(args_metr.output_dir) 163 | generate_data(args_metr) 164 | -------------------------------------------------------------------------------- /DFDGCN/scripts/data_preparation/PEMS04/__pycache__/generate_adj_mx.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/scripts/data_preparation/PEMS04/__pycache__/generate_adj_mx.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/scripts/data_preparation/PEMS04/generate_adj_mx.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | import pickle 4 | 5 | import numpy as np 6 | 7 | 8 | def get_adjacency_matrix(distance_df_filename: str, num_of_vertices: int, id_filename: str = None) -> tuple: 9 | """Generate adjacency matrix. 10 | 11 | Args: 12 | distance_df_filename (str): path of the csv file contains edges information 13 | num_of_vertices (int): number of vertices 14 | id_filename (str, optional): id filename. Defaults to None. 15 | 16 | Returns: 17 | tuple: two adjacency matrix. 18 | np.array: connectivity-based adjacency matrix A (A[i, j]=0 or A[i, j]=1) 19 | np.array: distance-based adjacency matrix A 20 | """ 21 | 22 | if "npy" in distance_df_filename: 23 | adj_mx = np.load(distance_df_filename) 24 | return adj_mx, None 25 | else: 26 | adjacency_matrix_connectivity = np.zeros((int(num_of_vertices), int( 27 | num_of_vertices)), dtype=np.float32) 28 | adjacency_matrix_distance = np.zeros((int(num_of_vertices), int(num_of_vertices)), 29 | dtype=np.float32) 30 | if id_filename: 31 | # the id in the distance file does not start from 0, so it needs to be remapped 32 | with open(id_filename, "r") as f: 33 | id_dict = {int(i): idx for idx, i in enumerate( 34 | f.read().strip().split("\n"))} # map node idx to 0-based index (start from 0) 35 | with open(distance_df_filename, "r") as f: 36 | f.readline() # omit the first line 37 | reader = csv.reader(f) 38 | for row in reader: 39 | if len(row) != 3: 40 | continue 41 | i, j, distance = int(row[0]), int(row[1]), float(row[2]) 42 | adjacency_matrix_connectivity[id_dict[i], id_dict[j]] = 1 43 | adjacency_matrix_connectivity[id_dict[j], id_dict[i]] = 1 44 | adjacency_matrix_distance[id_dict[i], 45 | id_dict[j]] = distance 46 | adjacency_matrix_distance[id_dict[j], 47 | id_dict[i]] = distance 48 | return adjacency_matrix_connectivity, adjacency_matrix_distance 49 | else: 50 | # ids in distance file start from 0 51 | with open(distance_df_filename, "r") as f: 52 | f.readline() 53 | reader = csv.reader(f) 54 | for row in reader: 55 | if len(row) != 3: 56 | continue 57 | i, j, distance = int(row[0]), int(row[1]), float(row[2]) 58 | adjacency_matrix_connectivity[i, j] = 1 59 | adjacency_matrix_connectivity[j, i] = 1 60 | adjacency_matrix_distance[i, j] = distance 61 | adjacency_matrix_distance[j, i] = distance 62 | return adjacency_matrix_connectivity, adjacency_matrix_distance 63 | 64 | 65 | def generate_adj_pems04(): 66 | distance_df_filename, num_of_vertices = "D:/myfile/BasicTS/datasets/raw_data/PEMS04/PEMS04.csv", 307 67 | if os.path.exists(distance_df_filename.split(".", maxsplit=1)[0] + ".txt"): 68 | id_filename = distance_df_filename.split(".", maxsplit=1)[0] + ".txt" 69 | else: 70 | id_filename = None 71 | adj_mx, distance_mx = get_adjacency_matrix( 72 | distance_df_filename, num_of_vertices, id_filename=id_filename) 73 | # the self loop is missing 74 | add_self_loop = False 75 | if add_self_loop: 76 | print("adding self loop to adjacency matrices.") 77 | adj_mx = adj_mx + np.identity(adj_mx.shape[0]) 78 | distance_mx = distance_mx + np.identity(distance_mx.shape[0]) 79 | else: 80 | print("kindly note that there is no self loop in adjacency matrices.") 81 | with open("D:/myfile/BasicTS/examples/datasets/raw_data/PEMS04/adj_PEMS04.pkl", "wb") as f: 82 | pickle.dump(adj_mx, f) 83 | with open("D:/myfile/BasicTS/examples/datasets/raw_data/PEMS04/adj_PEMS04_distance.pkl", "wb") as f: 84 | pickle.dump(distance_mx, f) 85 | -------------------------------------------------------------------------------- /DFDGCN/scripts/data_preparation/PEMS04/generate_training_data.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import shutil 4 | import pickle 5 | import argparse 6 | 7 | import numpy as np 8 | 9 | from generate_adj_mx import generate_adj_pems04 10 | # TODO: remove it when basicts can be installed by pip 11 | sys.path.append(os.path.abspath(__file__ + "/../../../..")) 12 | from basicts.data.transform import standard_transform 13 | 14 | 15 | def generate_data(args: argparse.Namespace): 16 | """Preprocess and generate train/valid/test datasets. 17 | Default settings of PEMS04 dataset: 18 | - Normalization method: standard norm. 19 | - Dataset division: 6:2:2. 20 | - Window size: history 12, future 12. 21 | - Channels (features): three channels [traffic flow, time of day, day of week] 22 | - Target: predict the traffic flow of the future 12 time steps. 23 | 24 | Args: 25 | args (argparse): configurations of preprocessing 26 | """ 27 | 28 | target_channel = args.target_channel 29 | future_seq_len = args.future_seq_len 30 | history_seq_len = args.history_seq_len 31 | add_time_of_day = args.tod 32 | add_day_of_week = args.dow 33 | output_dir = args.output_dir 34 | train_ratio = args.train_ratio 35 | valid_ratio = args.valid_ratio 36 | data_file_path = args.data_file_path 37 | graph_file_path = args.graph_file_path 38 | steps_per_day = args.steps_per_day 39 | 40 | # read data 41 | data = np.load(data_file_path)["data"] 42 | data = data[..., target_channel] 43 | print("raw time series shape: {0}".format(data.shape)) 44 | 45 | l, n, f = data.shape 46 | num_samples = l - (history_seq_len + future_seq_len) + 1 47 | train_num_short = round(num_samples * train_ratio) 48 | valid_num_short = round(num_samples * valid_ratio) 49 | test_num_short = num_samples - train_num_short - valid_num_short 50 | print("number of training samples:{0}".format(train_num_short)) 51 | print("number of validation samples:{0}".format(valid_num_short)) 52 | print("number of test samples:{0}".format(test_num_short)) 53 | 54 | index_list = [] 55 | for t in range(history_seq_len, num_samples + history_seq_len): 56 | index = (t-history_seq_len, t, t+future_seq_len) 57 | index_list.append(index) 58 | 59 | train_index = index_list[:train_num_short] 60 | valid_index = index_list[train_num_short: train_num_short + valid_num_short] 61 | test_index = index_list[train_num_short + 62 | valid_num_short: train_num_short + valid_num_short + test_num_short] 63 | 64 | scaler = standard_transform 65 | data_norm = scaler(data, output_dir, train_index, history_seq_len, future_seq_len) 66 | 67 | # add external feature 68 | feature_list = [data_norm] 69 | if add_time_of_day: 70 | # numerical time_of_day 71 | tod = [i % steps_per_day / 72 | steps_per_day for i in range(data_norm.shape[0])] 73 | tod = np.array(tod) 74 | tod_tiled = np.tile(tod, [1, n, 1]).transpose((2, 1, 0)) 75 | feature_list.append(tod_tiled) 76 | 77 | if add_day_of_week: 78 | # numerical day_of_week 79 | dow = [(i // steps_per_day) % 7 for i in range(data_norm.shape[0])] 80 | dow = np.array(dow) 81 | dow_tiled = np.tile(dow, [1, n, 1]).transpose((2, 1, 0)) 82 | feature_list.append(dow_tiled) 83 | 84 | processed_data = np.concatenate(feature_list, axis=-1) 85 | 86 | # dump data 87 | index = {} 88 | index["train"] = train_index 89 | index["valid"] = valid_index 90 | index["test"] = test_index 91 | with open(output_dir + "/index_in{0}_out{1}.pkl".format(history_seq_len, future_seq_len), "wb") as f: 92 | pickle.dump(index, f) 93 | 94 | data = {} 95 | data["processed_data"] = processed_data 96 | with open(output_dir + "/data_in{0}_out{1}.pkl".format(history_seq_len, future_seq_len), "wb") as f: 97 | pickle.dump(data, f) 98 | # copy adj 99 | if os.path.exists(args.graph_file_path): 100 | # copy 101 | shutil.copyfile(args.graph_file_path, output_dir + "/adj_mx.pkl") 102 | else: 103 | # generate and copy 104 | generate_adj_pems04() 105 | shutil.copyfile(graph_file_path, output_dir + "/adj_mx.pkl") 106 | 107 | 108 | if __name__ == "__main__": 109 | # sliding window size for generating history sequence and target sequence 110 | HISTORY_SEQ_LEN = 12 111 | FUTURE_SEQ_LEN = 12 112 | 113 | TRAIN_RATIO = 0.6 114 | VALID_RATIO = 0.2 115 | TARGET_CHANNEL = [0] # target channel(s) 116 | STEPS_PER_DAY = 288 117 | 118 | DATASET_NAME = "PEMS04" 119 | TOD = True # if add time_of_day feature 120 | DOW = True # if add day_of_week feature 121 | OUTPUT_DIR = "datasets/" + DATASET_NAME 122 | DATA_FILE_PATH = "datasets/raw_data/{0}/{0}.npz".format(DATASET_NAME) 123 | GRAPH_FILE_PATH = "datasets/raw_data/{0}/adj_{0}.pkl".format(DATASET_NAME) 124 | 125 | parser = argparse.ArgumentParser() 126 | parser.add_argument("--output_dir", type=str, 127 | default=OUTPUT_DIR, help="Output directory.") 128 | parser.add_argument("--data_file_path", type=str, 129 | default=DATA_FILE_PATH, help="Raw traffic readings.") 130 | parser.add_argument("--graph_file_path", type=str, 131 | default=GRAPH_FILE_PATH, help="Raw traffic readings.") 132 | parser.add_argument("--history_seq_len", type=int, 133 | default=HISTORY_SEQ_LEN, help="Sequence Length.") 134 | parser.add_argument("--future_seq_len", type=int, 135 | default=FUTURE_SEQ_LEN, help="Sequence Length.") 136 | parser.add_argument("--steps_per_day", type=int, 137 | default=STEPS_PER_DAY, help="Sequence Length.") 138 | parser.add_argument("--tod", type=bool, default=TOD, 139 | help="Add feature time_of_day.") 140 | parser.add_argument("--dow", type=bool, default=DOW, 141 | help="Add feature day_of_week.") 142 | parser.add_argument("--target_channel", type=list, 143 | default=TARGET_CHANNEL, help="Selected channels.") 144 | parser.add_argument("--train_ratio", type=float, 145 | default=TRAIN_RATIO, help="Train ratio") 146 | parser.add_argument("--valid_ratio", type=float, 147 | default=VALID_RATIO, help="Validate ratio.") 148 | args_metr = parser.parse_args() 149 | 150 | # print args 151 | print("-"*(20+45+5)) 152 | for key, value in sorted(vars(args_metr).items()): 153 | print("|{0:>20} = {1:<45}|".format(key, str(value))) 154 | print("-"*(20+45+5)) 155 | 156 | if os.path.exists(args_metr.output_dir): 157 | reply = str(input( 158 | f"{args_metr.output_dir} exists. Do you want to overwrite it? (y/n)")).lower().strip() 159 | if reply[0] != "y": 160 | sys.exit(0) 161 | else: 162 | os.makedirs(args_metr.output_dir) 163 | generate_data(args_metr) 164 | -------------------------------------------------------------------------------- /DFDGCN/scripts/data_preparation/PEMS07/generate_adj_mx.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | import pickle 4 | 5 | import numpy as np 6 | 7 | 8 | def get_adjacency_matrix(distance_df_filename: str, num_of_vertices: int, id_filename: str = None) -> tuple: 9 | """Generate adjacency matrix. 10 | 11 | Args: 12 | distance_df_filename (str): path of the csv file contains edges information 13 | num_of_vertices (int): number of vertices 14 | id_filename (str, optional): id filename. Defaults to None. 15 | 16 | Returns: 17 | tuple: two adjacency matrix. 18 | np.array: connectivity-based adjacency matrix A (A[i, j]=0 or A[i, j]=1) 19 | np.array: distance-based adjacency matrix A 20 | """ 21 | 22 | if "npy" in distance_df_filename: 23 | adj_mx = np.load(distance_df_filename) 24 | return adj_mx, None 25 | else: 26 | adjacency_matrix_connectivity = np.zeros((int(num_of_vertices), int( 27 | num_of_vertices)), dtype=np.float32) 28 | adjacency_matrix_distance = np.zeros((int(num_of_vertices), int(num_of_vertices)), 29 | dtype=np.float32) 30 | if id_filename: 31 | # the id in the distance file does not start from 0, so it needs to be remapped 32 | with open(id_filename, "r") as f: 33 | id_dict = {int(i): idx for idx, i in enumerate( 34 | f.read().strip().split("\n"))} # map node idx to 0-based index (start from 0) 35 | with open(distance_df_filename, "r") as f: 36 | f.readline() # omit the first line 37 | reader = csv.reader(f) 38 | for row in reader: 39 | if len(row) != 3: 40 | continue 41 | i, j, distance = int(row[0]), int(row[1]), float(row[2]) 42 | adjacency_matrix_connectivity[id_dict[i], id_dict[j]] = 1 43 | adjacency_matrix_connectivity[id_dict[j], id_dict[i]] = 1 44 | adjacency_matrix_distance[id_dict[i], 45 | id_dict[j]] = distance 46 | adjacency_matrix_distance[id_dict[j], 47 | id_dict[i]] = distance 48 | return adjacency_matrix_connectivity, adjacency_matrix_distance 49 | else: 50 | # ids in distance file start from 0 51 | with open(distance_df_filename, "r") as f: 52 | f.readline() 53 | reader = csv.reader(f) 54 | for row in reader: 55 | if len(row) != 3: 56 | continue 57 | i, j, distance = int(row[0]), int(row[1]), float(row[2]) 58 | adjacency_matrix_connectivity[i, j] = 1 59 | adjacency_matrix_connectivity[j, i] = 1 60 | adjacency_matrix_distance[i, j] = distance 61 | adjacency_matrix_distance[j, i] = distance 62 | return adjacency_matrix_connectivity, adjacency_matrix_distance 63 | 64 | 65 | def generate_adj_pems07(): 66 | distance_df_filename, num_of_vertices = "datasets/raw_data/PEMS07/PEMS07.csv", 883 67 | if os.path.exists(distance_df_filename.split(".", maxsplit=1)[0] + ".txt"): 68 | id_filename = distance_df_filename.split(".", maxsplit=1)[0] + ".txt" 69 | else: 70 | id_filename = None 71 | adj_mx, distance_mx = get_adjacency_matrix( 72 | distance_df_filename, num_of_vertices, id_filename=id_filename) 73 | # the self loop is missing 74 | add_self_loop = False 75 | if add_self_loop: 76 | print("adding self loop to adjacency matrices.") 77 | adj_mx = adj_mx + np.identity(adj_mx.shape[0]) 78 | distance_mx = distance_mx + np.identity(distance_mx.shape[0]) 79 | else: 80 | print("kindly note that there is no self loop in adjacency matrices.") 81 | with open("datasets/raw_data/PEMS07/adj_PEMS07.pkl", "wb") as f: 82 | pickle.dump(adj_mx, f) 83 | with open("datasets/raw_data/PEMS07/adj_PEMS07_distance.pkl", "wb") as f: 84 | pickle.dump(distance_mx, f) 85 | -------------------------------------------------------------------------------- /DFDGCN/scripts/data_preparation/PEMS07/generate_training_data.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import shutil 4 | import pickle 5 | import argparse 6 | 7 | import numpy as np 8 | 9 | from generate_adj_mx import generate_adj_pems07 10 | # TODO: remove it when basicts can be installed by pip 11 | sys.path.append(os.path.abspath(__file__ + "/../../../..")) 12 | from basicts.data.transform import standard_transform 13 | 14 | 15 | def generate_data(args: argparse.Namespace): 16 | """Preprocess and generate train/valid/test datasets. 17 | Default settings of PEMS07 dataset: 18 | - Normalization method: standard norm. 19 | - Dataset division: 6:2:2. 20 | - Window size: history 12, future 12. 21 | - Channels (features): three channels [traffic flow, time of day, day of week] 22 | - Target: predict the traffic flolw of the future 12 time steps. 23 | 24 | Args: 25 | args (argparse): configurations of preprocessing 26 | """ 27 | 28 | target_channel = args.target_channel 29 | future_seq_len = args.future_seq_len 30 | history_seq_len = args.history_seq_len 31 | add_time_of_day = args.tod 32 | add_day_of_week = args.dow 33 | output_dir = args.output_dir 34 | train_ratio = args.train_ratio 35 | valid_ratio = args.valid_ratio 36 | data_file_path = args.data_file_path 37 | graph_file_path = args.graph_file_path 38 | steps_per_day = args.steps_per_day 39 | 40 | # read data 41 | data = np.load(data_file_path)["data"] 42 | data = data[..., target_channel] 43 | print("raw time series shape: {0}".format(data.shape)) 44 | 45 | l, n, f = data.shape 46 | num_samples = l - (history_seq_len + future_seq_len) + 1 47 | train_num_short = round(num_samples * train_ratio) 48 | valid_num_short = round(num_samples * valid_ratio) 49 | test_num_short = num_samples - train_num_short - valid_num_short 50 | print("number of training samples:{0}".format(train_num_short)) 51 | print("number of validation samples:{0}".format(valid_num_short)) 52 | print("number of test samples:{0}".format(test_num_short)) 53 | 54 | index_list = [] 55 | for t in range(history_seq_len, num_samples + history_seq_len): 56 | index = (t-history_seq_len, t, t+future_seq_len) 57 | index_list.append(index) 58 | 59 | train_index = index_list[:train_num_short] 60 | valid_index = index_list[train_num_short: train_num_short + valid_num_short] 61 | test_index = index_list[train_num_short + 62 | valid_num_short: train_num_short + valid_num_short + test_num_short] 63 | 64 | scaler = standard_transform 65 | data_norm = scaler(data, output_dir, train_index, history_seq_len, future_seq_len) 66 | 67 | # add external feature 68 | feature_list = [data_norm] 69 | if add_time_of_day: 70 | # numerical time_of_day 71 | tod = [i % steps_per_day / 72 | steps_per_day for i in range(data_norm.shape[0])] 73 | tod = np.array(tod) 74 | tod_tiled = np.tile(tod, [1, n, 1]).transpose((2, 1, 0)) 75 | feature_list.append(tod_tiled) 76 | 77 | if add_day_of_week: 78 | # numerical day_of_week 79 | dow = [(i // steps_per_day) % 7 for i in range(data_norm.shape[0])] 80 | dow = np.array(dow) 81 | dow_tiled = np.tile(dow, [1, n, 1]).transpose((2, 1, 0)) 82 | feature_list.append(dow_tiled) 83 | 84 | processed_data = np.concatenate(feature_list, axis=-1) 85 | 86 | # dump data 87 | index = {} 88 | index["train"] = train_index 89 | index["valid"] = valid_index 90 | index["test"] = test_index 91 | with open(output_dir + "/index_in{0}_out{1}.pkl".format(history_seq_len, future_seq_len), "wb") as f: 92 | pickle.dump(index, f) 93 | 94 | data = {} 95 | data["processed_data"] = processed_data 96 | with open(output_dir + "/data_in{0}_out{1}.pkl".format(history_seq_len, future_seq_len), "wb") as f: 97 | pickle.dump(data, f) 98 | # copy adj 99 | if os.path.exists(args.graph_file_path): 100 | # copy 101 | shutil.copyfile(args.graph_file_path, output_dir + "/adj_mx.pkl") 102 | else: 103 | # generate and copy 104 | generate_adj_pems07() 105 | shutil.copyfile(graph_file_path, output_dir + "/adj_mx.pkl") 106 | 107 | 108 | if __name__ == "__main__": 109 | # sliding window size for generating history sequence and target sequence 110 | HISTORY_SEQ_LEN = 12 111 | FUTURE_SEQ_LEN = 12 112 | 113 | TRAIN_RATIO = 0.6 114 | VALID_RATIO = 0.2 115 | TARGET_CHANNEL = [0] # target channel(s) 116 | STEPS_PER_DAY = 288 117 | 118 | DATASET_NAME = "PEMS07" 119 | TOD = True # if add time_of_day feature 120 | DOW = True # if add day_of_week feature 121 | OUTPUT_DIR = "datasets/" + DATASET_NAME 122 | DATA_FILE_PATH = "datasets/raw_data/{0}/{0}.npz".format(DATASET_NAME) 123 | GRAPH_FILE_PATH = "datasets/raw_data/{0}/adj_{0}.pkl".format(DATASET_NAME) 124 | 125 | parser = argparse.ArgumentParser() 126 | parser.add_argument("--output_dir", type=str, 127 | default=OUTPUT_DIR, help="Output directory.") 128 | parser.add_argument("--data_file_path", type=str, 129 | default=DATA_FILE_PATH, help="Raw traffic readings.") 130 | parser.add_argument("--graph_file_path", type=str, 131 | default=GRAPH_FILE_PATH, help="Raw traffic readings.") 132 | parser.add_argument("--history_seq_len", type=int, 133 | default=HISTORY_SEQ_LEN, help="Sequence Length.") 134 | parser.add_argument("--future_seq_len", type=int, 135 | default=FUTURE_SEQ_LEN, help="Sequence Length.") 136 | parser.add_argument("--steps_per_day", type=int, 137 | default=STEPS_PER_DAY, help="Sequence Length.") 138 | parser.add_argument("--tod", type=bool, default=TOD, 139 | help="Add feature time_of_day.") 140 | parser.add_argument("--dow", type=bool, default=DOW, 141 | help="Add feature day_of_week.") 142 | parser.add_argument("--target_channel", type=list, 143 | default=TARGET_CHANNEL, help="Selected channels.") 144 | parser.add_argument("--train_ratio", type=float, 145 | default=TRAIN_RATIO, help="Train ratio") 146 | parser.add_argument("--valid_ratio", type=float, 147 | default=VALID_RATIO, help="Validate ratio.") 148 | args_metr = parser.parse_args() 149 | 150 | # print args 151 | print("-"*(20+45+5)) 152 | for key, value in sorted(vars(args_metr).items()): 153 | print("|{0:>20} = {1:<45}|".format(key, str(value))) 154 | print("-"*(20+45+5)) 155 | 156 | if os.path.exists(args_metr.output_dir): 157 | reply = str(input( 158 | f"{args_metr.output_dir} exists. Do you want to overwrite it? (y/n)")).lower().strip() 159 | if reply[0] != "y": 160 | sys.exit(0) 161 | else: 162 | os.makedirs(args_metr.output_dir) 163 | generate_data(args_metr) 164 | -------------------------------------------------------------------------------- /DFDGCN/scripts/data_preparation/PEMS08/__pycache__/generate_adj_mx.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GestaltCogTeam/DFDGCN/ebc2d2b55de7b34cd031fc968bea08a83e693b15/DFDGCN/scripts/data_preparation/PEMS08/__pycache__/generate_adj_mx.cpython-39.pyc -------------------------------------------------------------------------------- /DFDGCN/scripts/data_preparation/PEMS08/generate_adj_mx.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | import pickle 4 | 5 | import numpy as np 6 | 7 | 8 | def get_adjacency_matrix(distance_df_filename: str, num_of_vertices: int, id_filename: str = None) -> tuple: 9 | """Generate adjacency matrix. 10 | 11 | Args: 12 | distance_df_filename (str): path of the csv file contains edges information 13 | num_of_vertices (int): number of vertices 14 | id_filename (str, optional): id filename. Defaults to None. 15 | 16 | Returns: 17 | tuple: two adjacency matrix. 18 | np.array: connectivity-based adjacency matrix A (A[i, j]=0 or A[i, j]=1) 19 | np.array: distance-based adjacency matrix A 20 | """ 21 | 22 | if "npy" in distance_df_filename: 23 | adj_mx = np.load(distance_df_filename) 24 | return adj_mx, None 25 | else: 26 | adjacency_matrix_connectivity = np.zeros((int(num_of_vertices), int( 27 | num_of_vertices)), dtype=np.float32) 28 | adjacency_matrix_distance = np.zeros((int(num_of_vertices), int(num_of_vertices)), 29 | dtype=np.float32) 30 | if id_filename: 31 | # the id in the distance file does not start from 0, so it needs to be remapped 32 | with open(id_filename, "r") as f: 33 | id_dict = {int(i): idx for idx, i in enumerate( 34 | f.read().strip().split("\n"))} # map node idx to 0-based index (start from 0) 35 | with open(distance_df_filename, "r") as f: 36 | f.readline() # omit the first line 37 | reader = csv.reader(f) 38 | for row in reader: 39 | if len(row) != 3: 40 | continue 41 | i, j, distance = int(row[0]), int(row[1]), float(row[2]) 42 | adjacency_matrix_connectivity[id_dict[i], id_dict[j]] = 1 43 | adjacency_matrix_connectivity[id_dict[j], id_dict[i]] = 1 44 | adjacency_matrix_distance[id_dict[i], 45 | id_dict[j]] = distance 46 | adjacency_matrix_distance[id_dict[j], 47 | id_dict[i]] = distance 48 | return adjacency_matrix_connectivity, adjacency_matrix_distance 49 | else: 50 | # ids in distance file start from 0 51 | with open(distance_df_filename, "r") as f: 52 | f.readline() 53 | reader = csv.reader(f) 54 | for row in reader: 55 | if len(row) != 3: 56 | continue 57 | i, j, distance = int(row[0]), int(row[1]), float(row[2]) 58 | adjacency_matrix_connectivity[i, j] = 1 59 | adjacency_matrix_connectivity[j, i] = 1 60 | adjacency_matrix_distance[i, j] = distance 61 | adjacency_matrix_distance[j, i] = distance 62 | return adjacency_matrix_connectivity, adjacency_matrix_distance 63 | 64 | 65 | def generate_adj_pems08(): 66 | distance_df_filename, num_of_vertices = "D:/myfile/BasicTS/examples/datasets/raw_data/PEMS08/PEMS08.csv", 170 67 | if os.path.exists(distance_df_filename.split(".", maxsplit=1)[0] + ".txt"): 68 | id_filename = distance_df_filename.split(".", maxsplit=1)[0] + ".txt" 69 | else: 70 | id_filename = None 71 | adj_mx, distance_mx = get_adjacency_matrix( 72 | distance_df_filename, num_of_vertices, id_filename=id_filename) 73 | # the self loop is missing 74 | add_self_loop = False 75 | if add_self_loop: 76 | print("adding self loop to adjacency matrices.") 77 | adj_mx = adj_mx + np.identity(adj_mx.shape[0]) 78 | distance_mx = distance_mx + np.identity(distance_mx.shape[0]) 79 | else: 80 | print("kindly note that there is no self loop in adjacency matrices.") 81 | with open("D:/myfile/BasicTS/datasets/raw_data/PEMS08/adj_PEMS08.pkl", "wb") as f: 82 | pickle.dump(adj_mx, f) 83 | with open("D:/myfile/BasicTS/datasets/raw_data/PEMS08/adj_PEMS08_distance.pkl", "wb") as f: 84 | pickle.dump(distance_mx, f) 85 | -------------------------------------------------------------------------------- /DFDGCN/scripts/data_preparation/PEMS08/generate_training_data.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import shutil 4 | import pickle 5 | import argparse 6 | 7 | import numpy as np 8 | 9 | from generate_adj_mx import generate_adj_pems08 10 | # TODO: remove it when basicts can be installed by pip 11 | sys.path.append(os.path.abspath(__file__ + "/../../../..")) 12 | from basicts.data.transform import standard_transform 13 | 14 | 15 | def generate_data(args: argparse.Namespace): 16 | """Preprocess and generate train/valid/test datasets. 17 | Default settings of PEMS08 dataset: 18 | - Normalization method: standard norm. 19 | - Dataset division: 6:2:2. 20 | - Window size: history 12, future 12. 21 | - Channels (features): three channels [traffic flow, time of day, day of week] 22 | - Target: predict the traffic flow of the future 12 time steps. 23 | 24 | Args: 25 | args (argparse): configurations of preprocessing 26 | """ 27 | 28 | target_channel = args.target_channel 29 | future_seq_len = args.future_seq_len 30 | history_seq_len = args.history_seq_len 31 | add_time_of_day = args.tod 32 | add_day_of_week = args.dow 33 | output_dir = args.output_dir 34 | train_ratio = args.train_ratio 35 | valid_ratio = args.valid_ratio 36 | data_file_path = args.data_file_path 37 | graph_file_path = args.graph_file_path 38 | steps_per_day = args.steps_per_day 39 | 40 | # read data 41 | data = np.load(data_file_path)["data"] 42 | data = data[..., target_channel] 43 | print("raw time series shape: {0}".format(data.shape)) 44 | 45 | l, n, f = data.shape 46 | num_samples = l - (history_seq_len + future_seq_len) + 1 47 | train_num_short = round(num_samples * train_ratio) 48 | valid_num_short = round(num_samples * valid_ratio) 49 | test_num_short = num_samples - train_num_short - valid_num_short 50 | print("number of training samples:{0}".format(train_num_short)) 51 | print("number of validation samples:{0}".format(valid_num_short)) 52 | print("number of test samples:{0}".format(test_num_short)) 53 | 54 | index_list = [] 55 | for t in range(history_seq_len, num_samples + history_seq_len): 56 | index = (t-history_seq_len, t, t+future_seq_len) 57 | index_list.append(index) 58 | 59 | train_index = index_list[:train_num_short] 60 | valid_index = index_list[train_num_short: train_num_short + valid_num_short] 61 | test_index = index_list[train_num_short + 62 | valid_num_short: train_num_short + valid_num_short + test_num_short] 63 | 64 | scaler = standard_transform 65 | data_norm = scaler(data, output_dir, train_index, history_seq_len, future_seq_len) 66 | 67 | # add external feature 68 | feature_list = [data_norm] 69 | if add_time_of_day: 70 | # numerical time_of_day 71 | tod = [i % steps_per_day / 72 | steps_per_day for i in range(data_norm.shape[0])] 73 | tod = np.array(tod) 74 | tod_tiled = np.tile(tod, [1, n, 1]).transpose((2, 1, 0)) 75 | feature_list.append(tod_tiled) 76 | 77 | if add_day_of_week: 78 | # numerical day_of_week 79 | dow = [(i // steps_per_day) % 7 for i in range(data_norm.shape[0])] 80 | dow = np.array(dow) 81 | dow_tiled = np.tile(dow, [1, n, 1]).transpose((2, 1, 0)) 82 | feature_list.append(dow_tiled) 83 | 84 | processed_data = np.concatenate(feature_list, axis=-1) 85 | 86 | # dump data 87 | index = {} 88 | index["train"] = train_index 89 | index["valid"] = valid_index 90 | index["test"] = test_index 91 | with open(output_dir + "/index_in{0}_out{1}.pkl".format(history_seq_len, future_seq_len), "wb") as f: 92 | pickle.dump(index, f) 93 | 94 | data = {} 95 | data["processed_data"] = processed_data 96 | with open(output_dir + "/data_in{0}_out{1}.pkl".format(history_seq_len, future_seq_len), "wb") as f: 97 | pickle.dump(data, f) 98 | # copy adj 99 | if os.path.exists(args.graph_file_path): 100 | # copy 101 | shutil.copyfile(args.graph_file_path, output_dir + "/adj_mx.pkl") 102 | else: 103 | # generate and copy 104 | generate_adj_pems08() 105 | shutil.copyfile(graph_file_path, output_dir + "/adj_mx.pkl") 106 | 107 | 108 | if __name__ == "__main__": 109 | # sliding window size for generating history sequence and target sequence 110 | HISTORY_SEQ_LEN = 12 111 | FUTURE_SEQ_LEN = 12 112 | 113 | TRAIN_RATIO = 0.6 114 | VALID_RATIO = 0.2 115 | TARGET_CHANNEL = [0] # target channel(s) 116 | STEPS_PER_DAY = 288 117 | 118 | DATASET_NAME = "PEMS08" 119 | TOD = True # if add time_of_day feature 120 | DOW = True # if add day_of_week feature 121 | OUTPUT_DIR = "datasets/" + DATASET_NAME 122 | DATA_FILE_PATH = "datasets/raw_data/{0}/{0}.npz".format(DATASET_NAME) 123 | GRAPH_FILE_PATH = "datasets/raw_data/{0}/adj_{0}.pkl".format(DATASET_NAME) 124 | 125 | parser = argparse.ArgumentParser() 126 | parser.add_argument("--output_dir", type=str, 127 | default=OUTPUT_DIR, help="Output directory.") 128 | parser.add_argument("--data_file_path", type=str, 129 | default=DATA_FILE_PATH, help="Raw traffic readings.") 130 | parser.add_argument("--graph_file_path", type=str, 131 | default=GRAPH_FILE_PATH, help="Raw traffic readings.") 132 | parser.add_argument("--history_seq_len", type=int, 133 | default=HISTORY_SEQ_LEN, help="Sequence Length.") 134 | parser.add_argument("--future_seq_len", type=int, 135 | default=FUTURE_SEQ_LEN, help="Sequence Length.") 136 | parser.add_argument("--steps_per_day", type=int, 137 | default=STEPS_PER_DAY, help="Sequence Length.") 138 | parser.add_argument("--tod", type=bool, default=TOD, 139 | help="Add feature time_of_day.") 140 | parser.add_argument("--dow", type=bool, default=DOW, 141 | help="Add feature day_of_week.") 142 | parser.add_argument("--target_channel", type=list, 143 | default=TARGET_CHANNEL, help="Selected channels.") 144 | parser.add_argument("--train_ratio", type=float, 145 | default=TRAIN_RATIO, help="Train ratio") 146 | parser.add_argument("--valid_ratio", type=float, 147 | default=VALID_RATIO, help="Validate ratio.") 148 | args_metr = parser.parse_args() 149 | 150 | # print args 151 | print("-"*(20+45+5)) 152 | for key, value in sorted(vars(args_metr).items()): 153 | print("|{0:>20} = {1:<45}|".format(key, str(value))) 154 | print("-"*(20+45+5)) 155 | 156 | if os.path.exists(args_metr.output_dir): 157 | reply = str(input( 158 | f"{args_metr.output_dir} exists. Do you want to overwrite it? (y/n)")).lower().strip() 159 | if reply[0] != "y": 160 | sys.exit(0) 161 | else: 162 | os.makedirs(args_metr.output_dir) 163 | generate_data(args_metr) 164 | -------------------------------------------------------------------------------- /DFDGCN/scripts/data_visualization/stid_example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [] 9 | } 10 | ], 11 | "metadata": { 12 | "kernelspec": { 13 | "display_name": "Python 3.9.13 ('BasicTS')", 14 | "language": "python", 15 | "name": "python3" 16 | }, 17 | "language_info": { 18 | "name": "python", 19 | "version": "3.9.13" 20 | }, 21 | "orig_nbformat": 4, 22 | "vscode": { 23 | "interpreter": { 24 | "hash": "570e84022352e67d0b4c22ae3a496a4293591fb172aae1ab664241200ab519d2" 25 | } 26 | } 27 | }, 28 | "nbformat": 4, 29 | "nbformat_minor": 2 30 | } 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DFDGCN 2 | Dynamic Frequency Domain Graph Convolutional Network for Traffic Forecasting 3 | 4 | [![preprint](https://img.shields.io/static/v1?label=arXiv&message=2402.03885&color=B31B1B&logo=arXiv)](https://arxiv.org/abs/2312.11933 "DFDGCN") 5 | 6 | 7 | **This paper has been accepted by ICASSP2024.** 8 | 9 | We appreciate you reviewing our work and you can use it if you want to cite it: 10 | 11 | ```bibtex 12 | @inproceedings{li2024dynamic, 13 | title={Dynamic Frequency Domain Graph Convolutional Network for Traffic Forecasting}, 14 | author={Li, Yujie and Shao, Zezhi and Xu, Yongjun and Qiu, Qiang and Cao, Zhaogang and Wang, Fei}, 15 | booktitle={ICASSP 2024-2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, 16 | pages={5245--5249}, 17 | year={2024}, 18 | organization={IEEE} 19 | } 20 | ``` 21 | 22 | **Acknowledgement:** 23 | 24 | Our work is developed based on BasicTS, if you are interested in more work on time series forecasting, you can refer to this Standard and Fair Time Series Forecasting Benchmark and Toolkit: https://github.com/GestaltCogTeam/BasicTS 25 | 26 | [![Github](https://img.shields.io/badge/GitHub-100000?style=for-the-badge&logo=github&logoColor=white)](https://github.com/GestaltCogTeam/BasicTS) 27 | 28 | If you have questions please email liyujie23s@ict.ac.cn or liyujie231@mails.ucas.ac.cn 29 | --------------------------------------------------------------------------------