├── CALF ├── README.md ├── data_provider │ ├── __pycache__ │ │ ├── data_factory.cpython-39.pyc │ │ ├── data_loader.cpython-39.pyc │ │ ├── m4.cpython-39.pyc │ │ └── uea.cpython-39.pyc │ ├── data_factory.py │ ├── data_loader.py │ ├── m4.py │ └── uea.py ├── exp │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-39.pyc │ │ ├── ablUtils.cpython-39.pyc │ │ ├── exp_anomaly_detection.cpython-39.pyc │ │ ├── exp_basic.cpython-39.pyc │ │ ├── exp_classification.cpython-39.pyc │ │ ├── exp_imputation.cpython-39.pyc │ │ ├── exp_long_term_forecasting.cpython-39.pyc │ │ └── exp_short_term_forecasting.cpython-39.pyc │ ├── ablUtils.py │ ├── exp_anomaly_detection.py │ ├── exp_basic.py │ ├── exp_classification.py │ ├── exp_imputation.py │ ├── exp_long_term_forecasting.py │ └── exp_short_term_forecasting.py ├── models │ ├── Attention.py │ ├── Embed.py │ ├── GPT2_arch.py │ ├── GPT4TS.py │ ├── GPT4TS_ETT.py │ └── __pycache__ │ │ ├── Attention.cpython-39.pyc │ │ ├── Embed.cpython-39.pyc │ │ ├── GPT2_arch.cpython-39.pyc │ │ ├── GPT4TS.cpython-39.pyc │ │ └── GPT4TS_ETT.cpython-39.pyc ├── run.py ├── scripts │ └── long_term_forecasting │ │ ├── ETTh_GPT2.sh │ │ ├── ETTm_GPT2.sh │ │ ├── electricity.sh │ │ ├── traffic.sh │ │ └── weather.sh ├── utils │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-39.pyc │ │ ├── distillationLoss.cpython-39.pyc │ │ ├── ditill_utils.cpython-39.pyc │ │ ├── losses.cpython-39.pyc │ │ ├── m4_summary.cpython-39.pyc │ │ ├── metrics.cpython-39.pyc │ │ ├── print_args.cpython-39.pyc │ │ ├── timefeatures.cpython-39.pyc │ │ └── tools.cpython-39.pyc │ ├── distillationLoss.py │ ├── ditill_utils.py │ ├── losses.py │ ├── m4_summary.py │ ├── metrics.py │ ├── print_args.py │ ├── prompts.py │ ├── timefeatures.py │ └── tools.py └── wte_pca_500.pt ├── OFA ├── README.md ├── data_provider │ ├── __pycache__ │ │ ├── data_factory.cpython-39.pyc │ │ └── data_loader.cpython-39.pyc │ ├── data_factory.py │ └── data_loader.py ├── embed.py ├── main.py ├── models │ ├── Attention.py │ ├── DLinear.py │ ├── DLinear_plus.py │ ├── GPT4TS.py │ ├── GPT4TS_orig.py │ ├── NLinear.py │ ├── PatchTST.py │ └── __pycache__ │ │ ├── Attention.cpython-39.pyc │ │ ├── DLinear.cpython-39.pyc │ │ ├── DLinear_plus.cpython-39.pyc │ │ ├── GPT4TS.cpython-39.pyc │ │ ├── NLinear.cpython-39.pyc │ │ └── PatchTST.cpython-39.pyc ├── script │ ├── ETTh_GPT2.sh │ ├── ETTh_simple.sh │ ├── ETTm_GPT2.sh │ ├── ETTm_simple.sh │ ├── electricity.sh │ ├── illness.sh │ ├── simple │ │ ├── ETTh_simple.sh │ │ ├── ETTm_simple.sh │ │ ├── electricity.sh │ │ ├── illness.sh │ │ ├── traffic.sh │ │ └── weather.sh │ ├── traffic.sh │ └── weather.sh └── utils │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-39.pyc │ ├── ablUtils.cpython-39.pyc │ ├── metrics.cpython-39.pyc │ ├── timefeatures.cpython-39.pyc │ └── tools.cpython-39.pyc │ ├── ablUtils.py │ ├── metrics.py │ ├── timefeatures.py │ └── tools.py ├── PAttn ├── __pycache__ │ └── embed.cpython-311.pyc ├── data_provider │ ├── __pycache__ │ │ ├── data_factory.cpython-311.pyc │ │ ├── data_factory.cpython-39.pyc │ │ ├── data_loader.cpython-311.pyc │ │ └── data_loader.cpython-39.pyc │ ├── data_factory.py │ └── data_loader.py ├── embed.py ├── main.py ├── models │ ├── Attention.py │ ├── PAttn.py │ ├── PatchTST.py │ └── __pycache__ │ │ ├── Attention.cpython-311.pyc │ │ ├── Attention.cpython-39.pyc │ │ ├── DLinear.cpython-39.pyc │ │ ├── DLinear_plus.cpython-39.pyc │ │ ├── GPT4TS.cpython-39.pyc │ │ ├── NLinear.cpython-39.pyc │ │ ├── PAttn.cpython-311.pyc │ │ ├── PatchTST.cpython-311.pyc │ │ └── PatchTST.cpython-39.pyc ├── script │ ├── ETTh.sh │ ├── ETTm.sh │ ├── electricity.sh │ ├── illness.sh │ ├── traffic.sh │ └── weather.sh └── utils │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-311.pyc │ ├── __init__.cpython-39.pyc │ ├── ablUtils.cpython-311.pyc │ ├── ablUtils.cpython-39.pyc │ ├── metrics.cpython-311.pyc │ ├── metrics.cpython-39.pyc │ ├── timefeatures.cpython-311.pyc │ ├── timefeatures.cpython-39.pyc │ ├── tools.cpython-311.pyc │ └── tools.cpython-39.pyc │ ├── ablUtils.py │ ├── metrics.py │ ├── timefeatures.py │ └── tools.py ├── README.md ├── Time-LLM-exp ├── LICENSE ├── README.md ├── data_provider │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-39.pyc │ │ ├── data_factory.cpython-39.pyc │ │ ├── data_loader.cpython-39.pyc │ │ └── m4.cpython-39.pyc │ ├── data_factory.py │ ├── data_loader.py │ └── m4.py ├── ds_config_zero2.json ├── layers │ ├── AutoCorrelation.py │ ├── Autoformer_EncDec.py │ ├── Conv_Blocks.py │ ├── Embed.py │ ├── SelfAttention_Family.py │ ├── StandardNorm.py │ ├── Transformer_EncDec.py │ ├── __init__.py │ └── __pycache__ │ │ ├── AutoCorrelation.cpython-39.pyc │ │ ├── Autoformer_EncDec.cpython-39.pyc │ │ ├── Embed.cpython-39.pyc │ │ ├── StandardNorm.cpython-39.pyc │ │ └── __init__.cpython-39.pyc ├── models │ ├── Attention.py │ ├── TimeLLM.py │ ├── __init__.py │ └── __pycache__ │ │ ├── Attention.cpython-39.pyc │ │ ├── Autoformer.cpython-39.pyc │ │ ├── DLinear.cpython-39.pyc │ │ ├── TimeLLM.cpython-39.pyc │ │ ├── TimeLLM_eval.cpython-39.pyc │ │ └── __init__.cpython-39.pyc ├── requirements.txt ├── run_main.py ├── scripts │ ├── train_ori │ │ ├── TimeLLM_ECL.sh │ │ ├── TimeLLM_ETTh1.sh │ │ ├── TimeLLM_ETTh1_ETTh2.sh │ │ ├── TimeLLM_ETTh2.sh │ │ ├── TimeLLM_ETTm1.sh │ │ ├── TimeLLM_ETTm2.sh │ │ ├── TimeLLM_M4.sh │ │ ├── TimeLLM_Traffic.sh │ │ └── TimeLLM_Weather.sh │ └── train_script │ │ ├── TimeLLM_ECL.sh │ │ ├── TimeLLM_ETTh1.sh │ │ ├── TimeLLM_ETTh2.sh │ │ ├── TimeLLM_ETTm1.sh │ │ ├── TimeLLM_ETTm2.sh │ │ ├── TimeLLM_Traffic.sh │ │ ├── TimeLLM_Weather.sh │ │ └── illness.sh └── utils │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-39.pyc │ ├── timefeatures.cpython-39.pyc │ └── tools.cpython-39.pyc │ ├── losses.py │ ├── m4_summary.py │ ├── masking.py │ ├── metrics.py │ ├── timefeatures.py │ └── tools.py └── pic └── ablations.png /CALF/README.md: -------------------------------------------------------------------------------- 1 | # Taming Pre-trained LLMs for Generalized Time Series Forecasting through Cross-modal Knowledge Distillation 2 | 3 | [![](http://img.shields.io/badge/cs.LG-arXiv%3A2403.07300-B31B1B.svg)](https://arxiv.org/abs/2403.07300) 4 | 5 | 6 | ## Prerequisites 7 | Before proceeding, ensure Python 3.9 is installed. Install the required dependencies with the following command: 8 | 9 | ``` 10 | pip install -r requirements.txt 11 | ``` 12 | 13 | ## Dataset Preparation 14 | 15 | ### Long-term Forecasting 16 | Acquire datasets from [Autoformer](https://drive.google.com/drive/folders/1ZOYpTUa82_jCcxIdTmyr0LXQfvaM9vIy). Organize them in the `./datasets` directory as shown below: 17 | 18 | ``` 19 | datasets 20 | ├── electricity 21 | │ └── electricity.csv 22 | ├── ETT-small 23 | │ ├── ETTh1.csv 24 | │ ├── ETTh2.csv 25 | │ ├── ETTm1.csv 26 | │ └── ETTm2.csv 27 | ├── traffic 28 | │ └── traffic.csv 29 | └── weather 30 | └── weather.csv 31 | ``` 32 | 33 | ### Short-term Forecasting 34 | For short-term forecasting, download the M4 datasets from [Time-Series-Library](https://drive.google.com/drive/folders/15zio96o3NK4XOoR5L88oaWcJDVOiqQo9). Place the `m4` folder within `./datasets`. 35 | 36 | ## Preparing Word Token Embeddings 37 | 38 | Execute the command below to extract principal components from the word token embeddings: 39 | 40 | ``` 41 | python pca.py 42 | ``` 43 | 44 | These embeddings will be saved in `./wte_pca_500.pt`. 45 | 46 | ## Model Training 47 | 48 | Training scripts are located in the `./scripts` folder. For instance, to train the LLaTA model on the ETTh2 dataset for long-term forecasting, execute: 49 | 50 | ``` 51 | sh scripts/long_term_forecasting/ETTh2.sh 52 | ``` 53 | 54 | For short-term forecasting, use: 55 | 56 | ``` 57 | sh scripts/short_term_forecasting/m4.sh 58 | ``` 59 | 60 | Post-Training: 61 | 62 | - Trained models will be saved in `./checkpoints`. 63 | - Numerical results are available in `.npy` format under `./results`. 64 | - Detailed summaries of performance metrics can be found in `./results_{task_name}.txt`. 65 | 66 | ## Citation 67 | If this repository contributes to your research, please consider citing our work: 68 | 69 | ``` 70 | @article{liu2024taming, 71 | title={Taming Pre-trained LLMs for Generalised Time Series Forecasting via Cross-modal Knowledge Distillation}, 72 | author={Liu, Peiyuan and Guo, Hang and Dai, Tao and Li, Naiqi and Bao, Jigang and Ren, Xudong and Jiang, Yong and Xia, Shu-Tao}, 73 | journal={arXiv preprint arXiv:2403.07300}, 74 | year={2024}, 75 | arxiv={2403.07300} 76 | } 77 | ``` 78 | 79 | ## Acknowledgements 80 | 81 | Our gratitude extends to the authors of the following repositories for their foundational model implementations: 82 | 83 | - [Autoformer](https://github.com/thuml/Autoformer) 84 | - [Time-Series-Library](https://github.com/thuml/Time-Series-Library) 85 | - [PatchTST](https://github.com/yuqinie98/PatchTST) 86 | - [Crossformer](https://github.com/Thinklab-SJTU/Crossformer) 87 | - [One-Fits-All](https://github.com/DAMO-DI-ML/NeurIPS2023-One-Fits-All) 88 | 89 | ## Contact Us 90 | For inquiries or further assistance, contact us at [lpy23@mails.tsinghua.edu.cn](mailto:lpy23@mails.tsinghua.edu.cn) or open an issue on this repository. 91 | -------------------------------------------------------------------------------- /CALF/data_provider/__pycache__/data_factory.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/data_provider/__pycache__/data_factory.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/data_provider/__pycache__/data_loader.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/data_provider/__pycache__/data_loader.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/data_provider/__pycache__/m4.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/data_provider/__pycache__/m4.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/data_provider/__pycache__/uea.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/data_provider/__pycache__/uea.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/data_provider/data_factory.py: -------------------------------------------------------------------------------- 1 | from data_provider.data_loader import Dataset_ETT_hour, Dataset_ETT_minute, Dataset_Custom, Dataset_M4, PSMSegLoader, \ 2 | MSLSegLoader, SMAPSegLoader, SMDSegLoader, SWATSegLoader, UEAloader 3 | from data_provider.uea import collate_fn 4 | from torch.utils.data import DataLoader 5 | 6 | data_dict = { 7 | 'ETTh1': Dataset_ETT_hour, 8 | 'ETTh2': Dataset_ETT_hour, 9 | 'ETTm1': Dataset_ETT_minute, 10 | 'ETTm2': Dataset_ETT_minute, 11 | 'custom': Dataset_Custom, 12 | 'm4': Dataset_M4, 13 | 'PSM': PSMSegLoader, 14 | 'MSL': MSLSegLoader, 15 | 'SMAP': SMAPSegLoader, 16 | 'SMD': SMDSegLoader, 17 | 'SWAT': SWATSegLoader, 18 | 'UEA': UEAloader 19 | } 20 | 21 | def data_provider(args, flag, vali=False): 22 | Data = data_dict[args.data] 23 | timeenc = 0 if args.embed != 'timeF' else 1 24 | 25 | if flag == 'test': 26 | shuffle_flag = False 27 | drop_last = True 28 | if args.task_name == 'anomaly_detection' or args.task_name == 'classification': 29 | batch_size = args.batch_size 30 | else: 31 | batch_size = args.batch_size 32 | freq = args.freq 33 | else: 34 | shuffle_flag = True 35 | drop_last = True 36 | batch_size = args.batch_size # bsz for train and valid 37 | freq = args.freq 38 | 39 | if args.task_name == 'anomaly_detection': 40 | drop_last = False 41 | data_set = Data( 42 | root_path=args.root_path, 43 | win_size=args.seq_len, 44 | flag=flag, 45 | ) 46 | print(flag, len(data_set)) 47 | data_loader = DataLoader( 48 | data_set, 49 | batch_size=batch_size, 50 | shuffle=shuffle_flag, 51 | num_workers=args.num_workers, 52 | drop_last=drop_last) 53 | return data_set, data_loader 54 | elif args.task_name == 'classification': 55 | drop_last = False 56 | data_set = Data( 57 | root_path=args.root_path, 58 | flag=flag, 59 | ) 60 | 61 | data_loader = DataLoader( 62 | data_set, 63 | batch_size=batch_size, 64 | shuffle=shuffle_flag, 65 | num_workers=args.num_workers, 66 | drop_last=drop_last, 67 | collate_fn=lambda x: collate_fn(x, max_len=args.seq_len) 68 | ) 69 | return data_set, data_loader 70 | else: 71 | if args.data == 'm4': 72 | drop_last = False 73 | data_set = Data( 74 | # train_ratio=args.train_ratio, 75 | root_path=args.root_path, 76 | data_path=args.data_path, 77 | flag=flag, 78 | model_id = args.model_id , 79 | size=[args.seq_len, args.label_len, args.pred_len], 80 | features=args.features, 81 | target=args.target, 82 | timeenc=timeenc, 83 | freq=freq, 84 | seasonal_patterns=args.seasonal_patterns, 85 | percent=args.percent 86 | ) 87 | print(flag, len(data_set)) 88 | data_loader = DataLoader( 89 | data_set, 90 | batch_size=batch_size, 91 | shuffle=shuffle_flag, 92 | num_workers=args.num_workers, 93 | drop_last=drop_last) 94 | return data_set, data_loader 95 | -------------------------------------------------------------------------------- /CALF/exp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/exp/__init__.py -------------------------------------------------------------------------------- /CALF/exp/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/exp/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/exp/__pycache__/ablUtils.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/exp/__pycache__/ablUtils.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/exp/__pycache__/exp_anomaly_detection.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/exp/__pycache__/exp_anomaly_detection.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/exp/__pycache__/exp_basic.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/exp/__pycache__/exp_basic.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/exp/__pycache__/exp_classification.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/exp/__pycache__/exp_classification.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/exp/__pycache__/exp_imputation.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/exp/__pycache__/exp_imputation.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/exp/__pycache__/exp_long_term_forecasting.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/exp/__pycache__/exp_long_term_forecasting.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/exp/__pycache__/exp_short_term_forecasting.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/exp/__pycache__/exp_short_term_forecasting.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/exp/ablUtils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | from torch import optim 5 | import os 6 | import matplotlib.pyplot as plt 7 | import numpy as np 8 | import argparse 9 | import random 10 | 11 | def perturb_sequence(batch_x , shuffle_type , patch_size = 16 , mask_ratio= 0.2 ): 12 | ''' 13 | batch_x : shape [256, 336, 1] 14 | perturb time series input 15 | sf_all : shuffle the whole sequnece 16 | sf_half : shuffle first halp sequnece 17 | ex-half : exchange first and second half 18 | ''' 19 | assert shuffle_type in ['sf_all' , 'sf_half' , 'ex_half' ,'sf_patchs' , 'masking'] 20 | if shuffle_type == 'sf_all': 21 | perm = torch.randperm(batch_x.size(1)) 22 | return batch_x[:, perm, :] 23 | if shuffle_type == 'sf_half': 24 | mid_point = batch_x.size(1) // 2 25 | pre_half = batch_x[:, :mid_point, :] 26 | post_half = batch_x[:, mid_point:, :] 27 | perm = torch.randperm(pre_half.size(1)) 28 | shuffled_pre_half = pre_half[:, perm, :] 29 | return torch.cat((shuffled_pre_half, post_half), dim=1) 30 | if shuffle_type == 'ex_half': 31 | mid_point = batch_x.size(1) // 2 32 | pre_half = batch_x[:, :mid_point, :] 33 | post_half = batch_x[:, mid_point:, :] 34 | return torch.cat((post_half, pre_half), dim=1) 35 | if shuffle_type =='sf_patchs': 36 | num_patches= (batch_x.size(1) // patch_size ) 37 | shuffle_indices = torch.randperm(num_patches) 38 | shuffled_ts = [batch_x[:, i*patch_size:(i+1)*patch_size, :] for i in shuffle_indices] 39 | if num_patches * patch_size < batch_x.size(1): 40 | shuffled_ts.append(batch_x[:, num_patches*patch_size:, :]) 41 | return torch.cat(shuffled_ts , dim=1) 42 | if shuffle_type =='masking': 43 | input_length = batch_x.size(1) 44 | num_to_mask = int(input_length * mask_ratio ) 45 | mask_indices = torch.randperm(input_length)[:num_to_mask] 46 | masked_tensor = batch_x.clone() 47 | masked_tensor[:, mask_indices, :] = 0 48 | return masked_tensor 49 | 50 | # arrayTS = torch.rand(1, 32, 1) 51 | # print(arrayTS[0,:,0]) 52 | # arrayTS = perturb_sequence(arrayTS , 'masking' , patch_size = 4 , mask_ratio= 0.8 ) 53 | # print(arrayTS[0,:,0]) 54 | -------------------------------------------------------------------------------- /CALF/exp/exp_basic.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | from models import GPT4TS , GPT4TS_ETT 4 | 5 | class Exp_Basic(object): 6 | def __init__(self, args): 7 | self.args = args 8 | self.model_dict = { 9 | "GPT4TS": GPT4TS, 10 | } 11 | self.device = self._acquire_device() 12 | self.model = self._build_model().to(self.device) 13 | 14 | def _build_model(self): 15 | raise NotImplementedError 16 | return None 17 | 18 | def _acquire_device(self): 19 | if self.args.use_gpu: 20 | os.environ["CUDA_VISIBLE_DEVICES"] = str( 21 | self.args.gpu) if not self.args.use_multi_gpu else self.args.devices 22 | device = torch.device('cuda:{}'.format(self.args.gpu)) 23 | print('Use GPU: cuda:{}'.format(self.args.gpu)) 24 | else: 25 | device = torch.device('cpu') 26 | print('Use CPU') 27 | return device 28 | 29 | def _get_data(self): 30 | pass 31 | 32 | def vali(self): 33 | pass 34 | 35 | def train(self): 36 | pass 37 | 38 | def test(self): 39 | pass 40 | -------------------------------------------------------------------------------- /CALF/models/Attention.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | class ScaledDotProductAttention(nn.Module): 6 | ''' Scaled Dot-Product Attention ''' 7 | 8 | def __init__(self, temperature, attn_dropout=0.1): 9 | super().__init__() 10 | self.temperature = temperature 11 | self.dropout = nn.Dropout(attn_dropout) 12 | 13 | def forward(self, q, k, v, mask=None): 14 | 15 | attn = torch.matmul(q / self.temperature, k.transpose(2, 3)) 16 | 17 | if mask is not None: 18 | attn = attn.masked_fill(mask == 0, -1e9) 19 | 20 | attn = self.dropout(F.softmax(attn, dim=-1)) 21 | output = torch.matmul(attn, v) 22 | 23 | return output, attn 24 | 25 | class MultiHeadAttention(nn.Module): 26 | ''' Multi-Head Attention module ''' 27 | def __init__(self, d_model = -1 ,n_head = 8 , d_k = -1 , d_v = -1 , dropout=0.1): 28 | super().__init__() 29 | self.n_head = n_head 30 | d_k = d_model // n_head 31 | d_v = d_k 32 | self.d_k = d_k 33 | self.d_v = d_v 34 | 35 | self.w_qs = nn.Linear(d_model, n_head * d_k, bias=False) 36 | self.w_ks = nn.Linear(d_model, n_head * d_k, bias=False) 37 | self.w_vs = nn.Linear(d_model, n_head * d_v, bias=False) 38 | self.fc = nn.Linear(n_head * d_v, d_model, bias=False) 39 | 40 | self.attention = ScaledDotProductAttention(temperature=d_k ** 0.5) 41 | 42 | self.dropout = nn.Dropout(dropout) 43 | self.layer_norm = nn.LayerNorm(d_model, eps=1e-6) 44 | 45 | 46 | def forward(self, q, k, v, mask=None): 47 | 48 | d_k, d_v, n_head = self.d_k, self.d_v, self.n_head 49 | sz_b, len_q, len_k, len_v = q.size(0), q.size(1), k.size(1), v.size(1) 50 | 51 | residual = q 52 | 53 | # Pass through the pre-attention projection: b x lq x (n*dv) 54 | # Separate different heads: b x lq x n x dv 55 | q = self.w_qs(q).view(sz_b, len_q, n_head, d_k) 56 | k = self.w_ks(k).view(sz_b, len_k, n_head, d_k) 57 | v = self.w_vs(v).view(sz_b, len_v, n_head, d_v) 58 | 59 | # Transpose for attention dot product: b x n x lq x dv 60 | q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2) 61 | 62 | if mask is not None: 63 | mask = mask.unsqueeze(1) # For head axis broadcasting. 64 | 65 | q, attn = self.attention(q, k, v, mask=mask) 66 | 67 | # Transpose to move the head dimension back: b x lq x n x dv 68 | # Combine the last two dimensions to concatenate all the heads together: b x lq x (n*dv) 69 | q = q.transpose(1, 2).contiguous().view(sz_b, len_q, -1) 70 | q = self.dropout(self.fc(q)) 71 | q += residual 72 | 73 | q = self.layer_norm(q) 74 | 75 | return q, attn -------------------------------------------------------------------------------- /CALF/models/__pycache__/Attention.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/models/__pycache__/Attention.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/models/__pycache__/Embed.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/models/__pycache__/Embed.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/models/__pycache__/GPT2_arch.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/models/__pycache__/GPT2_arch.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/models/__pycache__/GPT4TS.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/models/__pycache__/GPT4TS.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/models/__pycache__/GPT4TS_ETT.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/models/__pycache__/GPT4TS_ETT.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/scripts/long_term_forecasting/ETTh_GPT2.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | file_name='ETTh_drop.txt' 3 | seq_len=96 4 | bootstrap_eval=0 5 | model=GPT4TS 6 | methods_h="ori dropAttn_keepWE llm_to_attn llm_to_trsf" 7 | pre_lens_h="96 192 336 720" 8 | gpt_loc=0 9 | itt=5 10 | # sh scripts/long_term_forecasting/ETTh_GPT2.sh 11 | echo $bootstrap_eval 12 | for pred_len in $pre_lens_h; 13 | do 14 | for eval_target in $methods_h; 15 | do 16 | python run.py \ 17 | --root_path ./datasets/ETT-small/ \ 18 | --data_path ETTh1.csv \ 19 | --is_training 1 \ 20 | --task_name long_term_forecast \ 21 | --model_id ETTh1'_test_'$seq_len'_'$pred_len'_'$eval_target \ 22 | --data ETTh1 \ 23 | --seq_len $seq_len \ 24 | --label_len 0 \ 25 | --pred_len $pred_len \ 26 | --batch_size 256 \ 27 | --learning_rate 0.0005 \ 28 | --lradj type1 \ 29 | --train_epochs 100 \ 30 | --d_model 768 \ 31 | --n_heads 4 \ 32 | --d_ff 768 \ 33 | --dropout 0.3 \ 34 | --enc_in 7 \ 35 | --c_out 7 \ 36 | --gpt_layer 6 \ 37 | --bootstrap_eval $bootstrap_eval \ 38 | --itr $itt \ 39 | --gpu $gpt_loc \ 40 | --model $model \ 41 | --r 8 \ 42 | --lora_alpha 32 \ 43 | --lora_dropout 0.1 \ 44 | --patience 10 \ 45 | --log_fine_name $file_name 46 | echo '=====================================================================================================================' 47 | done 48 | done 49 | 50 | 51 | seq_len=96 52 | model=GPT4TS 53 | for pred_len in $pre_lens_h; 54 | do 55 | for eval_target in $methods_h; 56 | do 57 | python run.py \ 58 | --root_path ./datasets/ETT-small/ \ 59 | --data_path ETTh2.csv \ 60 | --is_training 1 \ 61 | --task_name long_term_forecast \ 62 | --model_id ETTh2'_'$seq_len'_'$pred_len'_'$eval_target \ 63 | --data ETTh2 \ 64 | --seq_len $seq_len \ 65 | --label_len 0 \ 66 | --pred_len $pred_len \ 67 | --batch_size 256 \ 68 | --learning_rate 0.0005 \ 69 | --lradj type1 \ 70 | --train_epochs 100 \ 71 | --d_model 768 \ 72 | --n_heads 4 \ 73 | --d_ff 768 \ 74 | --dropout 0.3 \ 75 | --enc_in 7 \ 76 | --c_out 7 \ 77 | --gpt_layers 6 \ 78 | --itr $itt \ 79 | --model $model \ 80 | --gpu $gpt_loc \ 81 | --bootstrap_eval $bootstrap_eval \ 82 | --tmax 20 \ 83 | --cos 1 \ 84 | --r 8 \ 85 | --lora_alpha 32 \ 86 | --lora_dropout 0.1 \ 87 | --patience 5 \ 88 | --log_fine_name $file_name 89 | echo '=====================================================================================================================' 90 | done 91 | done -------------------------------------------------------------------------------- /CALF/scripts/long_term_forecasting/ETTm_GPT2.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | file_name='ETTm2_drop.txt' 3 | methods_h="ori dropAttn_keepWE llm_to_attn llm_to_trsf" 4 | # pre_lens_h="720" 5 | pre_lens_h="96 192 336 720" 6 | gpt_loc=0 7 | itt=5 8 | # sh scripts/long_term_forecasting/ETTm_GPT2.sh 9 | 10 | bootstrap_eval=1 11 | seq_len=96 12 | model=GPT4TS 13 | 14 | for pred_len in $pre_lens_h; 15 | do 16 | for eval_target in $methods_h; 17 | do 18 | python run.py \ 19 | --root_path ./datasets/ETT-small/ \ 20 | --data_path ETTm1.csv \ 21 | --is_training 1 \ 22 | --task_name long_term_forecast \ 23 | --model_id ETTm1'_'$seq_len'_'$pred_len'_'$eval_target \ 24 | --data ETTm1 \ 25 | --seq_len $seq_len \ 26 | --label_len 0 \ 27 | --pred_len $pred_len \ 28 | --batch_size 256 \ 29 | --learning_rate 0.0005 \ 30 | --lradj type1 \ 31 | --train_epochs 100 \ 32 | --d_model 768 \ 33 | --n_heads 4 \ 34 | --d_ff 768 \ 35 | --dropout 0.3 \ 36 | --enc_in 7 \ 37 | --c_out 7 \ 38 | --gpt_layer 6 \ 39 | --itr $itt \ 40 | --bootstrap_eval $bootstrap_eval \ 41 | --model $model \ 42 | --cos 1 \ 43 | --gpu $gpt_loc \ 44 | --tmax 20 \ 45 | --r 8 \ 46 | --lora_alpha 32 \ 47 | --lora_dropout 0.1 \ 48 | --patience 5 \ 49 | --log_fine_name $file_name 50 | echo '=====================================================================================================================' 51 | done 52 | done 53 | 54 | exit 55 | 56 | pre_lens_h="96" 57 | seq_len=96 58 | model=GPT4TS 59 | for pred_len in $pre_lens_h; 60 | do 61 | for eval_target in $methods_h; 62 | do 63 | python run.py \ 64 | --root_path ./datasets/ETT-small/ \ 65 | --data_path ETTm2.csv \ 66 | --is_training 1 \ 67 | --task_name long_term_forecast \ 68 | --model_id ETTm2'_'$seq_len'_'$pred_len'_'$eval_target \ 69 | --data ETTm2 \ 70 | --seq_len $seq_len \ 71 | --label_len 0 \ 72 | --pred_len $pred_len \ 73 | --batch_size 256 \ 74 | --learning_rate 0.0001 \ 75 | --lradj type1 \ 76 | --train_epochs 100 \ 77 | --d_model 768 \ 78 | --n_heads 4 \ 79 | --d_ff 768 \ 80 | --dropout 0.3 \ 81 | --enc_in 7 \ 82 | --c_out 7 \ 83 | --gpu $gpt_loc \ 84 | --gpt_layer 6 \ 85 | --itr $itt \ 86 | --model $model \ 87 | --bootstrap_eval $bootstrap_eval \ 88 | --r 8 \ 89 | --lora_alpha 32 \ 90 | --lora_dropout 0.1 \ 91 | --patience 5 \ 92 | --log_fine_name $file_name 93 | echo '=====================================================================================================================' 94 | done 95 | done 96 | -------------------------------------------------------------------------------- /CALF/scripts/long_term_forecasting/electricity.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | model=GPT4TS 3 | gpu_loc=2 4 | bootstrap_eval=1 5 | methods_h="ori dropAttn_keepWE llm_to_attn llm_to_trsf" 6 | filename='electricity.txt' 7 | seq_len=96 8 | itt=3 9 | # sh scripts/long_term_forecasting/electricity.sh 10 | for pred_len in 96 192 336 720; 11 | do 12 | for eval_target in $methods_h; 13 | do 14 | if [ "$eval_target" = 'ori' ]; then 15 | lr=0.0005 16 | bs=32 17 | else 18 | lr=0.001 19 | bs=64 20 | fi 21 | echo $eval_target"_"$pred_len"_"$bs"_"$lr 22 | python run.py \ 23 | --root_path ./datasets/electricity/ \ 24 | --data_path electricity.csv \ 25 | --is_training 1 \ 26 | --task_name long_term_forecast \ 27 | --model_id Electricity_$seq_len'_'$pred_len'_'$eval_target \ 28 | --data custom \ 29 | --seq_len $seq_len \ 30 | --label_len 0 \ 31 | --pred_len $pred_len \ 32 | --batch_size $bs \ 33 | --learning_rate $lr \ 34 | --train_epochs 20 \ 35 | --d_model 768 \ 36 | --n_heads 4 \ 37 | --d_ff 768 \ 38 | --dropout 0.3 \ 39 | --enc_in 7 \ 40 | --c_out 7 \ 41 | --gpt_layer 6 \ 42 | --itr $itt \ 43 | --model $model \ 44 | --cos 1 \ 45 | --tmax 10 \ 46 | --r 8 \ 47 | --lora_alpha 32 \ 48 | --lora_dropout 0.1 \ 49 | --patience 5 \ 50 | --task_loss smooth_l1 \ 51 | --distill_loss smooth_l1 \ 52 | --logits_loss smooth_l1 \ 53 | --gpu $gpu_loc \ 54 | --bootstrap_eval $bootstrap_eval \ 55 | --log_fine_name $filename 56 | echo '=====================================================================================================================' 57 | done 58 | done 59 | 60 | 61 | exit 62 | # Adjust parameters : 63 | filename='electricity_tc.txt' 64 | seq_len=96 65 | itt=1 66 | lrs='0.0001 0.001' 67 | bss='32' 68 | # sh scripts/long_term_forecasting/electricity.sh 69 | for pred_len in 96 192 336 720; 70 | do 71 | for eval_target in $methods_h; 72 | do 73 | for lr in $lrs; 74 | do 75 | for bs in $bss; 76 | do 77 | echo $eval_target"_"$pred_len"_"$bs"_"$lr 78 | python run.py \ 79 | --root_path ./datasets/electricity/ \ 80 | --data_path electricity.csv \ 81 | --is_training 1 \ 82 | --task_name long_term_forecast \ 83 | --model_id Electricity'_'$seq_len'_'$pred_len'_'$eval_target"_"$bs"_"$lr \ 84 | --data custom \ 85 | --seq_len $seq_len \ 86 | --label_len 0 \ 87 | --pred_len $pred_len \ 88 | --batch_size $bs \ 89 | --learning_rate $lr \ 90 | --train_epochs 20 \ 91 | --d_model 768 \ 92 | --n_heads 4 \ 93 | --d_ff 768 \ 94 | --dropout 0.3 \ 95 | --enc_in 7 \ 96 | --c_out 7 \ 97 | --gpt_layer 6 \ 98 | --itr $itt \ 99 | --model $model \ 100 | --cos 1 \ 101 | --tmax 10 \ 102 | --r 8 \ 103 | --lora_alpha 32 \ 104 | --lora_dropout 0.1 \ 105 | --patience 5 \ 106 | --task_loss smooth_l1 \ 107 | --distill_loss smooth_l1 \ 108 | --logits_loss smooth_l1 \ 109 | --gpu $gpu_loc \ 110 | --log_fine_name $filename 111 | echo '=====================================================================================================================' 112 | done 113 | done 114 | done 115 | done 116 | -------------------------------------------------------------------------------- /CALF/scripts/long_term_forecasting/traffic.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | model=GPT4TS 3 | gpu_loc=2 4 | 5 | methods_h="ori dropAttn_keepWE llm_to_attn llm_to_trsf" 6 | filename='traffic_.txt' 7 | seq_len=96 8 | itt=3 9 | # 10 | # sh scripts/long_term_forecasting/traffic.sh 11 | for pred_len in 96 192 336 720; 12 | do 13 | for eval_target in $methods_h; 14 | do 15 | if [ "$eval_target" = 'ori' ]; then 16 | lr=0.0005 17 | bs=8 18 | else 19 | lr=0.001 20 | bs=32 21 | fi 22 | echo $pred_len"_"$bs"_"$lr 23 | python run.py \ 24 | --root_path ./datasets/traffic/ \ 25 | --data_path traffic.csv \ 26 | --is_training 1 \ 27 | --task_name long_term_forecast \ 28 | --model_id Traffic_$seq_len'_'$pred_len'_'$eval_target \ 29 | --data custom \ 30 | --seq_len $seq_len \ 31 | --label_len 0 \ 32 | --pred_len $pred_len \ 33 | --batch_size $bs \ 34 | --learning_rate $lr \ 35 | --train_epochs 10 \ 36 | --d_model 768 \ 37 | --n_heads 4 \ 38 | --d_ff 768 \ 39 | --gpt_layer 6 \ 40 | --itr $itt \ 41 | --model $model \ 42 | --cos 1 \ 43 | --tmax 10 \ 44 | --r 8 \ 45 | --lora_alpha 32 \ 46 | --lora_dropout 0.1 \ 47 | --patience 5 \ 48 | --task_loss smooth_l1 \ 49 | --distill_loss smooth_l1 \ 50 | --logits_loss smooth_l1 \ 51 | --gpu $gpu_loc \ 52 | --log_fine_name $filename 53 | echo '=====================================================================================================================' 54 | done 55 | done 56 | 57 | 58 | 59 | # methods_h="dropAttn_keepWE llm_to_attn llm_to_trsf" 60 | # filename='traffic_tc.txt' 61 | # seq_len=96 62 | # itt=1 63 | # lrs='0.00005 0.0001 0.001' 64 | # bss='8 64' 65 | # # sh scripts/long_term_forecasting/traffic.sh 66 | # for pred_len in 96 336; 67 | # do 68 | # for eval_target in $methods_h; 69 | # do 70 | # for lr in $lrs; 71 | # do 72 | # for bs in $bss; 73 | # do 74 | # echo $pred_len"_"$bs"_"$lr 75 | # python run.py \ 76 | # --root_path ./datasets/traffic/ \ 77 | # --data_path traffic.csv \ 78 | # --is_training 1 \ 79 | # --task_name long_term_forecast \ 80 | # --model_id Traffic_$seq_len'_'$pred_len'_'$eval_target"_"$bs"_"$lr \ 81 | # --data custom \ 82 | # --seq_len $seq_len \ 83 | # --label_len 0 \ 84 | # --pred_len $pred_len \ 85 | # --batch_size $bs \ 86 | # --learning_rate $lr \ 87 | # --train_epochs 10 \ 88 | # --d_model 768 \ 89 | # --n_heads 4 \ 90 | # --d_ff 768 \ 91 | # --gpt_layer 6 \ 92 | # --itr $itt \ 93 | # --model $model \ 94 | # --cos 1 \ 95 | # --tmax 10 \ 96 | # --r 8 \ 97 | # --lora_alpha 32 \ 98 | # --lora_dropout 0.1 \ 99 | # --patience 5 \ 100 | # --task_loss smooth_l1 \ 101 | # --distill_loss smooth_l1 \ 102 | # --logits_loss smooth_l1 \ 103 | # --gpu $gpu_loc \ 104 | # --log_fine_name $filename 105 | # echo '=====================================================================================================================' 106 | # done 107 | # done 108 | # done 109 | # done -------------------------------------------------------------------------------- /CALF/scripts/long_term_forecasting/weather.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | model=GPT4TS 3 | gpu_loc=1 4 | # ori 5 | 6 | methods_h="ori dropAttn_keepWE llm_to_attn llm_to_trsf" 7 | filename='weather.txt' 8 | seq_len=96 9 | itt=3 10 | # sh scripts/long_term_forecasting/weather.sh 11 | for pred_len in 96 192 336 720; 12 | do 13 | for eval_target in $methods_h; 14 | do 15 | if [ "$eval_target" = 'ori' ]; then 16 | lr=0.0005 17 | bs=64 18 | else 19 | lr=0.001 20 | bs=256 21 | fi 22 | python run.py \ 23 | --root_path ./datasets/weather/ \ 24 | --data_path weather.csv \ 25 | --is_training 1 \ 26 | --task_name long_term_forecast \ 27 | --model_id Weather'_'$seq_len'_'$pred_len'_'$eval_target \ 28 | --data custom \ 29 | --seq_len $seq_len \ 30 | --label_len 0 \ 31 | --pred_len $pred_len \ 32 | --batch_size $bs \ 33 | --learning_rate $lr \ 34 | --train_epochs 100 \ 35 | --d_model 768 \ 36 | --n_heads 4 \ 37 | --d_ff 768 \ 38 | --dropout 0.3 \ 39 | --enc_in 7 \ 40 | --c_out 7 \ 41 | --lradj type3 \ 42 | --gpt_layer 6 \ 43 | --itr $itt \ 44 | --model $model \ 45 | --r 8 \ 46 | --lora_alpha 32 \ 47 | --lora_dropout 0.1 \ 48 | --patience 5 \ 49 | --task_loss smooth_l1 \ 50 | --distill_loss smooth_l1 \ 51 | --logits_loss smooth_l1 \ 52 | --gpu $gpu_loc \ 53 | --log_fine_name $filename 54 | echo '=====================================================================================================================' 55 | done 56 | done 57 | 58 | 59 | methods_h="dropAttn_keepWE llm_to_attn llm_to_trsf" 60 | filename='weather_tc.txt' 61 | seq_len=96 62 | itt=1 63 | lrs='0.01' 64 | bss='256' 65 | # sh scripts/long_term_forecasting/weather.sh 66 | for pred_len in 96 192 336 720; 67 | do 68 | for eval_target in $methods_h; 69 | do 70 | for lr in $lrs; 71 | do 72 | for bs in $bss; 73 | do 74 | python run.py \ 75 | --root_path ./datasets/weather/ \ 76 | --data_path weather.csv \ 77 | --is_training 1 \ 78 | --task_name long_term_forecast \ 79 | --model_id Weather'_'$seq_len'_'$pred_len'_'$eval_target"_"$bs"_"$lr \ 80 | --data custom \ 81 | --seq_len $seq_len \ 82 | --label_len 0 \ 83 | --pred_len $pred_len \ 84 | --batch_size $bs \ 85 | --learning_rate $lr \ 86 | --train_epochs 100 \ 87 | --d_model 768 \ 88 | --n_heads 4 \ 89 | --d_ff 768 \ 90 | --dropout 0.3 \ 91 | --enc_in 7 \ 92 | --c_out 7 \ 93 | --lradj type3 \ 94 | --gpt_layer 6 \ 95 | --itr $itt \ 96 | --model $model \ 97 | --r 8 \ 98 | --lora_alpha 32 \ 99 | --lora_dropout 0.1 \ 100 | --patience 5 \ 101 | --task_loss smooth_l1 \ 102 | --distill_loss smooth_l1 \ 103 | --logits_loss smooth_l1 \ 104 | --gpu $gpu_loc \ 105 | --log_fine_name $filename 106 | echo '=====================================================================================================================' 107 | done 108 | done 109 | done 110 | done 111 | 112 | -------------------------------------------------------------------------------- /CALF/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/utils/__init__.py -------------------------------------------------------------------------------- /CALF/utils/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/utils/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/utils/__pycache__/distillationLoss.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/utils/__pycache__/distillationLoss.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/utils/__pycache__/ditill_utils.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/utils/__pycache__/ditill_utils.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/utils/__pycache__/losses.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/utils/__pycache__/losses.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/utils/__pycache__/m4_summary.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/utils/__pycache__/m4_summary.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/utils/__pycache__/metrics.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/utils/__pycache__/metrics.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/utils/__pycache__/print_args.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/utils/__pycache__/print_args.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/utils/__pycache__/timefeatures.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/utils/__pycache__/timefeatures.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/utils/__pycache__/tools.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/utils/__pycache__/tools.cpython-39.pyc -------------------------------------------------------------------------------- /CALF/utils/distillationLoss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | import torch.nn as nn 4 | from .ditill_utils import * 5 | from copy import deepcopy 6 | 7 | from .losses import mape_loss, mase_loss, smape_loss 8 | 9 | loss_dict = { 10 | "l1": nn.L1Loss(), 11 | "smooth_l1": nn.SmoothL1Loss(), 12 | "ce": nn.CrossEntropyLoss(), 13 | "mse": nn.MSELoss(), 14 | "smape": smape_loss(), 15 | "mape": mape_loss(), 16 | "mase": mase_loss(), 17 | } 18 | 19 | class DistillationLoss(nn.Module): 20 | def __init__(self, distill_loss, logits_loss, task_loss, task_name, feature_w=0.01, logits_w=1.0, task_w=1.0): 21 | super(DistillationLoss, self).__init__() 22 | 23 | 24 | # logits_w = 0 25 | # feature_w= 0 26 | self.task_w = task_w 27 | self.logits_w = logits_w 28 | self.feature_w = feature_w 29 | 30 | # weight 1.0 1.0 0.01 31 | print('Loss function weights:' , task_w , logits_w , feature_w ) 32 | 33 | self.feature_loss = loss_dict[distill_loss] 34 | self.logits_loss = loss_dict[logits_loss] 35 | self.task_loss = loss_dict[task_loss] 36 | 37 | self.task_name = task_name 38 | 39 | def forward(self, outputs, batch_y, in_sample=None, freq_map=None, batch_y_mark=None): 40 | """ 41 | outputs_time: 隐藏层特征经过残差连接+任务head之后的结果 42 | intermidiate_feat_time: 大小为num_blk+1, 包含最初的输入特征，最后一个元素是没有经过残差和head的特征。 43 | """ 44 | outputs_text, outputs_time, intermidiate_feat_time, intermidiate_feat_text = ( 45 | outputs["outputs_text"], 46 | outputs["outputs_time"], 47 | outputs["intermidiate_time"], 48 | outputs["intermidiate_text"], 49 | ) 50 | 51 | # 1-----------------中间特征损失 52 | if intermidiate_feat_time is not None : 53 | feature_loss = sum( 54 | [ 55 | (0.8**idx) * self.feature_loss(feat_time, feat_text) 56 | for idx, (feat_time, feat_text) in enumerate( 57 | zip(intermidiate_feat_time[::-1], intermidiate_feat_text[::-1]) 58 | ) 59 | ] 60 | ) 61 | else: 62 | feature_loss = 0 63 | 64 | # 2----------------输出层的教师-学生损失 65 | if outputs_text is not None : 66 | if self.task_name == "long_term_forecast": 67 | logits_loss = self.logits_loss(outputs_time, outputs_text) 68 | elif self.task_name == "short_term_forecast": 69 | logits_loss = self.logits_loss(in_sample, freq_map, outputs_time, outputs_text, batch_y_mark) 70 | elif self.task_name == "classification": 71 | logits_loss = self.logits_loss(outputs_time, outputs_text) 72 | elif self.task_name == "imputation": 73 | logits_loss = self.logits_loss(outputs_time, outputs_text) 74 | elif self.task_name == "anomaly_detection": 75 | logits_loss = self.logits_loss(outputs_time, outputs_text) 76 | else: 77 | logits_loss = 0 78 | 79 | # 3----------------任务特定的标签损失 80 | batch_y = batch_y.to(outputs_time.device) 81 | 82 | if self.task_name == "long_term_forecast": 83 | task_loss = self.task_loss(outputs_time, batch_y) 84 | elif self.task_name == "short_term_forecast": 85 | task_loss = self.task_loss(in_sample, freq_map, outputs_time, batch_y, batch_y_mark) 86 | elif self.task_name == "classification": 87 | task_loss = self.task_loss(outputs_time, batch_y) 88 | elif self.task_name == "imputation": 89 | task_loss = self.task_loss(outputs_time, batch_y) 90 | elif self.task_name == "anomaly_detection": 91 | task_loss = self.task_loss(outputs_time, batch_y) 92 | 93 | total_loss = self.task_w * task_loss + self.logits_w * logits_loss + self.feature_w * feature_loss 94 | return total_loss 95 | -------------------------------------------------------------------------------- /CALF/utils/losses.py: -------------------------------------------------------------------------------- 1 | # This source code is provided for the purposes of scientific reproducibility 2 | # under the following limited license from Element AI Inc. The code is an 3 | # implementation of the N-BEATS model (Oreshkin et al., N-BEATS: Neural basis 4 | # expansion analysis for interpretable time series forecasting, 5 | # https://arxiv.org/abs/1905.10437). The copyright to the source code is 6 | # licensed under the Creative Commons - Attribution-NonCommercial 4.0 7 | # International license (CC BY-NC 4.0): 8 | # https://creativecommons.org/licenses/by-nc/4.0/. Any commercial use (whether 9 | # for the benefit of third parties or internally in production) requires an 10 | # explicit license. The subject-matter of the N-BEATS model and associated 11 | # materials are the property of Element AI Inc. and may be subject to patent 12 | # protection. No license to patents is granted hereunder (whether express or 13 | # implied). Copyright © 2020 Element AI Inc. All rights reserved. 14 | 15 | """ 16 | Loss functions for PyTorch. 17 | """ 18 | 19 | import torch as t 20 | import torch.nn as nn 21 | import numpy as np 22 | import pdb 23 | 24 | 25 | def divide_no_nan(a, b): 26 | """ 27 | a/b where the resulted NaN or Inf are replaced by 0. 28 | """ 29 | result = a / b 30 | result[result != result] = .0 31 | result[result == np.inf] = .0 32 | return result 33 | 34 | 35 | class mape_loss(nn.Module): 36 | def __init__(self): 37 | super(mape_loss, self).__init__() 38 | 39 | def forward(self, insample: t.Tensor, freq: int, 40 | forecast: t.Tensor, target: t.Tensor, mask: t.Tensor) -> t.float: 41 | """ 42 | MAPE loss as defined in: https://en.wikipedia.org/wiki/Mean_absolute_percentage_error 43 | 44 | :param forecast: Forecast values. Shape: batch, time 45 | :param target: Target values. Shape: batch, time 46 | :param mask: 0/1 mask. Shape: batch, time 47 | :return: Loss value 48 | """ 49 | weights = divide_no_nan(mask, target) 50 | return t.mean(t.abs((forecast - target) * weights)) 51 | 52 | 53 | class smape_loss(nn.Module): 54 | def __init__(self): 55 | super(smape_loss, self).__init__() 56 | 57 | def forward(self, insample: t.Tensor, freq: int, 58 | forecast: t.Tensor, target: t.Tensor, mask: t.Tensor) -> t.float: 59 | """ 60 | sMAPE loss as defined in https://robjhyndman.com/hyndsight/smape/ (Makridakis 1993) 61 | 62 | :param forecast: Forecast values. Shape: batch, time 63 | :param target: Target values. Shape: batch, time 64 | :param mask: 0/1 mask. Shape: batch, time 65 | :return: Loss value 66 | """ 67 | return 200 * t.mean(divide_no_nan(t.abs(forecast - target), 68 | t.abs(forecast.data) + t.abs(target.data)) * mask) 69 | 70 | 71 | class mase_loss(nn.Module): 72 | def __init__(self): 73 | super(mase_loss, self).__init__() 74 | 75 | def forward(self, insample: t.Tensor, freq: int, 76 | forecast: t.Tensor, target: t.Tensor, mask: t.Tensor) -> t.float: 77 | """ 78 | MASE loss as defined in "Scaled Errors" https://robjhyndman.com/papers/mase.pdf 79 | 80 | :param insample: Insample values. Shape: batch, time_i 81 | :param freq: Frequency value 82 | :param forecast: Forecast values. Shape: batch, time_o 83 | :param target: Target values. Shape: batch, time_o 84 | :param mask: 0/1 mask. Shape: batch, time_o 85 | :return: Loss value 86 | """ 87 | masep = t.mean(t.abs(insample[:, freq:] - insample[:, :-freq]), dim=1) 88 | masked_masep_inv = divide_no_nan(mask, masep[:, None]) 89 | return t.mean(t.abs(target - forecast) * masked_masep_inv) 90 | -------------------------------------------------------------------------------- /CALF/utils/metrics.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def RSE(pred, true): 5 | return np.sqrt(np.sum((true - pred) ** 2)) / np.sqrt(np.sum((true - true.mean()) ** 2)) 6 | 7 | 8 | def CORR(pred, true): 9 | u = ((true - true.mean(0)) * (pred - pred.mean(0))).sum(0) 10 | d = np.sqrt(((true - true.mean(0)) ** 2 * (pred - pred.mean(0)) ** 2).sum(0)) 11 | return (u / d).mean(-1) 12 | 13 | 14 | def MAE(pred, true): 15 | return np.mean(np.abs(pred - true)) 16 | 17 | 18 | def MSE(pred, true): 19 | return np.mean((pred - true) ** 2) 20 | 21 | 22 | def RMSE(pred, true): 23 | return np.sqrt(MSE(pred, true)) 24 | 25 | 26 | def MAPE(pred, true): 27 | return np.mean(np.abs(100 * (pred - true) / (true +1e-8))) 28 | 29 | 30 | def MSPE(pred, true): 31 | return np.mean(np.square((pred - true) / (true + 1e-8))) 32 | 33 | def SMAPE(pred, true): 34 | return np.mean(200 * np.abs(pred - true) / (np.abs(pred) + np.abs(true) + 1e-8)) 35 | # return np.mean(200 * np.abs(pred - true) / (pred + true + 1e-8)) 36 | 37 | def ND(pred, true): 38 | return np.mean(np.abs(true - pred)) / np.mean(np.abs(true)) 39 | 40 | def metric(pred, true): 41 | mae = MAE(pred, true) 42 | mse = MSE(pred, true) 43 | rmse = RMSE(pred, true) 44 | mape = MAPE(pred, true) 45 | mspe = MSPE(pred, true) 46 | 47 | return mae, mse, rmse, mape, mspe 48 | -------------------------------------------------------------------------------- /CALF/utils/print_args.py: -------------------------------------------------------------------------------- 1 | def print_args(args): 2 | print("\033[1m" + "Basic Config" + "\033[0m") 3 | print(f' {"Task Name:":<20}{args.task_name:<20}{"Is Training:":<20}{args.is_training:<20}') 4 | print(f' {"Model ID:":<20}{args.model_id:<20}{"Model:":<20}{args.model:<20}') 5 | print() 6 | 7 | print("\033[1m" + "Data Loader" + "\033[0m") 8 | print(f' {"Data:":<20}{args.data:<20}{"Root Path:":<20}{args.root_path:<20}') 9 | print(f' {"Data Path:":<20}{args.data_path:<20}{"Features:":<20}{args.features:<20}') 10 | print(f' {"Target:":<20}{args.target:<20}{"Freq:":<20}{args.freq:<20}') 11 | print(f' {"Checkpoints:":<20}{args.checkpoints:<20}') 12 | print() 13 | 14 | if args.task_name in ['long_term_forecast', 'short_term_forecast']: 15 | print("\033[1m" + "Forecasting Task" + "\033[0m") 16 | print(f' {"Seq Len:":<20}{args.seq_len:<20}{"Label Len:":<20}{args.label_len:<20}') 17 | print(f' {"Pred Len:":<20}{args.pred_len:<20}{"Seasonal Patterns:":<20}{args.seasonal_patterns:<20}') 18 | print(f' {"Inverse:":<20}{args.inverse:<20}') 19 | print() 20 | 21 | if args.task_name == 'imputation': 22 | print("\033[1m" + "Imputation Task" + "\033[0m") 23 | print(f' {"Mask Rate:":<20}{args.mask_rate:<20}') 24 | print() 25 | 26 | if args.task_name == 'anomaly_detection': 27 | print("\033[1m" + "Anomaly Detection Task" + "\033[0m") 28 | print(f' {"Anomaly Ratio:":<20}{args.anomaly_ratio:<20}') 29 | print() 30 | 31 | print("\033[1m" + "Model Parameters" + "\033[0m") 32 | print(f' {"Top k:":<20}{args.top_k:<20}{"Num Kernels:":<20}{args.num_kernels:<20}') 33 | print(f' {"Enc In:":<20}{args.enc_in:<20}{"Dec In:":<20}{args.dec_in:<20}') 34 | print(f' {"C Out:":<20}{args.c_out:<20}{"d model:":<20}{args.d_model:<20}') 35 | print(f' {"n heads:":<20}{args.n_heads:<20}{"e layers:":<20}{args.e_layers:<20}') 36 | print(f' {"d layers:":<20}{args.d_layers:<20}{"d FF:":<20}{args.d_ff:<20}') 37 | print(f' {"Moving Avg:":<20}{args.moving_avg:<20}{"Factor:":<20}{args.factor:<20}') 38 | print(f' {"Distil:":<20}{args.distil:<20}{"Dropout:":<20}{args.dropout:<20}') 39 | print(f' {"Embed:":<20}{args.embed:<20}{"Activation:":<20}{args.activation:<20}') 40 | print(f' {"Output Attention:":<20}{args.output_attention:<20}') 41 | print() 42 | 43 | print("\033[1m" + "Run Parameters" + "\033[0m") 44 | print(f' {"Num Workers:":<20}{args.num_workers:<20}{"Itr:":<20}{args.itr:<20}') 45 | print(f' {"Train Epochs:":<20}{args.train_epochs:<20}{"Batch Size:":<20}{args.batch_size:<20}') 46 | print(f' {"Patience:":<20}{args.patience:<20}{"Learning Rate:":<20}{args.learning_rate:<20}') 47 | print(f' {"Des:":<20}{args.des:<20}{"Distill Loss:":<20}{args.distill_loss:<20}{"Logits Loss:":<20}{args.logits_loss:<20}{"Task Loss:":<20}{args.task_loss:<20}') 48 | print(f' {"Lradj:":<20}{args.lradj:<20}{"Use Amp:":<20}{args.use_amp:<20}') 49 | print() 50 | 51 | print("\033[1m" + "GPU" + "\033[0m") 52 | print(f' {"Use GPU:":<20}{args.use_gpu:<20}{"GPU:":<20}{args.gpu:<20}') 53 | print(f' {"Use Multi GPU:":<20}{args.use_multi_gpu:<20}{"Devices:":<20}{args.devices:<20}') 54 | print() 55 | 56 | print("\033[1m" + "De-stationary Projector Params" + "\033[0m") 57 | p_hidden_dims_str = ', '.join(map(str, args.p_hidden_dims)) 58 | print(f' {"P Hidden Dims:":<20}{p_hidden_dims_str:<20}{"P Hidden Layers:":<20}{args.p_hidden_layers:<20}') 59 | print() 60 | 61 | print("\033[1m" + "Distill Loss Weight" + "\033[0m") 62 | p_hidden_dims_str = ', '.join(map(str, args.p_hidden_dims)) 63 | print(f' {"Feature Weight:":<20}{args.feature_w:<20}{"Logits Weight:":<20}{args.logits_w:<20}{"Task Weight:":<20}{args.task_w:<20}') 64 | print() -------------------------------------------------------------------------------- /CALF/utils/prompts.py: -------------------------------------------------------------------------------- 1 | ETTh_prompt = "The Electricity Transformer Temperature (ETT) dataset contains 7 factors of electricity transformer from July 2016 to July 2018, which is recorded by hour." 2 | 3 | ETTm_prompt = "The Electricity Transformer Temperature (ETT) dataset contains 7 factors of electricity transformer from July 2016 to July 2018, which is recorded by minute." 4 | 5 | Traffic_prompt = "Traffic dataset collects hourly road occupancy rates measured by 862 sensors of San Francisco Bay area freeways from January 2015 to December 2016." 6 | 7 | Weather_prompt = "Weather dataset includes 21 meteorological factors collected every 10 minutes from the Weather Station of the Max Planck Biogeochemistry Institute in 2020." 8 | 9 | Electricity_prompt = "Electricity dataset records the hourly electricity consumption data of 321 clients." 10 | 11 | prompt_dict = {"ETTh1": ETTh_prompt, 12 | "ETTh2": ETTh_prompt, 13 | "ETTm1": ETTm_prompt, 14 | "ETTm2": ETTm_prompt, 15 | "traffic": Traffic_prompt, 16 | "weather": Weather_prompt, 17 | "electricity": Electricity_prompt} -------------------------------------------------------------------------------- /CALF/utils/timefeatures.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import numpy as np 4 | import pandas as pd 5 | from pandas.tseries import offsets 6 | from pandas.tseries.frequencies import to_offset 7 | 8 | 9 | class TimeFeature: 10 | def __init__(self): 11 | pass 12 | 13 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 14 | pass 15 | 16 | def __repr__(self): 17 | return self.__class__.__name__ + "()" 18 | 19 | 20 | class SecondOfMinute(TimeFeature): 21 | """Minute of hour encoded as value between [-0.5, 0.5]""" 22 | 23 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 24 | return index.second / 59.0 - 0.5 25 | 26 | 27 | class MinuteOfHour(TimeFeature): 28 | """Minute of hour encoded as value between [-0.5, 0.5]""" 29 | 30 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 31 | return index.minute / 59.0 - 0.5 32 | 33 | 34 | class HourOfDay(TimeFeature): 35 | """Hour of day encoded as value between [-0.5, 0.5]""" 36 | 37 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 38 | return index.hour / 23.0 - 0.5 39 | 40 | 41 | class DayOfWeek(TimeFeature): 42 | """Hour of day encoded as value between [-0.5, 0.5]""" 43 | 44 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 45 | return index.dayofweek / 6.0 - 0.5 46 | 47 | 48 | class DayOfMonth(TimeFeature): 49 | """Day of month encoded as value between [-0.5, 0.5]""" 50 | 51 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 52 | return (index.day - 1) / 30.0 - 0.5 53 | 54 | 55 | class DayOfYear(TimeFeature): 56 | """Day of year encoded as value between [-0.5, 0.5]""" 57 | 58 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 59 | return (index.dayofyear - 1) / 365.0 - 0.5 60 | 61 | 62 | class MonthOfYear(TimeFeature): 63 | """Month of year encoded as value between [-0.5, 0.5]""" 64 | 65 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 66 | return (index.month - 1) / 11.0 - 0.5 67 | 68 | 69 | class WeekOfYear(TimeFeature): 70 | """Week of year encoded as value between [-0.5, 0.5]""" 71 | 72 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 73 | return (index.isocalendar().week - 1) / 52.0 - 0.5 74 | 75 | 76 | def time_features_from_frequency_str(freq_str: str) -> List[TimeFeature]: 77 | """ 78 | Returns a list of time features that will be appropriate for the given frequency string. 79 | Parameters 80 | ---------- 81 | freq_str 82 | Frequency string of the form [multiple][granularity] such as "12H", "5min", "1D" etc. 83 | """ 84 | 85 | features_by_offsets = { 86 | offsets.YearEnd: [], 87 | offsets.QuarterEnd: [MonthOfYear], 88 | offsets.MonthEnd: [MonthOfYear], 89 | offsets.Week: [DayOfMonth, WeekOfYear], 90 | offsets.Day: [DayOfWeek, DayOfMonth, DayOfYear], 91 | offsets.BusinessDay: [DayOfWeek, DayOfMonth, DayOfYear], 92 | offsets.Hour: [HourOfDay, DayOfWeek, DayOfMonth, DayOfYear], 93 | offsets.Minute: [ 94 | MinuteOfHour, 95 | HourOfDay, 96 | DayOfWeek, 97 | DayOfMonth, 98 | DayOfYear, 99 | ], 100 | offsets.Second: [ 101 | SecondOfMinute, 102 | MinuteOfHour, 103 | HourOfDay, 104 | DayOfWeek, 105 | DayOfMonth, 106 | DayOfYear, 107 | ], 108 | } 109 | 110 | offset = to_offset(freq_str) 111 | 112 | for offset_type, feature_classes in features_by_offsets.items(): 113 | if isinstance(offset, offset_type): 114 | return [cls() for cls in feature_classes] 115 | 116 | supported_freq_msg = f""" 117 | Unsupported frequency {freq_str} 118 | The following frequencies are supported: 119 | Y - yearly 120 | alias: A 121 | M - monthly 122 | W - weekly 123 | D - daily 124 | B - business days 125 | H - hourly 126 | T - minutely 127 | alias: min 128 | S - secondly 129 | """ 130 | raise RuntimeError(supported_freq_msg) 131 | 132 | 133 | def time_features(dates, freq='h'): 134 | return np.vstack([feat(dates) for feat in time_features_from_frequency_str(freq)]) 135 | -------------------------------------------------------------------------------- /CALF/wte_pca_500.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/CALF/wte_pca_500.pt -------------------------------------------------------------------------------- /OFA/README.md: -------------------------------------------------------------------------------- 1 | # One Fits All: Power General Time Series Analysis by Pretrained LM (NeurIPS 2023 Spotlight) 2 | 3 | Tian Zhou, Peisong Niu, Xue Wang, Liang Sun, Rong Jin, "One Fits All: Power General Time Series Analysis by Pretrained LM,", NeurIPS, 2023. [[paper](https://arxiv.org/abs/2302.11939)] 4 | 5 | ## Long-term Learning 6 | 7 | ![image](../pic/long_term_result.png) 8 | 9 | ## Get Start 10 | 11 | - The code is the same as few-shot leanring with 100 percent. 12 | - Install Python>=3.8, PyTorch 1.8.1. 13 | - Download data. You can obtain all the benchmarks from [[TimesNet](https://github.com/thuml/Time-Series-Library)]. 14 | - For electricity and traffic datasets with a batch size of 2048, we utilize 4 V100 GPUs, while for other datasets, we use a single V100 GPU. 15 | - Train the model. We provide the experiment scripts of all benchmarks under the folder `./scripts`. You can reproduce the experiment results by: 16 | 17 | ```bash 18 | bash ./scripts/ETTh1.sh 19 | bash ./scripts/ETTh2.sh 20 | ``` 21 | 22 | ## Citation 23 | 24 | If you find this repo useful, please cite our paper. 25 | 26 | ``` 27 | @inproceedings{zhou2023onefitsall, 28 | title={{One Fits All}: Power General Time Series Analysis by Pretrained LM}, 29 | author={Tian Zhou, Peisong Niu, Xue Wang, Liang Sun, Rong Jin}, 30 | booktitle={NeurIPS}, 31 | year={2023} 32 | } 33 | ``` -------------------------------------------------------------------------------- /OFA/data_provider/__pycache__/data_factory.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/OFA/data_provider/__pycache__/data_factory.cpython-39.pyc -------------------------------------------------------------------------------- /OFA/data_provider/__pycache__/data_loader.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/OFA/data_provider/__pycache__/data_loader.cpython-39.pyc -------------------------------------------------------------------------------- /OFA/data_provider/data_factory.py: -------------------------------------------------------------------------------- 1 | from data_provider.data_loader import Dataset_Pred, Dataset_TSF, Dataset_Custom_BS, Dataset_ETT_hour, Dataset_ETT_minute 2 | from torch.utils.data import DataLoader 3 | 4 | data_dict = { 5 | # 'custom': Dataset_Custom, 6 | 'custom': Dataset_Custom_BS, 7 | 'tsf_data': Dataset_TSF, 8 | 'ett_h': Dataset_ETT_hour, 9 | 'ett_m': Dataset_ETT_minute, 10 | } 11 | def data_provider(args, flag, drop_last_test=True, train_all=False): 12 | Data = data_dict[args.data] 13 | timeenc = 0 if args.embed != 'timeF' else 1 14 | percent = args.percent 15 | max_len = args.max_len 16 | 17 | if flag == 'test': 18 | shuffle_flag = False 19 | drop_last = drop_last_test 20 | batch_size = args.batch_size 21 | freq = args.freq 22 | elif flag == 'pred': 23 | shuffle_flag = False 24 | drop_last = False 25 | batch_size = 1 26 | freq = args.freq 27 | Data = Dataset_Pred 28 | elif flag == 'val': 29 | shuffle_flag = True 30 | drop_last = drop_last_test 31 | batch_size = args.batch_size 32 | freq = args.freq 33 | else: 34 | shuffle_flag = True 35 | drop_last = True 36 | batch_size = args.batch_size 37 | freq = args.freq 38 | 39 | data_set = Data( 40 | model_id=args.model_id , 41 | root_path=args.root_path, 42 | data_path=args.data_path, 43 | flag=flag, 44 | size=[args.seq_len, args.label_len, args.pred_len], 45 | features=args.features, 46 | target=args.target, 47 | timeenc=timeenc, 48 | freq=freq, 49 | percent=percent, 50 | max_len=max_len, 51 | train_all=train_all 52 | ) 53 | data_loader = DataLoader( 54 | data_set, 55 | batch_size=batch_size, 56 | shuffle=shuffle_flag, 57 | num_workers=args.num_workers, 58 | drop_last=drop_last) 59 | return data_set, data_loader 60 | -------------------------------------------------------------------------------- /OFA/models/Attention.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | class ScaledDotProductAttention(nn.Module): 6 | ''' Scaled Dot-Product Attention ''' 7 | 8 | def __init__(self, temperature, attn_dropout=0.1): 9 | super().__init__() 10 | self.temperature = temperature 11 | self.dropout = nn.Dropout(attn_dropout) 12 | 13 | def forward(self, q, k, v, mask=None): 14 | 15 | attn = torch.matmul(q / self.temperature, k.transpose(2, 3)) 16 | 17 | if mask is not None: 18 | attn = attn.masked_fill(mask == 0, -1e9) 19 | 20 | attn = self.dropout(F.softmax(attn, dim=-1)) 21 | output = torch.matmul(attn, v) 22 | 23 | return output, attn 24 | 25 | class MultiHeadAttention(nn.Module): 26 | ''' Multi-Head Attention module ''' 27 | def __init__(self, d_model = -1 ,n_head = 8 , d_k = -1 , d_v = -1 , dropout=0.1): 28 | super().__init__() 29 | self.n_head = n_head 30 | d_k = d_model // n_head 31 | d_v = d_k 32 | self.d_k = d_k 33 | self.d_v = d_v 34 | 35 | self.w_qs = nn.Linear(d_model, n_head * d_k, bias=False) 36 | self.w_ks = nn.Linear(d_model, n_head * d_k, bias=False) 37 | self.w_vs = nn.Linear(d_model, n_head * d_v, bias=False) 38 | self.fc = nn.Linear(n_head * d_v, d_model, bias=False) 39 | 40 | self.attention = ScaledDotProductAttention(temperature=d_k ** 0.5) 41 | 42 | self.dropout = nn.Dropout(dropout) 43 | self.layer_norm = nn.LayerNorm(d_model, eps=1e-6) 44 | 45 | 46 | def forward(self, q, k, v, mask=None): 47 | 48 | d_k, d_v, n_head = self.d_k, self.d_v, self.n_head 49 | sz_b, len_q, len_k, len_v = q.size(0), q.size(1), k.size(1), v.size(1) 50 | 51 | residual = q 52 | 53 | # Pass through the pre-attention projection: b x lq x (n*dv) 54 | # Separate different heads: b x lq x n x dv 55 | q = self.w_qs(q).view(sz_b, len_q, n_head, d_k) 56 | k = self.w_ks(k).view(sz_b, len_k, n_head, d_k) 57 | v = self.w_vs(v).view(sz_b, len_v, n_head, d_v) 58 | 59 | # Transpose for attention dot product: b x n x lq x dv 60 | q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2) 61 | 62 | if mask is not None: 63 | mask = mask.unsqueeze(1) # For head axis broadcasting. 64 | 65 | q, attn = self.attention(q, k, v, mask=mask) 66 | 67 | # Transpose to move the head dimension back: b x lq x n x dv 68 | # Combine the last two dimensions to concatenate all the heads together: b x lq x (n*dv) 69 | q = q.transpose(1, 2).contiguous().view(sz_b, len_q, -1) 70 | q = self.dropout(self.fc(q)) 71 | q += residual 72 | 73 | q = self.layer_norm(q) 74 | 75 | return q, attn -------------------------------------------------------------------------------- /OFA/models/DLinear.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | import numpy as np 5 | 6 | class moving_avg(nn.Module): 7 | """ 8 | Moving average block to highlight the trend of time series 9 | """ 10 | def __init__(self, kernel_size, stride): 11 | super(moving_avg, self).__init__() 12 | self.kernel_size = kernel_size 13 | self.avg = nn.AvgPool1d(kernel_size=kernel_size, stride=stride, padding=0) 14 | 15 | def forward(self, x): 16 | # padding on the both ends of time series 17 | front = x[:, 0:1, :].repeat(1, (self.kernel_size - 1) // 2, 1) 18 | end = x[:, -1:, :].repeat(1, (self.kernel_size - 1) // 2, 1) 19 | x = torch.cat([front, x, end], dim=1) 20 | x = self.avg(x.permute(0, 2, 1)) 21 | x = x.permute(0, 2, 1) 22 | return x 23 | 24 | class series_decomp(nn.Module): 25 | """ 26 | Series decomposition block 27 | """ 28 | def __init__(self, kernel_size): 29 | super(series_decomp, self).__init__() 30 | self.moving_avg = moving_avg(kernel_size, stride=1) 31 | 32 | def forward(self, x): 33 | moving_mean = self.moving_avg(x) 34 | res = x - moving_mean 35 | return res, moving_mean 36 | 37 | class DLinear(nn.Module): 38 | """ 39 | Decomposition-Linear 40 | """ 41 | def __init__(self, configs, device): 42 | super(DLinear, self).__init__() 43 | self.seq_len = configs.seq_len 44 | self.pred_len = configs.pred_len 45 | 46 | # Decompsition Kernel Size 47 | kernel_size = configs.kernel_size 48 | self.decompsition = series_decomp(kernel_size) 49 | self.individual = True 50 | self.channels = configs.enc_in 51 | 52 | if self.individual: 53 | self.Linear_Seasonal = nn.ModuleList() 54 | self.Linear_Trend = nn.ModuleList() 55 | 56 | for i in range(self.channels): 57 | self.Linear_Seasonal.append(nn.Linear(self.seq_len,self.pred_len)) 58 | self.Linear_Trend.append(nn.Linear(self.seq_len,self.pred_len)) 59 | 60 | # Use this two lines if you want to visualize the weights 61 | # self.Linear_Seasonal[i].weight = nn.Parameter((1/self.seq_len)*torch.ones([self.pred_len,self.seq_len])) 62 | # self.Linear_Trend[i].weight = nn.Parameter((1/self.seq_len)*torch.ones([self.pred_len,self.seq_len])) 63 | else: 64 | self.Linear_Seasonal = nn.Linear(self.seq_len,self.pred_len) 65 | self.Linear_Trend = nn.Linear(self.seq_len,self.pred_len) 66 | 67 | # Use this two lines if you want to visualize the weights 68 | # self.Linear_Seasonal.weight = nn.Parameter((1/self.seq_len)*torch.ones([self.pred_len,self.seq_len])) 69 | # self.Linear_Trend.weight = nn.Parameter((1/self.seq_len)*torch.ones([self.pred_len,self.seq_len])) 70 | 71 | def forward(self, x, itr): 72 | # [256 , 7 , 96 ] 73 | x = x.permute(0,2,1) 74 | # x: [Batch, Input length, Channel] 75 | seasonal_init, trend_init = self.decompsition(x) 76 | seasonal_init, trend_init = seasonal_init.permute(0,2,1), trend_init.permute(0,2,1) 77 | if self.individual: 78 | seasonal_output = torch.zeros([seasonal_init.size(0),seasonal_init.size(1),self.pred_len],dtype=seasonal_init.dtype).to(seasonal_init.device) 79 | trend_output = torch.zeros([trend_init.size(0),trend_init.size(1),self.pred_len],dtype=trend_init.dtype).to(trend_init.device) 80 | for i in range(self.channels): 81 | seasonal_output[:,i,:] = self.Linear_Seasonal[i](seasonal_init[:,i,:]) 82 | trend_output[:,i,:] = self.Linear_Trend[i](trend_init[:,i,:]) 83 | else: 84 | seasonal_output = self.Linear_Seasonal(seasonal_init) 85 | trend_output = self.Linear_Trend(trend_init) 86 | 87 | x = seasonal_output + trend_output 88 | return x 89 | # return x.permute(0,2,1) # to [Batch, Output length, Channel] 90 | -------------------------------------------------------------------------------- /OFA/models/GPT4TS_orig.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | from torch import optim 5 | 6 | from transformers.models.gpt2.modeling_gpt2 import GPT2Model 7 | from transformers import BertTokenizer, BertModel 8 | from einops import rearrange 9 | from embed import DataEmbedding, DataEmbedding_wo_time 10 | from transformers.models.gpt2.configuration_gpt2 import GPT2Config 11 | 12 | class GPT4TS(nn.Module): 13 | 14 | def __init__(self, configs, device): 15 | super(GPT4TS, self).__init__() 16 | self.is_gpt = configs.is_gpt 17 | self.patch_size = configs.patch_size 18 | self.pretrain = configs.pretrain 19 | self.stride = configs.stride 20 | self.patch_num = (configs.seq_len - self.patch_size) // self.stride + 1 21 | 22 | self.padding_patch_layer = nn.ReplicationPad1d((0, self.stride)) 23 | self.patch_num += 1 24 | 25 | if configs.is_gpt: 26 | if configs.pretrain: 27 | self.gpt2 = GPT2Model.from_pretrained('gpt2', output_attentions=True, output_hidden_states=True) # loads a pretrained GPT-2 base model 28 | else: 29 | print("------------------no pretrain------------------") 30 | self.gpt2 = GPT2Model(GPT2Config()) 31 | self.gpt2.h = self.gpt2.h[:configs.gpt_layers] 32 | print("gpt2 = {}".format(self.gpt2)) 33 | 34 | self.in_layer = nn.Linear(configs.patch_size, configs.d_model) 35 | self.out_layer = nn.Linear(configs.d_model * self.patch_num, configs.pred_len) 36 | 37 | if configs.freeze and configs.pretrain: 38 | for i, (name, param) in enumerate(self.gpt2.named_parameters()): 39 | if 'ln' in name or 'wpe' in name: 40 | param.requires_grad = True 41 | else: 42 | param.requires_grad = False 43 | 44 | for layer in (self.gpt2, self.in_layer, self.out_layer): 45 | layer.to(device=device) 46 | layer.train() 47 | 48 | self.cnt = 0 49 | 50 | def forward(self, x, itr): 51 | B, L, M = x.shape 52 | 53 | means = x.mean(1, keepdim=True).detach() 54 | x = x - means 55 | stdev = torch.sqrt(torch.var(x, dim=1, keepdim=True, unbiased=False)+ 1e-5).detach() 56 | x /= stdev 57 | 58 | x = rearrange(x, 'b l m -> b m l') 59 | 60 | x = self.padding_patch_layer(x) 61 | x = x.unfold(dimension=-1, size=self.patch_size, step=self.stride) 62 | x = rearrange(x, 'b m n p -> (b m) n p') 63 | 64 | outputs = self.in_layer(x) 65 | if self.is_gpt: 66 | outputs = self.gpt2(inputs_embeds=outputs).last_hidden_state 67 | 68 | outputs = self.out_layer(outputs.reshape(B*M, -1)) 69 | outputs = rearrange(outputs, '(b m) l -> b l m', b=B) 70 | 71 | outputs = outputs * stdev 72 | outputs = outputs + means 73 | 74 | return outputs 75 | -------------------------------------------------------------------------------- /OFA/models/NLinear.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | import numpy as np 5 | 6 | class NLinear(nn.Module): 7 | """ 8 | Normalization-Linear 9 | """ 10 | def __init__(self, configs): 11 | super(NLinear, self).__init__() 12 | self.seq_len = configs.seq_len 13 | self.pred_len = configs.pred_len 14 | 15 | # Use this line if you want to visualize the weights 16 | # self.Linear.weight = nn.Parameter((1/self.seq_len)*torch.ones([self.pred_len,self.seq_len])) 17 | self.channels = configs.enc_in 18 | self.individual = False 19 | if self.individual: 20 | self.Linear = nn.ModuleList() 21 | for i in range(self.channels): 22 | self.Linear.append(nn.Linear(self.seq_len,self.pred_len)) 23 | else: 24 | self.Linear = nn.Linear(self.seq_len, self.pred_len) 25 | 26 | def forward(self, x , ii): 27 | # x: [Batch, Input length, Channel] 28 | seq_last = x[:,-1:,:].detach() 29 | x = x - seq_last 30 | if self.individual: 31 | output = torch.zeros([x.size(0),self.pred_len,x.size(2)],dtype=x.dtype).to(x.device) 32 | for i in range(self.channels): 33 | output[:,:,i] = self.Linear[i](x[:,:,i]) 34 | x = output 35 | else: 36 | x = self.Linear(x.permute(0,2,1)).permute(0,2,1) 37 | x = x + seq_last 38 | return x # [Batch, Output length, Channel] -------------------------------------------------------------------------------- /OFA/models/__pycache__/Attention.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/OFA/models/__pycache__/Attention.cpython-39.pyc -------------------------------------------------------------------------------- /OFA/models/__pycache__/DLinear.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/OFA/models/__pycache__/DLinear.cpython-39.pyc -------------------------------------------------------------------------------- /OFA/models/__pycache__/DLinear_plus.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/OFA/models/__pycache__/DLinear_plus.cpython-39.pyc -------------------------------------------------------------------------------- /OFA/models/__pycache__/GPT4TS.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/OFA/models/__pycache__/GPT4TS.cpython-39.pyc -------------------------------------------------------------------------------- /OFA/models/__pycache__/NLinear.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/OFA/models/__pycache__/NLinear.cpython-39.pyc -------------------------------------------------------------------------------- /OFA/models/__pycache__/PatchTST.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/OFA/models/__pycache__/PatchTST.cpython-39.pyc -------------------------------------------------------------------------------- /OFA/script/ETTh_GPT2.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | filename=ETTh_drop.txt 3 | model=GPT4TS 4 | percent=100 5 | itt=5 6 | gpu_loc=0 7 | inp_len=336 8 | run_file=main.py 9 | # run_file=eval_bs.py 10 | 11 | # bash ./scripts/ETTh_GPT2.sh 12 | 13 | pre_lens_h="96 192 336 720" 14 | methods_h="ori removeLLM llm_to_attn llm_to_trsf" 15 | 16 | # ETTh1 17 | for pred_len in $pre_lens_h; 18 | do 19 | for method in $methods_h; 20 | do 21 | if [ $method == 'ori' ]; then 22 | lr=0.0001 23 | else 24 | lr=0.001 25 | fi 26 | echo $method"_"$lr 27 | python $run_file \ 28 | --root_path ./datasets/ETT-small/ \ 29 | --data_path ETTh1.csv \ 30 | --model_id 'ETTh1_test_'$inp_len'_'$pred_len'_'$method'_ofa' \ 31 | --data ett_h \ 32 | --seq_len $inp_len \ 33 | --label_len 0 \ 34 | --pred_len $pred_len \ 35 | --batch_size 256 \ 36 | --lradj type4 \ 37 | --learning_rate $lr \ 38 | --train_epochs 10 \ 39 | --decay_fac 0.5 \ 40 | --d_model 768 \ 41 | --n_heads 4 \ 42 | --d_ff 768 \ 43 | --dropout 0.3 \ 44 | --enc_in 7 \ 45 | --c_out 7 \ 46 | --freq 0 \ 47 | --itr $itt \ 48 | --patch_size 16 \ 49 | --stride 8 \ 50 | --percent 100 \ 51 | --gpt_layer 6 \ 52 | --model $model \ 53 | --gpu_loc $gpu_loc \ 54 | --tmax 20 \ 55 | --cos 1 \ 56 | --is_gpt 1 \ 57 | --save_file_name $filename 58 | done 59 | done 60 | 61 | for pred_len in $pre_lens_h; 62 | do 63 | for method in $methods_h; 64 | do 65 | if [ $method == 'ori' ]; then 66 | lr=0.0001 67 | else 68 | lr=0.001 69 | fi 70 | echo $method"_"$lr 71 | python $run_file \ 72 | --root_path ./datasets/ETT-small/ \ 73 | --data_path ETTh2.csv \ 74 | --model_id 'ETTh2_'$inp_len'_'$pred_len'_'$method'_ofa' \ 75 | --data ett_h \ 76 | --seq_len $inp_len \ 77 | --label_len 0 \ 78 | --pred_len $pred_len \ 79 | --batch_size 256 \ 80 | --decay_fac 0.5 \ 81 | --learning_rate $lr \ 82 | --train_epochs 10 \ 83 | --d_model 768 \ 84 | --n_heads 4 \ 85 | --d_ff 768 \ 86 | --dropout 1 \ 87 | --itr $itt \ 88 | --enc_in 7 \ 89 | --c_out 7 \ 90 | --freq 0 \ 91 | --patch_size 16 \ 92 | --stride 8 \ 93 | --percent $percent \ 94 | --gpt_layer 6 \ 95 | --model $model \ 96 | --cos 1 \ 97 | --gpu_loc $gpu_loc \ 98 | --tmax 20 \ 99 | --pretrain 1 \ 100 | --is_gpt 1 \ 101 | --save_file_name $filename 102 | done 103 | done 104 | -------------------------------------------------------------------------------- /OFA/script/ETTh_simple.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | # eval_bs.py 3 | tag_file=main.py 4 | run_data=0 5 | 6 | model=DLinear_plus 7 | percent=100 8 | patience=10 9 | 10 | # bash ./scripts/ETTh_simple.sh 11 | inp_len_h=96 12 | pre_lens_h="96 192 336 720" 13 | methods_h='single_linr multi_decp_trsf single_linr_decp multi_linr_trsf multi_patch_attn multi_patch_decp' 14 | filename_h=ETTh_simple.txt 15 | 16 | # linr 0.0005 17 | # attn 0.00005 18 | 19 | gpu_loc=0 20 | itt=5 21 | 22 | for pred_len in $pre_lens_h; 23 | do 24 | for method in $methods_h; 25 | do 26 | if echo "$method" | grep -q "linr"; then 27 | lr=0.0005 28 | else 29 | lr=0.00005 30 | fi 31 | echo $lr'_'$pred_len'_'$method 32 | python $tag_file \ 33 | --root_path ./datasets/ETT-small/ \ 34 | --data_path ETTh1.csv \ 35 | --model_id 'ETTh1_'$inp_len_h'_'$pred_len'_simple_'$method \ 36 | --data ett_h \ 37 | --seq_len $inp_len_h \ 38 | --label_len 0 \ 39 | --pred_len $pred_len \ 40 | --batch_size 512 \ 41 | --lradj type4 \ 42 | --learning_rate $lr \ 43 | --train_epochs 100 \ 44 | --decay_fac 0.5 \ 45 | --d_model 768 \ 46 | --n_heads 4 \ 47 | --d_ff 768 \ 48 | --dropout 0.3 \ 49 | --enc_in 7 \ 50 | --c_out 7 \ 51 | --freq 0 \ 52 | --patch_size 16 \ 53 | --stride 8 \ 54 | --patience $patience \ 55 | --percent 100 \ 56 | --gpt_layer 6 \ 57 | --itr $itt \ 58 | --model $model \ 59 | --tmax 20 \ 60 | --cos 1 \ 61 | --method $method \ 62 | --gpu_loc $gpu_loc \ 63 | --is_gpt 1 \ 64 | --save_file_name $filename_h 65 | done 66 | done 67 | 68 | 69 | for pred_len in $pre_lens_h; 70 | do 71 | for method in $methods_h; 72 | do 73 | if echo "$method" | grep -q "linr"; then 74 | lr=0.0005 75 | else 76 | lr=0.00005 77 | fi 78 | python $tag_file \ 79 | --root_path ./datasets/ETT-small/ \ 80 | --data_path ETTh2.csv \ 81 | --model_id 'ETTh2_'$inp_len_h'_'$pred_len'_simple_'$method \ 82 | --data ett_h \ 83 | --seq_len $inp_len_h \ 84 | --label_len 0 \ 85 | --pred_len $pred_len \ 86 | --batch_size 512 \ 87 | --decay_fac 0.5 \ 88 | --learning_rate $lr \ 89 | --train_epochs 10 \ 90 | --d_model 768 \ 91 | --n_heads 4 \ 92 | --d_ff 768 \ 93 | --dropout 1 \ 94 | --enc_in 7 \ 95 | --c_out 7 \ 96 | --freq 0 \ 97 | --patch_size 16 \ 98 | --stride 8 \ 99 | --percent 100 \ 100 | --gpt_layer 6 \ 101 | --patience $patience \ 102 | --itr $itt \ 103 | --model $model \ 104 | --cos 1 \ 105 | --method $method \ 106 | --gpu_loc $gpu_loc \ 107 | --tmax 20 \ 108 | --pretrain 1 \ 109 | --is_gpt 1 \ 110 | --save_file_name $filename_h 111 | done 112 | done 113 | -------------------------------------------------------------------------------- /OFA/script/ETTm_GPT2.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | filename=ETTm2_drop.txt 3 | model=GPT4TS 4 | percent=100 5 | itt=5 6 | gpu_loc=1 7 | inp_len=512 8 | run_file=main.py 9 | #run_file=eval_bs.py 10 | 11 | # bash ./scripts/ETTm_GPT2.sh 12 | # pre_lens_h="96 192 336 720" 13 | pre_lens_h="96 192 336 720" 14 | methods_h="ori removeLLM llm_to_attn llm_to_trsf" 15 | 16 | for pred_len in $pre_lens_h; 17 | do 18 | for method in $methods_h; 19 | do 20 | if [ $method == 'ori' ]; then 21 | lr=0.0001 22 | bs=512 23 | else 24 | lr=0.00005 25 | bs=512 26 | fi 27 | echo $method"_"$lr 28 | python $run_file \ 29 | --root_path ./datasets/ETT-small/ \ 30 | --data_path ETTm1.csv \ 31 | --model_id 'ETTm1_'$inp_len'_'$pred_len'_'$method'_ofa' \ 32 | --data ett_m \ 33 | --label_len 48 \ 34 | --pred_len $pred_len \ 35 | --batch_size $bs \ 36 | --train_epochs 10 \ 37 | --decay_fac 0.75 \ 38 | --d_model 768 \ 39 | --n_heads 4 \ 40 | --d_ff 768 \ 41 | --dropout 0.3 \ 42 | --itr $itt \ 43 | --enc_in 7 \ 44 | --c_out 7 \ 45 | --freq 0 \ 46 | --patch_size 16 \ 47 | --stride 16 \ 48 | --percent $percent \ 49 | --gpt_layer 6 \ 50 | --cos 1 \ 51 | --is_gpt 1 \ 52 | --seq_len $inp_len \ 53 | --learning_rate $lr \ 54 | --model $model \ 55 | --gpu_loc $gpu_loc \ 56 | --save_file_name $filename 57 | done 58 | done 59 | 60 | for pred_len in $pre_lens_h; 61 | do 62 | for method in $methods_h; 63 | do 64 | if [ $method == 'ori' ]; then 65 | lr=0.0001 66 | bs=512 67 | else 68 | lr=0.00005 69 | bs=512 70 | fi 71 | echo $method"_"$lr 72 | python $run_file \ 73 | --root_path ./datasets/ETT-small/ \ 74 | --data_path ETTm2.csv \ 75 | --model_id 'ETTm2_'$inp_len'_'$pred_len'_'$method'_ofa' \ 76 | --data ett_m \ 77 | --label_len 48 \ 78 | --pred_len $pred_len \ 79 | --batch_size $bs \ 80 | --train_epochs 10 \ 81 | --decay_fac 0.75 \ 82 | --d_model 768 \ 83 | --n_heads 4 \ 84 | --d_ff 768 \ 85 | --dropout 0.3 \ 86 | --enc_in 7 \ 87 | --c_out 7 \ 88 | --freq 0 \ 89 | --patch_size 16 \ 90 | --stride 16 \ 91 | --percent $percent \ 92 | --gpt_layer 6 \ 93 | --itr $itt \ 94 | --cos 1 \ 95 | --is_gpt 1 \ 96 | --seq_len $inp_len \ 97 | --learning_rate $lr \ 98 | --model $model \ 99 | --gpu_loc $gpu_loc \ 100 | --save_file_name $filename 101 | done 102 | done 103 | -------------------------------------------------------------------------------- /OFA/script/ETTm_simple.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2,3" 2 | 3 | tag_file=main.py 4 | 5 | model=DLinear_plus 6 | percent=100 7 | patience=10 8 | 9 | # bash ./scripts/ETTm_simple.sh 10 | inp_len_m=512 11 | # pre_lens_m="96 192 336 720" 12 | pre_lens_m="192" 13 | # methods_m='single_linr single_linr_decp multi_linr_trsf multi_decp_trsf multi_patch_attn multi_patch_decp' 14 | methods_m='multi_decp_trsf multi_patch_attn multi_patch_decp single_linr single_linr_decp multi_linr_trsf' 15 | filename_m=ETTm_simple_extra.txt 16 | 17 | gpu_loc=0 18 | itt=5 19 | # ETTm 20 | for pred_len in $pre_lens_m; 21 | do 22 | for method in $methods_m; 23 | do 24 | if echo "$method" | grep -q "linr"; then 25 | lr=0.0005 26 | else 27 | lr=0.00005 28 | fi 29 | python $tag_file \ 30 | --root_path ./datasets/ETT-small/ \ 31 | --data_path ETTm1.csv \ 32 | --model_id 'ETTm1_'$inp_len_m'_'$pred_len'_simple_'$method \ 33 | --data ett_m \ 34 | --seq_len $inp_len_m \ 35 | --label_len 0 \ 36 | --pred_len $pred_len \ 37 | --batch_size 1024 \ 38 | --learning_rate $lr \ 39 | --train_epochs 10 \ 40 | --decay_fac 0.75 \ 41 | --d_model 768 \ 42 | --n_heads 4 \ 43 | --d_ff 768 \ 44 | --dropout 0.3 \ 45 | --enc_in 7 \ 46 | --c_out 7 \ 47 | --freq 0 \ 48 | --patch_size 16 \ 49 | --stride 16 \ 50 | --percent $percent \ 51 | --gpt_layer 6 \ 52 | --gpu_loc $gpu_loc \ 53 | --patience $patience \ 54 | --itr $itt \ 55 | --method $method \ 56 | --model $model \ 57 | --cos 1 \ 58 | --is_gpt 1 \ 59 | --save_file_name $filename_m 60 | done 61 | done 62 | 63 | # ETTm2 64 | for pred_len in $pre_lens_m; 65 | do 66 | for method in $methods_m; 67 | do 68 | if echo "$method" | grep -q "linr"; then 69 | lr=0.0005 70 | else 71 | lr=0.00005 72 | fi 73 | python $tag_file \ 74 | --root_path ./datasets/ETT-small/ \ 75 | --data_path ETTm2.csv \ 76 | --model_id 'ETTm2_'$inp_len_m'_'$pred_len'_simple_'$method \ 77 | --data ett_m \ 78 | --seq_len $inp_len_m \ 79 | --label_len 0 \ 80 | --pred_len $pred_len \ 81 | --batch_size 1024 \ 82 | --learning_rate $lr \ 83 | --train_epochs 10 \ 84 | --decay_fac 0.75 \ 85 | --d_model 768 \ 86 | --n_heads 4 \ 87 | --d_ff 768 \ 88 | --dropout 0.3 \ 89 | --enc_in 7 \ 90 | --c_out 7 \ 91 | --freq 0 \ 92 | --patch_size 16 \ 93 | --stride 16 \ 94 | --percent $percent \ 95 | --gpt_layer 6 \ 96 | --method $method \ 97 | --gpu_loc $gpu_loc \ 98 | --itr $itt \ 99 | --patience $patience \ 100 | --model $model \ 101 | --cos 1 \ 102 | --is_gpt 1 \ 103 | --save_file_name $filename_m 104 | done 105 | done -------------------------------------------------------------------------------- /OFA/script/electricity.sh: -------------------------------------------------------------------------------- 1 | seq_len=512 2 | model=GPT4TS 3 | pre_lens_h='96 192 336 720' 4 | filename=Electricity.txt 5 | methods_h="ori removeLLM llm_to_attn llm_to_trsf" 6 | gpu_loc=0 7 | percent=100 8 | for pred_len in $pre_lens_h; 9 | do 10 | for method in $methods_h; 11 | do 12 | if [ $method == 'ori' ]; then 13 | lr=0.0001 14 | else 15 | lr=0.00005 16 | fi 17 | python eval_bs.py \ 18 | --root_path ./datasets/electricity/ \ 19 | --data_path electricity.csv \ 20 | --model_id 'Electricity_'$seq_len'_'$pred_len'_'$method'_ofa' \ 21 | --data custom \ 22 | --seq_len $seq_len \ 23 | --label_len 48 \ 24 | --pred_len $pred_len \ 25 | --batch_size 2048 \ 26 | --learning_rate $lr \ 27 | --train_epochs 10 \ 28 | --decay_fac 0.75 \ 29 | --d_model 768 \ 30 | --n_heads 4 \ 31 | --d_ff 768 \ 32 | --dropout 0.3 \ 33 | --enc_in 7 \ 34 | --c_out 7 \ 35 | --freq 0 \ 36 | --patch_size 16 \ 37 | --stride 8 \ 38 | --percent $percent \ 39 | --gpt_layer 6 \ 40 | --itr 1 \ 41 | --model $model \ 42 | --cos 1 \ 43 | --tmax 10 \ 44 | --is_gpt 1 \ 45 | --gpu_loc $gpu_loc \ 46 | --save_file_name $filename 47 | done 48 | done -------------------------------------------------------------------------------- /OFA/script/illness.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES=0 2 | 3 | seq_len=104 4 | percent=100 5 | 6 | model=GPT4TS 7 | filename=illness.txt 8 | pre_lens_h="96 192 336 720" 9 | methods_h="ori removeLLM llm_to_attn llm_to_trsf" 10 | gpu_loc=0 11 | 12 | for pred_len in $pre_lens_h; 13 | do 14 | for method in $methods_h; 15 | do 16 | if [ $method == 'ori' ]; then 17 | lr=0.0001 18 | else 19 | lr=0.00005 20 | fi 21 | python main.py \ 22 | --root_path ./datasets/illness/ \ 23 | --data_path national_illness.csv \ 24 | --model_id 'ofa_Illness_'$seq_len'_'$pred_len'_'$method \ 25 | --data custom \ 26 | --seq_len $seq_len \ 27 | --label_len 18 \ 28 | --pred_len $pred_len \ 29 | --batch_size 16 \ 30 | --learning_rate $lr \ 31 | --train_epochs 10 \ 32 | --decay_fac 0.75 \ 33 | --d_model 768 \ 34 | --n_heads 4 \ 35 | --d_ff 768 \ 36 | --freq 0 \ 37 | --patch_size 24 \ 38 | --stride 2 \ 39 | --all 1 \ 40 | --percent $percent \ 41 | --gpt_layer 6 \ 42 | --itr 3 \ 43 | --model $model \ 44 | --is_gpt 1 \ 45 | --gpu_loc $gpu_loc \ 46 | --save_file_name $filename 47 | done 48 | done 49 | 50 | -------------------------------------------------------------------------------- /OFA/script/simple/ETTh_simple.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2,3" 2 | 3 | model=DLinear_plus 4 | percent=100 5 | patience=10 6 | tag_file=main.py 7 | 8 | inp_len_h=96 9 | pre_lens_h="96 192 336 720" 10 | filename_h=ETTh_simple.txt 11 | methods_h='multi_linr_trsf multi_patch_attn' 12 | 13 | gpu_loc=0 14 | itt=5 15 | 16 | for pred_len in $pre_lens_h; 17 | do 18 | for method in $methods_h; 19 | do 20 | if echo "$method" | grep -q "linr"; then 21 | lr=0.0005 22 | else 23 | lr=0.00005 24 | fi 25 | python $tag_file \ 26 | --root_path ./datasets/ETT-small/ \ 27 | --data_path ETTh1.csv \ 28 | --model_id 'ETTh1_'$inp_len_h'_'$pred_len'_simple_'$method \ 29 | --data ett_h \ 30 | --seq_len $inp_len_h \ 31 | --label_len 0 \ 32 | --pred_len $pred_len \ 33 | --batch_size 512 \ 34 | --lradj type4 \ 35 | --learning_rate $lr \ 36 | --train_epochs 100 \ 37 | --decay_fac 0.5 \ 38 | --d_model 768 \ 39 | --n_heads 4 \ 40 | --d_ff 768 \ 41 | --dropout 0.3 \ 42 | --enc_in 7 \ 43 | --c_out 7 \ 44 | --freq 0 \ 45 | --patch_size 16 \ 46 | --stride 8 \ 47 | --patience $patience \ 48 | --percent 100 \ 49 | --gpt_layer 6 \ 50 | --itr $itt \ 51 | --model $model \ 52 | --tmax 20 \ 53 | --cos 1 \ 54 | --method $method \ 55 | --gpu_loc $gpu_loc \ 56 | --is_gpt 1 \ 57 | --save_file_name $filename_h 58 | done 59 | done 60 | 61 | for pred_len in $pre_lens_h; 62 | do 63 | for method in $methods_h; 64 | do 65 | if echo "$method" | grep -q "linr"; then 66 | lr=0.0005 67 | else 68 | lr=0.00005 69 | fi 70 | python $tag_file \ 71 | --root_path ./datasets/ETT-small/ \ 72 | --data_path ETTh2.csv \ 73 | --model_id 'ETTh2_'$inp_len_h'_'$pred_len'_simple_'$method \ 74 | --data ett_h \ 75 | --seq_len $inp_len_h \ 76 | --label_len 0 \ 77 | --pred_len $pred_len \ 78 | --batch_size 512 \ 79 | --decay_fac 0.5 \ 80 | --learning_rate $lr \ 81 | --train_epochs 10 \ 82 | --d_model 768 \ 83 | --n_heads 4 \ 84 | --d_ff 768 \ 85 | --dropout 1 \ 86 | --enc_in 7 \ 87 | --c_out 7 \ 88 | --freq 0 \ 89 | --patch_size 16 \ 90 | --stride 8 \ 91 | --percent 100 \ 92 | --gpt_layer 6 \ 93 | --patience $patience \ 94 | --itr $itt \ 95 | --model $model \ 96 | --cos 1 \ 97 | --method $method \ 98 | --gpu_loc $gpu_loc \ 99 | --tmax 20 \ 100 | --pretrain 1 \ 101 | --is_gpt 1 \ 102 | --save_file_name $filename_h 103 | done 104 | done 105 | -------------------------------------------------------------------------------- /OFA/script/simple/ETTm_simple.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2,3" 2 | model=DLinear_plus 3 | percent=100 4 | patience=10 5 | tag_file=main.py 6 | 7 | inp_len_m=512 8 | pre_lens_m="96 192 336 720" 9 | filename_m=ETTm_simple.txt 10 | methods_h='multi_linr_trsf multi_patch_attn' 11 | 12 | gpu_loc=1 13 | itt=5 14 | for pred_len in $pre_lens_m; 15 | do 16 | for method in $methods_m; 17 | do 18 | if echo "$method" | grep -q "linr"; then 19 | lr=0.0005 20 | else 21 | lr=0.00005 22 | fi 23 | python $tag_file \ 24 | --root_path ./datasets/ETT-small/ \ 25 | --data_path ETTm1.csv \ 26 | --model_id 'ETTm1_'$inp_len_m'_'$pred_len'_simple_'$method \ 27 | --data ett_m \ 28 | --seq_len $inp_len_m \ 29 | --label_len 0 \ 30 | --pred_len $pred_len \ 31 | --batch_size 1024 \ 32 | --learning_rate $lr \ 33 | --train_epochs 10 \ 34 | --decay_fac 0.75 \ 35 | --d_model 768 \ 36 | --n_heads 4 \ 37 | --d_ff 768 \ 38 | --dropout 0.3 \ 39 | --enc_in 7 \ 40 | --c_out 7 \ 41 | --freq 0 \ 42 | --patch_size 16 \ 43 | --stride 16 \ 44 | --percent $percent \ 45 | --gpt_layer 6 \ 46 | --gpu_loc $gpu_loc \ 47 | --patience $patience \ 48 | --itr $itt \ 49 | --method $method \ 50 | --model $model \ 51 | --cos 1 \ 52 | --is_gpt 1 \ 53 | --save_file_name $filename_m 54 | done 55 | done 56 | 57 | # ETTm2 58 | for pred_len in $pre_lens_m; 59 | do 60 | for method in $methods_m; 61 | do 62 | if echo "$method" | grep -q "linr"; then 63 | lr=0.0005 64 | else 65 | lr=0.00005 66 | fi 67 | python $tag_file \ 68 | --root_path ./datasets/ETT-small/ \ 69 | --data_path ETTm2.csv \ 70 | --model_id 'ETTm2_'$inp_len_m'_'$pred_len'_simple_'$method \ 71 | --data ett_m \ 72 | --seq_len $inp_len_m \ 73 | --label_len 0 \ 74 | --pred_len $pred_len \ 75 | --batch_size 1024 \ 76 | --learning_rate $lr \ 77 | --train_epochs 10 \ 78 | --decay_fac 0.75 \ 79 | --d_model 768 \ 80 | --n_heads 4 \ 81 | --d_ff 768 \ 82 | --dropout 0.3 \ 83 | --enc_in 7 \ 84 | --c_out 7 \ 85 | --freq 0 \ 86 | --patch_size 16 \ 87 | --stride 16 \ 88 | --percent $percent \ 89 | --gpt_layer 6 \ 90 | --method $method \ 91 | --gpu_loc $gpu_loc \ 92 | --itr $itt \ 93 | --patience $patience \ 94 | --model $model \ 95 | --cos 1 \ 96 | --is_gpt 1 \ 97 | --save_file_name $filename_m 98 | done 99 | done -------------------------------------------------------------------------------- /OFA/script/simple/electricity.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | 3 | seq_len=512 4 | model=DLinear_plus 5 | 6 | gpu_loc=0 7 | percent=100 8 | 9 | methods_h='multi_linr_trsf multi_patch_attn' 10 | 11 | pre_lens_h='96 192 336 720' 12 | filename=Electricity 13 | 14 | for pred_len in $pre_lens_h; 15 | do 16 | for method in $methods_h; 17 | do 18 | if echo "$method" | grep -q "linr"; then 19 | lr=0.0005 20 | else 21 | lr=0.00005 22 | fi 23 | if echo "$method" | grep -q "single"; then 24 | bs=4096 25 | else 26 | bs=16 27 | fi 28 | python $tag_file \ 29 | --root_path ./datasets/electricity/ \ 30 | --data_path electricity.csv \ 31 | --model_id 'Electricity_'$seq_len'_'$pred_len'_'$method \ 32 | --data custom \ 33 | --method $method \ 34 | --seq_len $seq_len \ 35 | --label_len 0 \ 36 | --pred_len $pred_len \ 37 | --batch_size $bs \ 38 | --learning_rate $lr \ 39 | --train_epochs 10 \ 40 | --decay_fac 0.75 \ 41 | --d_model 768 \ 42 | --n_heads 4 \ 43 | --d_ff 768 \ 44 | --dropout 0.3 \ 45 | --enc_in 7 \ 46 | --c_out 7 \ 47 | --freq 0 \ 48 | --patch_size 16 \ 49 | --stride 8 \ 50 | --percent $percent \ 51 | --gpt_layer 6 \ 52 | --itr 1 \ 53 | --model $model \ 54 | --cos 1 \ 55 | --tmax 10 \ 56 | --is_gpt 1 \ 57 | --gpu_loc $gpu_loc \ 58 | --save_file_name $filename 59 | done 60 | done -------------------------------------------------------------------------------- /OFA/script/simple/illness.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | 3 | seq_len=104 4 | percent=100 5 | gpu_loc=0 6 | model=DLinear_plus 7 | 8 | tag_file=main.py 9 | methods_h='multi_linr_trsf multi_patch_attn' 10 | 11 | filename=Illness_simple.txt 12 | pre_lens_h="24 36 48 60" 13 | for pred_len in $pre_lens_h; 14 | do 15 | for method in $methods_h; 16 | do 17 | if echo "$method" | grep -q "linr"; then 18 | lr=0.0005 19 | else 20 | lr=0.00005 21 | fi 22 | if echo "$method" | grep -q "single"; then 23 | bs=64 24 | else 25 | bs=8 26 | fi 27 | python $tag_file\ 28 | --root_path ./datasets/illness/ \ 29 | --data_path national_illness.csv \ 30 | --model_id 'Illness_'$seq_len'_'$pred_len'_'$method \ 31 | --data custom \ 32 | --seq_len $seq_len \ 33 | --label_len 0 \ 34 | --method $method \ 35 | --pred_len $pred_len \ 36 | --batch_size $bs \ 37 | --learning_rate $lr \ 38 | --train_epochs 10 \ 39 | --decay_fac 0.75 \ 40 | --d_model 768 \ 41 | --n_heads 4 \ 42 | --d_ff 768 \ 43 | --freq 0 \ 44 | --patch_size 24 \ 45 | --stride 2 \ 46 | --all 1 \ 47 | --percent $percent \ 48 | --gpt_layer 6 \ 49 | --itr 1 \ 50 | --model $model \ 51 | --is_gpt 1 \ 52 | --gpu_loc $gpu_loc \ 53 | --save_file_name $filename 54 | done 55 | done 56 | -------------------------------------------------------------------------------- /OFA/script/simple/traffic.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | seq_len=512 3 | percent=100 4 | filename=traffic_simple.txt 5 | 6 | gpu_loc=1 7 | tag_file=main.py 8 | methods_h='multi_linr_trsf multi_patch_attn' 9 | model=DLinear_plus 10 | pre_lens_h="96 192 336 720" 11 | for pred_len in $pre_lens_h; 12 | do 13 | for method in $methods_h; 14 | do 15 | if echo "$method" | grep -q "linr"; then 16 | lr=0.0005 17 | else 18 | lr=0.00005 19 | fi 20 | if echo "$method" | grep -q "single"; then 21 | bs=8192 22 | else 23 | bs=32 24 | fi 25 | python $tag_file \ 26 | --root_path ./datasets/traffic/ \ 27 | --data_path traffic.csv \ 28 | --model_id 'traffic_'$seq_len'_'$pred_len'_'$method \ 29 | --data custom \ 30 | --method $method \ 31 | --seq_len $seq_len \ 32 | --label_len 0 \ 33 | --pred_len $pred_len \ 34 | --batch_size $bs \ 35 | --learning_rate $lr \ 36 | --train_epochs 10 \ 37 | --decay_fac 0.75 \ 38 | --d_model 768 \ 39 | --n_heads 4 \ 40 | --d_ff 768 \ 41 | --freq 0 \ 42 | --patch_size 16 \ 43 | --stride 8 \ 44 | --all 1 \ 45 | --percent $percent \ 46 | --gpt_layer 6 \ 47 | --itr 1 \ 48 | --model $model \ 49 | --patience 3 \ 50 | --cos 1 \ 51 | --tmax 10 \ 52 | --is_gpt 1 \ 53 | --gpu_loc $gpu_loc \ 54 | --save_file_name $filename 55 | done 56 | done 57 | -------------------------------------------------------------------------------- /OFA/script/simple/weather.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2,3" 2 | seq_len=512 3 | percent=100 4 | gpu_loc=2 5 | filename=weather_simple.txt 6 | methods_h='multi_linr_trsf multi_patch_attn' 7 | 8 | model=DLinear_plus 9 | pre_lens_h='96 192 336 720' 10 | 11 | for pred_len in $pre_lens_h; 12 | do 13 | for method in $methods_h; 14 | do 15 | if echo "$method" | grep -q "linr"; then 16 | lr=0.0005 17 | else 18 | lr=0.00005 19 | fi 20 | if echo "$method" | grep -q "single"; then 21 | bs=4096 22 | else 23 | bs=256 24 | fi 25 | python $tag_file \ 26 | --root_path ./datasets/weather/ \ 27 | --data_path weather.csv \ 28 | --model_id 'weather_'$seq_len'_'$pred_len'_'$method \ 29 | --data custom \ 30 | --seq_len $seq_len \ 31 | --method $method \ 32 | --label_len 0 \ 33 | --pred_len $pred_len \ 34 | --batch_size $bs \ 35 | --learning_rate $lr \ 36 | --train_epochs 10 \ 37 | --decay_fac 0.9 \ 38 | --d_model 768 \ 39 | --n_heads 4 \ 40 | --d_ff 768 \ 41 | --dropout 0.3 \ 42 | --enc_in 7 \ 43 | --c_out 7 \ 44 | --freq 0 \ 45 | --lradj type3 \ 46 | --patch_size 16 \ 47 | --stride 8 \ 48 | --percent $percent \ 49 | --gpt_layer 6 \ 50 | --itr 2 \ 51 | --model $model \ 52 | --is_gpt 1 \ 53 | --gpu_loc $gpu_loc \ 54 | --save_file_name $filename 55 | done 56 | done 57 | -------------------------------------------------------------------------------- /OFA/script/traffic.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | seq_len=512 3 | 4 | model=GPT4TS 5 | filename=traffic.txt 6 | percent=100 7 | pre_lens_h="96 192 336 720" 8 | methods_h="ori removeLLM llm_to_attn llm_to_trsf" 9 | # methods_h="llm_to_trsf ori" 10 | gpu_loc=2 11 | 12 | # bash ./scripts/traffic.sh 13 | 14 | for pred_len in $pre_lens_h; 15 | do 16 | for method in $methods_h; 17 | do 18 | if [ $method == 'ori' ]; then 19 | lr=0.001 20 | else 21 | lr=0.0005 22 | fi 23 | python main.py \ 24 | --root_path ./datasets/traffic/ \ 25 | --data_path traffic.csv \ 26 | --model_id 'traffic_'$seq_len'_'$pred_len'_'$method'_ofa' \ 27 | --data custom \ 28 | --seq_len $seq_len \ 29 | --label_len 48 \ 30 | --pred_len $pred_len \ 31 | --batch_size 8192 \ 32 | --learning_rate $lr \ 33 | --train_epochs 10 \ 34 | --decay_fac 0.75 \ 35 | --d_model 768 \ 36 | --n_heads 4 \ 37 | --d_ff 768 \ 38 | --freq 0 \ 39 | --patch_size 16 \ 40 | --stride 8 \ 41 | --all 1 \ 42 | --percent $percent \ 43 | --gpt_layer 6 \ 44 | --itr 3 \ 45 | --model $model \ 46 | --patience 3 \ 47 | --cos 1 \ 48 | --tmax 10 \ 49 | --is_gpt 1 \ 50 | --gpu_loc $gpu_loc \ 51 | --save_file_name $filename 52 | done 53 | done 54 | -------------------------------------------------------------------------------- /OFA/script/weather.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | 3 | seq_len=512 4 | percent=100 5 | model=GPT4TS 6 | filename=weather.txt 7 | pre_lens_h="96 192 336 720" 8 | methods_h="ori removeLLM llm_to_attn llm_to_trsf" 9 | gpu_loc=2 10 | 11 | # bash ./scripts/weather.sh 12 | 13 | for pred_len in $pre_lens_h; 14 | do 15 | for method in $methods_h; 16 | do 17 | if [ $method == 'ori' ]; then 18 | lr=0.0001 19 | else 20 | lr=0.00005 21 | fi 22 | python main.py \ 23 | --root_path ./datasets/weather/ \ 24 | --data_path weather.csv \ 25 | --model_id 'ofa_weather_'$seq_len'_'$pred_len'_'$method \ 26 | --data custom \ 27 | --seq_len $seq_len \ 28 | --label_len 48 \ 29 | --pred_len $pred_len \ 30 | --batch_size 1024 \ 31 | --learning_rate $lr \ 32 | --train_epochs 10 \ 33 | --decay_fac 0.9 \ 34 | --d_model 768 \ 35 | --n_heads 4 \ 36 | --d_ff 768 \ 37 | --dropout 0.3 \ 38 | --enc_in 7 \ 39 | --c_out 7 \ 40 | --freq 0 \ 41 | --lradj type3 \ 42 | --patch_size 16 \ 43 | --stride 8 \ 44 | --percent $percent \ 45 | --gpt_layer 6 \ 46 | --itr 3 \ 47 | --model $model \ 48 | --is_gpt 1 \ 49 | --gpu_loc $gpu_loc \ 50 | --save_file_name $filename 51 | done 52 | done 53 | -------------------------------------------------------------------------------- /OFA/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/OFA/utils/__init__.py -------------------------------------------------------------------------------- /OFA/utils/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/OFA/utils/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /OFA/utils/__pycache__/ablUtils.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/OFA/utils/__pycache__/ablUtils.cpython-39.pyc -------------------------------------------------------------------------------- /OFA/utils/__pycache__/metrics.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/OFA/utils/__pycache__/metrics.cpython-39.pyc -------------------------------------------------------------------------------- /OFA/utils/__pycache__/timefeatures.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/OFA/utils/__pycache__/timefeatures.cpython-39.pyc -------------------------------------------------------------------------------- /OFA/utils/__pycache__/tools.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/OFA/utils/__pycache__/tools.cpython-39.pyc -------------------------------------------------------------------------------- /OFA/utils/ablUtils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | from torch import optim 5 | import os 6 | import matplotlib.pyplot as plt 7 | import numpy as np 8 | import argparse 9 | import random 10 | 11 | def perturb_sequence(batch_x , shuffle_type , patch_size = 16 , mask_ratio= 0.2 ): 12 | ''' 13 | batch_x : shape [256, 336, 1] 14 | perturb time series input 15 | sf_all : shuffle the whole sequnece 16 | sf_half : shuffle first halp sequnece 17 | ex-half : exchange first and second half 18 | ''' 19 | assert shuffle_type in ['sf_all' , 'sf_half' , 'ex_half' ,'sf_patchs' , 'masking'] 20 | if shuffle_type == 'sf_all': 21 | perm = torch.randperm(batch_x.size(1)) 22 | return batch_x[:, perm, :] 23 | if shuffle_type == 'sf_half': 24 | mid_point = batch_x.size(1) // 2 25 | pre_half = batch_x[:, :mid_point, :] 26 | post_half = batch_x[:, mid_point:, :] 27 | perm = torch.randperm(pre_half.size(1)) 28 | shuffled_pre_half = pre_half[:, perm, :] 29 | return torch.cat((shuffled_pre_half, post_half), dim=1) 30 | if shuffle_type == 'ex_half': 31 | mid_point = batch_x.size(1) // 2 32 | pre_half = batch_x[:, :mid_point, :] 33 | post_half = batch_x[:, mid_point:, :] 34 | return torch.cat((post_half, pre_half), dim=1) 35 | if shuffle_type =='sf_patchs': 36 | num_patches= (batch_x.size(1) // patch_size ) 37 | shuffle_indices = torch.randperm(num_patches) 38 | shuffled_ts = [batch_x[:, i*patch_size:(i+1)*patch_size, :] for i in shuffle_indices] 39 | if num_patches * patch_size < batch_x.size(1): 40 | shuffled_ts.append(batch_x[:, num_patches*patch_size:, :]) 41 | return torch.cat(shuffled_ts , dim=1) 42 | if shuffle_type =='masking': 43 | input_length = batch_x.size(1) 44 | num_to_mask = int(input_length * mask_ratio ) 45 | mask_indices = torch.randperm(input_length)[:num_to_mask] 46 | masked_tensor = batch_x.clone() 47 | masked_tensor[:, mask_indices, :] = 0 48 | return masked_tensor 49 | 50 | # arrayTS = torch.rand(1, 32, 1) 51 | # print(arrayTS[0,:,0]) 52 | # arrayTS = perturb_sequence(arrayTS , 'masking' , patch_size = 4 , mask_ratio= 0.8 ) 53 | # print(arrayTS[0,:,0]) 54 | -------------------------------------------------------------------------------- /OFA/utils/metrics.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def RSE(pred, true): 5 | return np.sqrt(np.sum((true - pred) ** 2)) / np.sqrt(np.sum((true - true.mean()) ** 2)) 6 | 7 | 8 | def CORR(pred, true): 9 | u = ((true - true.mean(0)) * (pred - pred.mean(0))).sum(0) 10 | d = np.sqrt(((true - true.mean(0)) ** 2 * (pred - pred.mean(0)) ** 2).sum(0)) 11 | return (u / d).mean(-1) 12 | 13 | 14 | def MAE(pred, true): 15 | return np.mean(np.abs(pred - true)) 16 | 17 | 18 | def MSE(pred, true): 19 | return np.mean((pred - true) ** 2) 20 | 21 | 22 | def RMSE(pred, true): 23 | return np.sqrt(MSE(pred, true)) 24 | 25 | 26 | def MAPE(pred, true): 27 | return np.mean(np.abs(100 * (pred - true) / (true +1e-8))) 28 | 29 | 30 | def MSPE(pred, true): 31 | return np.mean(np.square((pred - true) / (true + 1e-8))) 32 | 33 | def SMAPE(pred, true): 34 | return np.mean(200 * np.abs(pred - true) / (np.abs(pred) + np.abs(true) + 1e-8)) 35 | # return np.mean(200 * np.abs(pred - true) / (pred + true + 1e-8)) 36 | 37 | def ND(pred, true): 38 | return np.mean(np.abs(true - pred)) / np.mean(np.abs(true)) 39 | 40 | def metric(pred, true): 41 | mae = MAE(pred, true) 42 | mse = MSE(pred, true) 43 | rmse = RMSE(pred, true) 44 | mape = MAPE(pred, true) 45 | mspe = MSPE(pred, true) 46 | smape = SMAPE(pred, true) 47 | nd = ND(pred, true) 48 | 49 | return mae, mse, rmse, mape, mspe, smape, nd 50 | -------------------------------------------------------------------------------- /OFA/utils/timefeatures.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import numpy as np 4 | import pandas as pd 5 | from pandas.tseries import offsets 6 | from pandas.tseries.frequencies import to_offset 7 | 8 | 9 | class TimeFeature: 10 | def __init__(self): 11 | pass 12 | 13 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 14 | pass 15 | 16 | def __repr__(self): 17 | return self.__class__.__name__ + "()" 18 | 19 | 20 | class SecondOfMinute(TimeFeature): 21 | """Minute of hour encoded as value between [-0.5, 0.5]""" 22 | 23 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 24 | return index.second / 59.0 - 0.5 25 | 26 | 27 | class MinuteOfHour(TimeFeature): 28 | """Minute of hour encoded as value between [-0.5, 0.5]""" 29 | 30 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 31 | return index.minute / 59.0 - 0.5 32 | 33 | 34 | class HourOfDay(TimeFeature): 35 | """Hour of day encoded as value between [-0.5, 0.5]""" 36 | 37 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 38 | return index.hour / 23.0 - 0.5 39 | 40 | 41 | class DayOfWeek(TimeFeature): 42 | """Hour of day encoded as value between [-0.5, 0.5]""" 43 | 44 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 45 | return index.dayofweek / 6.0 - 0.5 46 | 47 | 48 | class DayOfMonth(TimeFeature): 49 | """Day of month encoded as value between [-0.5, 0.5]""" 50 | 51 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 52 | return (index.day - 1) / 30.0 - 0.5 53 | 54 | 55 | class DayOfYear(TimeFeature): 56 | """Day of year encoded as value between [-0.5, 0.5]""" 57 | 58 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 59 | return (index.dayofyear - 1) / 365.0 - 0.5 60 | 61 | 62 | class MonthOfYear(TimeFeature): 63 | """Month of year encoded as value between [-0.5, 0.5]""" 64 | 65 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 66 | return (index.month - 1) / 11.0 - 0.5 67 | 68 | 69 | class WeekOfYear(TimeFeature): 70 | """Week of year encoded as value between [-0.5, 0.5]""" 71 | 72 | def __call__(self, index: pd.DatetimeIndex) -> np.ndarray: 73 | return (index.isocalendar().week - 1) / 52.0 - 0.5 74 | 75 | 76 | def time_features_from_frequency_str(freq_str: str) -> List[TimeFeature]: 77 | """ 78 | Returns a list of time features that will be appropriate for the given frequency string. 79 | Parameters 80 | ---------- 81 | freq_str 82 | Frequency string of the form [multiple][granularity] such as "12H", "5min", "1D" etc. 83 | """ 84 | 85 | features_by_offsets = { 86 | offsets.YearEnd: [], 87 | offsets.QuarterEnd: [MonthOfYear], 88 | offsets.MonthEnd: [MonthOfYear], 89 | offsets.Week: [DayOfMonth, WeekOfYear], 90 | offsets.Day: [DayOfWeek, DayOfMonth, DayOfYear], 91 | offsets.BusinessDay: [DayOfWeek, DayOfMonth, DayOfYear], 92 | offsets.Hour: [HourOfDay, DayOfWeek, DayOfMonth, DayOfYear], 93 | offsets.Minute: [ 94 | MinuteOfHour, 95 | HourOfDay, 96 | DayOfWeek, 97 | DayOfMonth, 98 | DayOfYear, 99 | ], 100 | offsets.Second: [ 101 | SecondOfMinute, 102 | MinuteOfHour, 103 | HourOfDay, 104 | DayOfWeek, 105 | DayOfMonth, 106 | DayOfYear, 107 | ], 108 | } 109 | 110 | offset = to_offset(freq_str) 111 | 112 | for offset_type, feature_classes in features_by_offsets.items(): 113 | if isinstance(offset, offset_type): 114 | return [cls() for cls in feature_classes] 115 | 116 | supported_freq_msg = f""" 117 | Unsupported frequency {freq_str} 118 | The following frequencies are supported: 119 | Y - yearly 120 | alias: A 121 | M - monthly 122 | W - weekly 123 | D - daily 124 | B - business days 125 | H - hourly 126 | T - minutely 127 | alias: min 128 | S - secondly 129 | """ 130 | raise RuntimeError(supported_freq_msg) 131 | 132 | 133 | def time_features(dates, freq='h'): 134 | return np.vstack([feat(dates) for feat in time_features_from_frequency_str(freq)]) 135 | -------------------------------------------------------------------------------- /PAttn/__pycache__/embed.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/__pycache__/embed.cpython-311.pyc -------------------------------------------------------------------------------- /PAttn/data_provider/__pycache__/data_factory.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/data_provider/__pycache__/data_factory.cpython-311.pyc -------------------------------------------------------------------------------- /PAttn/data_provider/__pycache__/data_factory.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/data_provider/__pycache__/data_factory.cpython-39.pyc -------------------------------------------------------------------------------- /PAttn/data_provider/__pycache__/data_loader.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/data_provider/__pycache__/data_loader.cpython-311.pyc -------------------------------------------------------------------------------- /PAttn/data_provider/__pycache__/data_loader.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/data_provider/__pycache__/data_loader.cpython-39.pyc -------------------------------------------------------------------------------- /PAttn/data_provider/data_factory.py: -------------------------------------------------------------------------------- 1 | from data_provider.data_loader import Dataset_Custom, Dataset_ETT_hour, Dataset_ETT_minute 2 | from torch.utils.data import DataLoader 3 | 4 | data_dict = { 5 | 'custom': Dataset_Custom, 6 | 'ett_h': Dataset_ETT_hour, 7 | 'ett_m': Dataset_ETT_minute, 8 | } 9 | def data_provider(args, flag, drop_last_test=True, train_all=False): 10 | Data = data_dict[args.data] 11 | timeenc = 0 if args.embed != 'timeF' else 1 12 | percent = args.percent 13 | max_len = args.max_len 14 | 15 | if flag == 'test': 16 | shuffle_flag = False 17 | drop_last = drop_last_test 18 | batch_size = args.batch_size 19 | freq = args.freq 20 | elif flag == 'val': 21 | shuffle_flag = True 22 | drop_last = drop_last_test 23 | batch_size = args.batch_size 24 | freq = args.freq 25 | else: 26 | shuffle_flag = True 27 | drop_last = True 28 | batch_size = args.batch_size 29 | freq = args.freq 30 | 31 | data_set = Data( 32 | model_id=args.model_id , 33 | root_path=args.root_path, 34 | data_path=args.data_path, 35 | flag=flag, 36 | size=[args.seq_len, args.label_len, args.pred_len], 37 | features=args.features, 38 | target=args.target, 39 | timeenc=timeenc, 40 | freq=freq, 41 | percent=percent, 42 | max_len=max_len, 43 | train_all=train_all 44 | ) 45 | data_loader = DataLoader( 46 | data_set, 47 | batch_size=batch_size, 48 | shuffle=shuffle_flag, 49 | num_workers=args.num_workers, 50 | drop_last=drop_last) 51 | return data_set, data_loader 52 | -------------------------------------------------------------------------------- /PAttn/models/Attention.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | class ScaledDotProductAttention(nn.Module): 6 | ''' Scaled Dot-Product Attention ''' 7 | 8 | def __init__(self, temperature, attn_dropout=0.1): 9 | super().__init__() 10 | self.temperature = temperature 11 | self.dropout = nn.Dropout(attn_dropout) 12 | 13 | def forward(self, q, k, v, mask=None): 14 | 15 | attn = torch.matmul(q / self.temperature, k.transpose(2, 3)) 16 | 17 | if mask is not None: 18 | attn = attn.masked_fill(mask == 0, -1e9) 19 | 20 | attn = self.dropout(F.softmax(attn, dim=-1)) 21 | output = torch.matmul(attn, v) 22 | 23 | return output, attn 24 | 25 | class MultiHeadAttention(nn.Module): 26 | ''' Multi-Head Attention module ''' 27 | def __init__(self, d_model = -1 ,n_head = 8 , d_k = -1 , d_v = -1 , dropout=0.1): 28 | super().__init__() 29 | self.n_head = n_head 30 | d_k = d_model // n_head 31 | d_v = d_k 32 | self.d_k = d_k 33 | self.d_v = d_v 34 | 35 | self.w_qs = nn.Linear(d_model, n_head * d_k, bias=False) 36 | self.w_ks = nn.Linear(d_model, n_head * d_k, bias=False) 37 | self.w_vs = nn.Linear(d_model, n_head * d_v, bias=False) 38 | self.fc = nn.Linear(n_head * d_v, d_model, bias=False) 39 | 40 | self.attention = ScaledDotProductAttention(temperature=d_k ** 0.5) 41 | 42 | self.dropout = nn.Dropout(dropout) 43 | self.layer_norm = nn.LayerNorm(d_model, eps=1e-6) 44 | 45 | 46 | def forward(self, q, k, v, mask=None): 47 | 48 | d_k, d_v, n_head = self.d_k, self.d_v, self.n_head 49 | sz_b, len_q, len_k, len_v = q.size(0), q.size(1), k.size(1), v.size(1) 50 | 51 | residual = q 52 | 53 | # Pass through the pre-attention projection: b x lq x (n*dv) 54 | # Separate different heads: b x lq x n x dv 55 | q = self.w_qs(q).view(sz_b, len_q, n_head, d_k) 56 | k = self.w_ks(k).view(sz_b, len_k, n_head, d_k) 57 | v = self.w_vs(v).view(sz_b, len_v, n_head, d_v) 58 | 59 | # Transpose for attention dot product: b x n x lq x dv 60 | q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2) 61 | 62 | if mask is not None: 63 | mask = mask.unsqueeze(1) # For head axis broadcasting. 64 | 65 | q, attn = self.attention(q, k, v, mask=mask) 66 | 67 | # Transpose to move the head dimension back: b x lq x n x dv 68 | # Combine the last two dimensions to concatenate all the heads together: b x lq x (n*dv) 69 | q = q.transpose(1, 2).contiguous().view(sz_b, len_q, -1) 70 | q = self.dropout(self.fc(q)) 71 | q += residual 72 | 73 | q = self.layer_norm(q) 74 | 75 | return q, attn -------------------------------------------------------------------------------- /PAttn/models/PAttn.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | import numpy as np 5 | from einops import rearrange 6 | from models.Attention import MultiHeadAttention 7 | 8 | class Encoder_LLaTA(nn.Module): 9 | def __init__(self, input_dim , hidden_dim=768, num_heads=12, num_encoder_layers=1): 10 | super(Encoder_LLaTA, self).__init__() 11 | self.linear = nn.Linear(input_dim, hidden_dim) 12 | encoder_layer = nn.TransformerEncoderLayer(d_model=hidden_dim, nhead=num_heads) 13 | self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_encoder_layers) 14 | 15 | def forward(self, x): 16 | x = self.linear(x) 17 | x = self.transformer_encoder(x.transpose(0, 1)).transpose(0, 1) 18 | return x 19 | 20 | class moving_avg(nn.Module): 21 | """ 22 | Moving average block to highlight the trend of time series 23 | """ 24 | def __init__(self, kernel_size, stride): 25 | super(moving_avg, self).__init__() 26 | self.kernel_size = kernel_size 27 | self.avg = nn.AvgPool1d(kernel_size=kernel_size, stride=stride, padding=0) 28 | 29 | def forward(self, x): 30 | # padding on the both ends of time series 31 | front = x[:, 0:1, :].repeat(1, (self.kernel_size - 1) // 2, 1) 32 | end = x[:, -1:, :].repeat(1, (self.kernel_size - 1) // 2, 1) 33 | x = torch.cat([front, x, end], dim=1) 34 | x = self.avg(x.permute(0, 2, 1)) 35 | x = x.permute(0, 2, 1) 36 | return x 37 | 38 | class series_decomp(nn.Module): 39 | """ 40 | Series decomposition block 41 | """ 42 | def __init__(self, kernel_size): 43 | super(series_decomp, self).__init__() 44 | self.moving_avg = moving_avg(kernel_size, stride=1) 45 | 46 | def forward(self, x): 47 | moving_mean = self.moving_avg(x) 48 | res = x - moving_mean 49 | return res, moving_mean 50 | 51 | class PAttn(nn.Module): 52 | """ 53 | pattn PAttn 54 | 55 | Decomposition-Linear 56 | """ 57 | def __init__(self, configs, device): 58 | super(PAttn, self).__init__() 59 | self.seq_len = configs.seq_len 60 | self.pred_len = configs.pred_len 61 | self.patch_size = configs.patch_size 62 | self.stride = configs.patch_size //2 63 | 64 | self.d_model = configs.d_model 65 | self.method = configs.method 66 | 67 | self.patch_num = (configs.seq_len - self.patch_size) // self.stride + 2 68 | self.padding_patch_layer = nn.ReplicationPad1d((0, self.stride)) 69 | self.in_layer = nn.Linear(self.patch_size, self.d_model) 70 | self.basic_attn = MultiHeadAttention(d_model =self.d_model ) 71 | self.out_layer = nn.Linear(self.d_model * self.patch_num, configs.pred_len) 72 | 73 | def norm(self, x, dim =0, means= None , stdev=None): 74 | if means is not None : 75 | return x * stdev + means 76 | else : 77 | means = x.mean(dim, keepdim=True).detach() 78 | x = x - means 79 | stdev = torch.sqrt(torch.var(x, dim=dim, keepdim=True, unbiased=False)+ 1e-5).detach() 80 | x /= stdev 81 | return x , means , stdev 82 | 83 | def forward(self, x): 84 | if self.method == 'PAttn': 85 | B , C = x.size(0) , x.size(1) 86 | # [Batch, Channel, 336] 87 | x , means, stdev = self.norm(x , dim=2) 88 | # [Batch, Channel, 344] 89 | x = self.padding_patch_layer(x) 90 | # [Batch, Channel, 12, 16] 91 | x = x.unfold(dimension=-1, size=self.patch_size, step=self.stride) 92 | # [Batch, Channel, 12, 768] 93 | x = self.in_layer(x) 94 | x = rearrange(x, 'b c m l -> (b c) m l') 95 | x , _ = self.basic_attn( x ,x ,x ) 96 | x = rearrange(x, '(b c) m l -> b c (m l)' , b=B , c=C) 97 | x = self.out_layer(x) 98 | x = self.norm(x , means=means, stdev=stdev ) 99 | return x 100 | -------------------------------------------------------------------------------- /PAttn/models/__pycache__/Attention.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/models/__pycache__/Attention.cpython-311.pyc -------------------------------------------------------------------------------- /PAttn/models/__pycache__/Attention.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/models/__pycache__/Attention.cpython-39.pyc -------------------------------------------------------------------------------- /PAttn/models/__pycache__/DLinear.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/models/__pycache__/DLinear.cpython-39.pyc -------------------------------------------------------------------------------- /PAttn/models/__pycache__/DLinear_plus.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/models/__pycache__/DLinear_plus.cpython-39.pyc -------------------------------------------------------------------------------- /PAttn/models/__pycache__/GPT4TS.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/models/__pycache__/GPT4TS.cpython-39.pyc -------------------------------------------------------------------------------- /PAttn/models/__pycache__/NLinear.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/models/__pycache__/NLinear.cpython-39.pyc -------------------------------------------------------------------------------- /PAttn/models/__pycache__/PAttn.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/models/__pycache__/PAttn.cpython-311.pyc -------------------------------------------------------------------------------- /PAttn/models/__pycache__/PatchTST.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/models/__pycache__/PatchTST.cpython-311.pyc -------------------------------------------------------------------------------- /PAttn/models/__pycache__/PatchTST.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/models/__pycache__/PatchTST.cpython-39.pyc -------------------------------------------------------------------------------- /PAttn/script/ETTh.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2,3" 2 | 3 | percent=100 4 | patience=10 5 | tag_file=main.py 6 | 7 | inp_len_h=96 8 | pre_lens_h="96 192 336 720" 9 | filename_h=ETTh_simple.txt 10 | 11 | model=PAttn 12 | methods_h='PAttn' 13 | 14 | gpu_loc=0 15 | itt=1 16 | lr=0.00005 17 | 18 | for pred_len in $pre_lens_h; 19 | do 20 | for method in $methods_h; 21 | do 22 | python $tag_file \ 23 | --root_path ./datasets/ETT-small/ \ 24 | --data_path ETTh1.csv \ 25 | --model_id 'ETTh1_'$inp_len_h'_'$pred_len'_simple_'$method \ 26 | --data ett_h \ 27 | --seq_len $inp_len_h \ 28 | --label_len 0 \ 29 | --pred_len $pred_len \ 30 | --batch_size 512 \ 31 | --lradj type4 \ 32 | --learning_rate $lr \ 33 | --train_epochs 100 \ 34 | --decay_fac 0.5 \ 35 | --d_model 768 \ 36 | --n_heads 4 \ 37 | --d_ff 768 \ 38 | --dropout 0.3 \ 39 | --enc_in 7 \ 40 | --c_out 7 \ 41 | --freq 0 \ 42 | --patch_size 16 \ 43 | --stride 8 \ 44 | --patience $patience \ 45 | --percent 100 \ 46 | --gpt_layer 6 \ 47 | --itr $itt \ 48 | --model $model \ 49 | --tmax 20 \ 50 | --cos 1 \ 51 | --method $method \ 52 | --gpu_loc $gpu_loc \ 53 | --is_gpt 1 \ 54 | --save_file_name $filename_h 55 | done 56 | done 57 | 58 | for pred_len in $pre_lens_h; 59 | do 60 | for method in $methods_h; 61 | do 62 | python $tag_file \ 63 | --root_path ./datasets/ETT-small/ \ 64 | --data_path ETTh2.csv \ 65 | --model_id 'ETTh2_'$inp_len_h'_'$pred_len'_simple_'$method \ 66 | --data ett_h \ 67 | --seq_len $inp_len_h \ 68 | --label_len 0 \ 69 | --pred_len $pred_len \ 70 | --batch_size 512 \ 71 | --decay_fac 0.5 \ 72 | --learning_rate $lr \ 73 | --train_epochs 10 \ 74 | --d_model 768 \ 75 | --n_heads 4 \ 76 | --d_ff 768 \ 77 | --dropout 1 \ 78 | --enc_in 7 \ 79 | --c_out 7 \ 80 | --freq 0 \ 81 | --patch_size 16 \ 82 | --stride 8 \ 83 | --percent 100 \ 84 | --gpt_layer 6 \ 85 | --patience $patience \ 86 | --itr $itt \ 87 | --model $model \ 88 | --cos 1 \ 89 | --method $method \ 90 | --gpu_loc $gpu_loc \ 91 | --tmax 20 \ 92 | --pretrain 1 \ 93 | --is_gpt 1 \ 94 | --save_file_name $filename_h 95 | done 96 | done 97 | -------------------------------------------------------------------------------- /PAttn/script/ETTm.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2,3" 2 | model=PAttn 3 | methods_m='PAttn' 4 | 5 | percent=100 6 | patience=10 7 | tag_file=main.py 8 | 9 | inp_len_m=512 10 | pre_lens_m="96 192 336 720" 11 | filename_m=ETTm_simple.txt 12 | lr=0.00005 13 | gpu_loc=0 14 | itt=1 15 | for pred_len in $pre_lens_m; 16 | do 17 | for method in $methods_m; 18 | do 19 | python $tag_file \ 20 | --root_path ./datasets/ETT-small/ \ 21 | --data_path ETTm1.csv \ 22 | --model_id 'ETTm1_'$inp_len_m'_'$pred_len'_simple_'$method \ 23 | --data ett_m \ 24 | --seq_len $inp_len_m \ 25 | --label_len 0 \ 26 | --pred_len $pred_len \ 27 | --batch_size 1024 \ 28 | --learning_rate $lr \ 29 | --train_epochs 10 \ 30 | --decay_fac 0.75 \ 31 | --d_model 768 \ 32 | --n_heads 4 \ 33 | --d_ff 768 \ 34 | --dropout 0.3 \ 35 | --enc_in 7 \ 36 | --c_out 7 \ 37 | --freq 0 \ 38 | --patch_size 16 \ 39 | --stride 16 \ 40 | --percent $percent \ 41 | --gpt_layer 6 \ 42 | --gpu_loc $gpu_loc \ 43 | --patience $patience \ 44 | --itr $itt \ 45 | --method $method \ 46 | --model $model \ 47 | --cos 1 \ 48 | --is_gpt 1 \ 49 | --save_file_name $filename_m 50 | done 51 | done 52 | 53 | # ETTm2 54 | for pred_len in $pre_lens_m; 55 | do 56 | for method in $methods_m; 57 | do 58 | if echo "$method" | grep -q "linr"; then 59 | lr=0.0005 60 | else 61 | lr=0.00005 62 | fi 63 | python $tag_file \ 64 | --root_path ./datasets/ETT-small/ \ 65 | --data_path ETTm2.csv \ 66 | --model_id 'ETTm2_'$inp_len_m'_'$pred_len'_simple_'$method \ 67 | --data ett_m \ 68 | --seq_len $inp_len_m \ 69 | --label_len 0 \ 70 | --pred_len $pred_len \ 71 | --batch_size 1024 \ 72 | --learning_rate $lr \ 73 | --train_epochs 10 \ 74 | --decay_fac 0.75 \ 75 | --d_model 768 \ 76 | --n_heads 4 \ 77 | --d_ff 768 \ 78 | --dropout 0.3 \ 79 | --enc_in 7 \ 80 | --c_out 7 \ 81 | --freq 0 \ 82 | --patch_size 16 \ 83 | --stride 16 \ 84 | --percent $percent \ 85 | --gpt_layer 6 \ 86 | --method $method \ 87 | --gpu_loc $gpu_loc \ 88 | --itr $itt \ 89 | --patience $patience \ 90 | --model $model \ 91 | --cos 1 \ 92 | --is_gpt 1 \ 93 | --save_file_name $filename_m 94 | done 95 | done -------------------------------------------------------------------------------- /PAttn/script/electricity.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | 3 | seq_len=512 4 | 5 | model=PAttn 6 | methods_h='PAttn' 7 | 8 | gpu_loc=0 9 | percent=100 10 | 11 | pre_lens_h='96 192 336 720' 12 | filename=Electricity 13 | 14 | for pred_len in $pre_lens_h; 15 | do 16 | for method in $methods_h; 17 | do 18 | lr=0.00005 19 | bs=16 20 | fi 21 | python $tag_file \ 22 | --root_path ./datasets/electricity/ \ 23 | --data_path electricity.csv \ 24 | --model_id 'Electricity_'$seq_len'_'$pred_len'_'$method \ 25 | --data custom \ 26 | --method $method \ 27 | --seq_len $seq_len \ 28 | --label_len 0 \ 29 | --pred_len $pred_len \ 30 | --batch_size $bs \ 31 | --learning_rate $lr \ 32 | --train_epochs 10 \ 33 | --decay_fac 0.75 \ 34 | --d_model 768 \ 35 | --n_heads 4 \ 36 | --d_ff 768 \ 37 | --dropout 0.3 \ 38 | --enc_in 7 \ 39 | --c_out 7 \ 40 | --freq 0 \ 41 | --patch_size 16 \ 42 | --stride 8 \ 43 | --percent $percent \ 44 | --gpt_layer 6 \ 45 | --itr 1 \ 46 | --model $model \ 47 | --cos 1 \ 48 | --tmax 10 \ 49 | --is_gpt 1 \ 50 | --gpu_loc $gpu_loc \ 51 | --save_file_name $filename 52 | done 53 | done -------------------------------------------------------------------------------- /PAttn/script/illness.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | 3 | seq_len=104 4 | percent=100 5 | gpu_loc=0 6 | model=PAttn 7 | methods_h='PAttn' 8 | 9 | tag_file=main.py 10 | 11 | 12 | filename=Illness_simple.txt 13 | pre_lens_h="24 36 48 60" 14 | for pred_len in $pre_lens_h; 15 | do 16 | for method in $methods_h; 17 | do 18 | lr=0.00005 19 | bs=8 20 | python $tag_file\ 21 | --root_path ./datasets/illness/ \ 22 | --data_path national_illness.csv \ 23 | --model_id 'Illness_'$seq_len'_'$pred_len'_'$method \ 24 | --data custom \ 25 | --seq_len $seq_len \ 26 | --label_len 0 \ 27 | --method $method \ 28 | --pred_len $pred_len \ 29 | --batch_size $bs \ 30 | --learning_rate $lr \ 31 | --train_epochs 10 \ 32 | --decay_fac 0.75 \ 33 | --d_model 768 \ 34 | --n_heads 4 \ 35 | --d_ff 768 \ 36 | --freq 0 \ 37 | --patch_size 24 \ 38 | --stride 2 \ 39 | --all 1 \ 40 | --percent $percent \ 41 | --gpt_layer 6 \ 42 | --itr 1 \ 43 | --model $model \ 44 | --is_gpt 1 \ 45 | --gpu_loc $gpu_loc \ 46 | --save_file_name $filename 47 | done 48 | done 49 | -------------------------------------------------------------------------------- /PAttn/script/traffic.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | seq_len=512 3 | percent=100 4 | filename=traffic.txt 5 | 6 | gpu_loc=0 7 | tag_file=main.py 8 | model=PAttn 9 | methods_h='PAttn' 10 | pre_lens_h="96 192 336 720" 11 | lr=0.00005 12 | bs=32 13 | for pred_len in $pre_lens_h; 14 | do 15 | for method in $methods_h; 16 | do 17 | python $tag_file \ 18 | --root_path ./datasets/traffic/ \ 19 | --data_path traffic.csv \ 20 | --model_id 'traffic_'$seq_len'_'$pred_len'_'$method \ 21 | --data custom \ 22 | --method $method \ 23 | --seq_len $seq_len \ 24 | --label_len 0 \ 25 | --pred_len $pred_len \ 26 | --batch_size $bs \ 27 | --learning_rate $lr \ 28 | --train_epochs 10 \ 29 | --decay_fac 0.75 \ 30 | --d_model 768 \ 31 | --n_heads 4 \ 32 | --d_ff 768 \ 33 | --freq 0 \ 34 | --patch_size 16 \ 35 | --stride 8 \ 36 | --all 1 \ 37 | --percent $percent \ 38 | --gpt_layer 6 \ 39 | --itr 1 \ 40 | --model $model \ 41 | --patience 3 \ 42 | --cos 1 \ 43 | --tmax 10 \ 44 | --is_gpt 1 \ 45 | --gpu_loc $gpu_loc \ 46 | --save_file_name $filename 47 | done 48 | done 49 | -------------------------------------------------------------------------------- /PAttn/script/weather.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2,3" 2 | seq_len=512 3 | percent=100 4 | gpu_loc=0 5 | filename=weather.txt 6 | model=PAttn 7 | methods_h='PAttn' 8 | pre_lens_h='96 192 336 720' 9 | lr=0.00005 10 | bs=256 11 | for pred_len in $pre_lens_h; 12 | do 13 | for method in $methods_h; 14 | do 15 | python main.py \ 16 | --root_path ./datasets/weather/ \ 17 | --data_path weather.csv \ 18 | --model_id 'weather_'$seq_len'_'$pred_len'_'$method \ 19 | --data custom \ 20 | --seq_len $seq_len \ 21 | --method $method \ 22 | --label_len 0 \ 23 | --pred_len $pred_len \ 24 | --batch_size $bs \ 25 | --learning_rate $lr \ 26 | --train_epochs 10 \ 27 | --decay_fac 0.9 \ 28 | --d_model 768 \ 29 | --n_heads 4 \ 30 | --d_ff 768 \ 31 | --dropout 0.3 \ 32 | --enc_in 7 \ 33 | --c_out 7 \ 34 | --freq 0 \ 35 | --lradj type3 \ 36 | --patch_size 16 \ 37 | --stride 8 \ 38 | --percent $percent \ 39 | --gpt_layer 6 \ 40 | --itr 1 \ 41 | --model $model \ 42 | --is_gpt 1 \ 43 | --gpu_loc $gpu_loc \ 44 | --save_file_name $filename 45 | done 46 | done 47 | -------------------------------------------------------------------------------- /PAttn/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/utils/__init__.py -------------------------------------------------------------------------------- /PAttn/utils/__pycache__/__init__.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/utils/__pycache__/__init__.cpython-311.pyc -------------------------------------------------------------------------------- /PAttn/utils/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/utils/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /PAttn/utils/__pycache__/ablUtils.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/utils/__pycache__/ablUtils.cpython-311.pyc -------------------------------------------------------------------------------- /PAttn/utils/__pycache__/ablUtils.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/utils/__pycache__/ablUtils.cpython-39.pyc -------------------------------------------------------------------------------- /PAttn/utils/__pycache__/metrics.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/utils/__pycache__/metrics.cpython-311.pyc -------------------------------------------------------------------------------- /PAttn/utils/__pycache__/metrics.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/utils/__pycache__/metrics.cpython-39.pyc -------------------------------------------------------------------------------- /PAttn/utils/__pycache__/timefeatures.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/utils/__pycache__/timefeatures.cpython-311.pyc -------------------------------------------------------------------------------- /PAttn/utils/__pycache__/timefeatures.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/utils/__pycache__/timefeatures.cpython-39.pyc -------------------------------------------------------------------------------- /PAttn/utils/__pycache__/tools.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/utils/__pycache__/tools.cpython-311.pyc -------------------------------------------------------------------------------- /PAttn/utils/__pycache__/tools.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/PAttn/utils/__pycache__/tools.cpython-39.pyc -------------------------------------------------------------------------------- /PAttn/utils/ablUtils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | from torch import optim 5 | import os 6 | import matplotlib.pyplot as plt 7 | import numpy as np 8 | import argparse 9 | import random 10 | 11 | def perturb_sequence(batch_x , shuffle_type , patch_size = 16 , mask_ratio= 0.2 ): 12 | ''' 13 | batch_x : shape [256, 336, 1] 14 | perturb time series input 15 | sf_all : shuffle the whole sequnece 16 | sf_half : shuffle first halp sequnece 17 | ex-half : exchange first and second half 18 | ''' 19 | assert shuffle_type in ['sf_all' , 'sf_half' , 'ex_half' ,'sf_patchs' , 'masking'] 20 | if shuffle_type == 'sf_all': 21 | perm = torch.randperm(batch_x.size(1)) 22 | return batch_x[:, perm, :] 23 | if shuffle_type == 'sf_half': 24 | mid_point = batch_x.size(1) // 2 25 | pre_half = batch_x[:, :mid_point, :] 26 | post_half = batch_x[:, mid_point:, :] 27 | perm = torch.randperm(pre_half.size(1)) 28 | shuffled_pre_half = pre_half[:, perm, :] 29 | return torch.cat((shuffled_pre_half, post_half), dim=1) 30 | if shuffle_type == 'ex_half': 31 | mid_point = batch_x.size(1) // 2 32 | pre_half = batch_x[:, :mid_point, :] 33 | post_half = batch_x[:, mid_point:, :] 34 | return torch.cat((post_half, pre_half), dim=1) 35 | if shuffle_type =='sf_patchs': 36 | num_patches= (batch_x.size(1) // patch_size ) 37 | shuffle_indices = torch.randperm(num_patches) 38 | shuffled_ts = [batch_x[:, i*patch_size:(i+1)*patch_size, :] for i in shuffle_indices] 39 | if num_patches * patch_size < batch_x.size(1): 40 | shuffled_ts.append(batch_x[:, num_patches*patch_size:, :]) 41 | return torch.cat(shuffled_ts , dim=1) 42 | if shuffle_type =='masking': 43 | input_length = batch_x.size(1) 44 | num_to_mask = int(input_length * mask_ratio ) 45 | mask_indices = torch.randperm(input_length)[:num_to_mask] 46 | masked_tensor = batch_x.clone() 47 | masked_tensor[:, mask_indices, :] = 0 48 | return masked_tensor 49 | 50 | # arrayTS = torch.rand(1, 32, 1) 51 | # print(arrayTS[0,:,0]) 52 | # arrayTS = perturb_sequence(arrayTS , 'masking' , patch_size = 4 , mask_ratio= 0.8 ) 53 | # print(arrayTS[0,:,0]) 54 | -------------------------------------------------------------------------------- /PAttn/utils/metrics.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def RSE(pred, true): 5 | return np.sqrt(np.sum((true - pred) ** 2)) / np.sqrt(np.sum((true - true.mean()) ** 2)) 6 | 7 | 8 | def CORR(pred, true): 9 | u = ((true - true.mean(0)) * (pred - pred.mean(0))).sum(0) 10 | d = np.sqrt(((true - true.mean(0)) ** 2 * (pred - pred.mean(0)) ** 2).sum(0)) 11 | return (u / d).mean(-1) 12 | 13 | 14 | def MAE(pred, true): 15 | return np.mean(np.abs(pred - true)) 16 | 17 | 18 | def MSE(pred, true): 19 | return np.mean((pred - true) ** 2) 20 | 21 | 22 | def RMSE(pred, true): 23 | return np.sqrt(MSE(pred, true)) 24 | 25 | 26 | def MAPE(pred, true): 27 | return np.mean(np.abs(100 * (pred - true) / (true +1e-8))) 28 | 29 | 30 | def MSPE(pred, true): 31 | return np.mean(np.square((pred - true) / (true + 1e-8))) 32 | 33 | def SMAPE(pred, true): 34 | return np.mean(200 * np.abs(pred - true) / (np.abs(pred) + np.abs(true) + 1e-8)) 35 | # return np.mean(200 * np.abs(pred - true) / (pred + true + 1e-8)) 36 | 37 | def ND(pred, true): 38 | return np.mean(np.abs(true - pred)) / np.mean(np.abs(true)) 39 | 40 | def metric(pred, true): 41 | mae = MAE(pred, true) 42 | mse = MSE(pred, true) 43 | rmse = RMSE(pred, true) 44 | mape = MAPE(pred, true) 45 | mspe = MSPE(pred, true) 46 | smape = SMAPE(pred, true) 47 | nd = ND(pred, true) 48 | 49 | return mae, mse, rmse, mape, mspe, smape, nd 50 | -------------------------------------------------------------------------------- /Time-LLM-exp/README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 |

(ICLR'24) Time-LLM: Time Series Forecasting by Reprogramming Large Language Models

5 |
6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |

14 | 15 | 16 | ``` 17 | @inproceedings{jin2023time, 18 | title={{Time-LLM}: Time series forecasting by reprogramming large language models}, 19 | author={Jin, Ming and Wang, Shiyu and Ma, Lintao and Chu, Zhixuan and Zhang, James Y and Shi, Xiaoming and Chen, Pin-Yu and Liang, Yuxuan and Li, Yuan-Fang and Pan, Shirui and Wen, Qingsong}, 20 | booktitle={International Conference on Learning Representations (ICLR)}, 21 | year={2024} 22 | } 23 | ``` 24 | 25 | ## Updates 26 | 🚩 **News** (March 2024): Time-LLM has been upgraded to serve as a general framework for repurposing a wide range of language models to time series forecasting. It now defaults to supporting Llama-7B and includes compatibility with two additional smaller PLMs (GPT-2 and BERT). Simply adjust `--llm_model` and `--llm_dim` to switch backbones. 27 | 28 | ## Introduction 29 | Time-LLM is a reprogramming framework to repurpose LLMs for general time series forecasting with the backbone language models kept intact. 30 | Notably, we show that time series analysis (e.g., forecasting) can be cast as yet another "language task" that can be effectively tackled by an off-the-shelf LLM. 31 | 32 |

33 | 34 |

35 | 36 | - Time-LLM comprises two key components: (1) reprogramming the input time series into text prototype representations that are more natural for the LLM, and (2) augmenting the input context with declarative prompts (e.g., domain expert knowledge and task instructions) to guide LLM reasoning. 37 | 38 |

39 | 40 |

41 | 42 | ## Requirements 43 | Use python 3.11 from MiniConda 44 | 45 | - torch==2.2.2 46 | - accelerate==0.28.0 47 | - einops==0.7.0 48 | - matplotlib==3.7.0 49 | - numpy==1.23.5 50 | - pandas==1.5.3 51 | - scikit_learn==1.2.2 52 | - scipy==1.12.0 53 | - tqdm==4.65.0 54 | - peft==0.4.0 55 | - transformers==4.31.0 56 | - deepspeed==0.14.0 57 | - sentencepiece==0.2.0 58 | 59 | To install all dependencies: 60 | ``` 61 | pip install -r requirements.txt 62 | ``` 63 | 64 | ## Datasets 65 | You can access the well pre-processed datasets from [[Google Drive]](https://drive.google.com/file/d/1NF7VEefXCmXuWNbnNe858WvQAkJ_7wuP/view?usp=sharing), then place the downloaded contents under `./dataset` 66 | 67 | 68 | 69 | ## Acknowledgement 70 | Our implementation adapts [Time-Series-Library](https://github.com/thuml/Time-Series-Library) and [OFA (GPT4TS)](https://github.com/DAMO-DI-ML/NeurIPS2023-One-Fits-All) as the code base and have extensively modified it to our purposes. We thank the authors for sharing their implementations and related resources. 71 | -------------------------------------------------------------------------------- /Time-LLM-exp/data_provider/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Time-LLM-exp/data_provider/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/data_provider/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/data_provider/__pycache__/data_factory.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/data_provider/__pycache__/data_factory.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/data_provider/__pycache__/data_loader.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/data_provider/__pycache__/data_loader.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/data_provider/__pycache__/m4.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/data_provider/__pycache__/m4.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/data_provider/data_factory.py: -------------------------------------------------------------------------------- 1 | from data_provider.data_loader import Dataset_ETT_hour, Dataset_ETT_minute, Dataset_Custom, Dataset_M4 2 | from torch.utils.data import DataLoader 3 | 4 | data_dict = { 5 | 'ETTh1': Dataset_ETT_hour, 6 | 'ETTh2': Dataset_ETT_hour, 7 | 'ETTm1': Dataset_ETT_minute, 8 | 'ETTm2': Dataset_ETT_minute, 9 | 'ECL': Dataset_Custom, 10 | 'Traffic': Dataset_Custom, 11 | 'Illness': Dataset_Custom, 12 | 'Weather': Dataset_Custom, 13 | 'm4': Dataset_M4, 14 | } 15 | 16 | def data_provider(args, flag): 17 | Data = data_dict[args.data] 18 | timeenc = 0 if args.embed != 'timeF' else 1 19 | percent = args.percent 20 | 21 | if flag == 'test': 22 | shuffle_flag = False 23 | drop_last = True 24 | batch_size = args.batch_size 25 | freq = args.freq 26 | else: 27 | shuffle_flag = True 28 | drop_last = True 29 | batch_size = args.batch_size 30 | freq = args.freq 31 | 32 | if args.data == 'm4': 33 | drop_last = False 34 | data_set = Data( 35 | root_path=args.root_path, 36 | data_path=args.data_path, 37 | flag=flag, 38 | size=[args.seq_len, args.label_len, args.pred_len], 39 | features=args.features, 40 | target=args.target, 41 | timeenc=timeenc, 42 | freq=freq, 43 | seasonal_patterns=args.seasonal_patterns 44 | ) 45 | else: 46 | data_set = Data( 47 | root_path=args.root_path, 48 | data_path=args.data_path, 49 | flag=flag, 50 | size=[args.seq_len, args.label_len, args.pred_len], 51 | features=args.features, 52 | target=args.target, 53 | timeenc=timeenc, 54 | freq=freq, 55 | percent=percent, 56 | seasonal_patterns=args.seasonal_patterns 57 | ) 58 | data_loader = DataLoader( 59 | data_set, 60 | batch_size=batch_size, 61 | shuffle=shuffle_flag, 62 | num_workers=args.num_workers, 63 | drop_last=drop_last) 64 | return data_set, data_loader 65 | -------------------------------------------------------------------------------- /Time-LLM-exp/ds_config_zero2.json: -------------------------------------------------------------------------------- 1 | { 2 | "bf16": { 3 | "enabled": true, 4 | "auto_cast": true 5 | }, 6 | "zero_optimization": { 7 | "stage": 2, 8 | "allgather_partitions": true, 9 | "allgather_bucket_size": 2e8, 10 | "overlap_comm": true, 11 | "reduce_scatter": true, 12 | "reduce_bucket_size": 2e8, 13 | "contiguous_gradients": true, 14 | "sub_group_size": 1e9 15 | }, 16 | "gradient_accumulation_steps": "auto", 17 | "train_batch_size": "auto", 18 | "train_micro_batch_size_per_gpu": "auto", 19 | "steps_per_print": 10, 20 | "wall_clock_breakdown": false 21 | } -------------------------------------------------------------------------------- /Time-LLM-exp/layers/Conv_Blocks.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | 5 | class Inception_Block_V1(nn.Module): 6 | def __init__(self, in_channels, out_channels, num_kernels=6, init_weight=True): 7 | super(Inception_Block_V1, self).__init__() 8 | self.in_channels = in_channels 9 | self.out_channels = out_channels 10 | self.num_kernels = num_kernels 11 | kernels = [] 12 | for i in range(self.num_kernels): 13 | kernels.append(nn.Conv2d(in_channels, out_channels, kernel_size=2 * i + 1, padding=i)) 14 | self.kernels = nn.ModuleList(kernels) 15 | if init_weight: 16 | self._initialize_weights() 17 | 18 | def _initialize_weights(self): 19 | for m in self.modules(): 20 | if isinstance(m, nn.Conv2d): 21 | nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') 22 | if m.bias is not None: 23 | nn.init.constant_(m.bias, 0) 24 | 25 | def forward(self, x): 26 | res_list = [] 27 | for i in range(self.num_kernels): 28 | res_list.append(self.kernels[i](x)) 29 | res = torch.stack(res_list, dim=-1).mean(-1) 30 | return res 31 | 32 | 33 | class Inception_Block_V2(nn.Module): 34 | def __init__(self, in_channels, out_channels, num_kernels=6, init_weight=True): 35 | super(Inception_Block_V2, self).__init__() 36 | self.in_channels = in_channels 37 | self.out_channels = out_channels 38 | self.num_kernels = num_kernels 39 | kernels = [] 40 | for i in range(self.num_kernels // 2): 41 | kernels.append(nn.Conv2d(in_channels, out_channels, kernel_size=[1, 2 * i + 3], padding=[0, i + 1])) 42 | kernels.append(nn.Conv2d(in_channels, out_channels, kernel_size=[2 * i + 3, 1], padding=[i + 1, 0])) 43 | kernels.append(nn.Conv2d(in_channels, out_channels, kernel_size=1)) 44 | self.kernels = nn.ModuleList(kernels) 45 | if init_weight: 46 | self._initialize_weights() 47 | 48 | def _initialize_weights(self): 49 | for m in self.modules(): 50 | if isinstance(m, nn.Conv2d): 51 | nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') 52 | if m.bias is not None: 53 | nn.init.constant_(m.bias, 0) 54 | 55 | def forward(self, x): 56 | res_list = [] 57 | for i in range(self.num_kernels + 1): 58 | res_list.append(self.kernels[i](x)) 59 | res = torch.stack(res_list, dim=-1).mean(-1) 60 | return res 61 | -------------------------------------------------------------------------------- /Time-LLM-exp/layers/StandardNorm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | 5 | class Normalize(nn.Module): 6 | def __init__(self, num_features: int, eps=1e-5, affine=False, subtract_last=False, non_norm=False): 7 | """ 8 | :param num_features: the number of features or channels 9 | :param eps: a value added for numerical stability 10 | :param affine: if True, RevIN has learnable affine parameters 11 | """ 12 | super(Normalize, self).__init__() 13 | self.num_features = num_features 14 | self.eps = eps 15 | self.affine = affine 16 | self.subtract_last = subtract_last 17 | self.non_norm = non_norm 18 | if self.affine: 19 | self._init_params() 20 | 21 | def forward(self, x, mode: str): 22 | if mode == 'norm': 23 | self._get_statistics(x) 24 | x = self._normalize(x) 25 | elif mode == 'denorm': 26 | x = self._denormalize(x) 27 | else: 28 | raise NotImplementedError 29 | return x 30 | 31 | def _init_params(self): 32 | # initialize RevIN params: (C,) 33 | self.affine_weight = nn.Parameter(torch.ones(self.num_features)) 34 | self.affine_bias = nn.Parameter(torch.zeros(self.num_features)) 35 | 36 | def _get_statistics(self, x): 37 | dim2reduce = tuple(range(1, x.ndim - 1)) 38 | if self.subtract_last: 39 | self.last = x[:, -1, :].unsqueeze(1) 40 | else: 41 | self.mean = torch.mean(x, dim=dim2reduce, keepdim=True).detach() 42 | self.stdev = torch.sqrt(torch.var(x, dim=dim2reduce, keepdim=True, unbiased=False) + self.eps).detach() 43 | 44 | def _normalize(self, x): 45 | if self.non_norm: 46 | return x 47 | if self.subtract_last: 48 | x = x - self.last 49 | else: 50 | x = x - self.mean 51 | x = x / self.stdev 52 | if self.affine: 53 | x = x * self.affine_weight 54 | x = x + self.affine_bias 55 | return x 56 | 57 | def _denormalize(self, x): 58 | if self.non_norm: 59 | return x 60 | if self.affine: 61 | x = x - self.affine_bias 62 | x = x / (self.affine_weight + self.eps * self.eps) 63 | x = x * self.stdev 64 | if self.subtract_last: 65 | x = x + self.last 66 | else: 67 | x = x + self.mean 68 | return x 69 | -------------------------------------------------------------------------------- /Time-LLM-exp/layers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/layers/__init__.py -------------------------------------------------------------------------------- /Time-LLM-exp/layers/__pycache__/AutoCorrelation.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/layers/__pycache__/AutoCorrelation.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/layers/__pycache__/Autoformer_EncDec.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/layers/__pycache__/Autoformer_EncDec.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/layers/__pycache__/Embed.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/layers/__pycache__/Embed.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/layers/__pycache__/StandardNorm.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/layers/__pycache__/StandardNorm.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/layers/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/layers/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/models/Attention.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | class ScaledDotProductAttention(nn.Module): 6 | ''' Scaled Dot-Product Attention ''' 7 | 8 | def __init__(self, temperature, attn_dropout=0.1): 9 | super().__init__() 10 | self.temperature = temperature 11 | self.dropout = nn.Dropout(attn_dropout) 12 | 13 | def forward(self, q, k, v, mask=None): 14 | 15 | attn = torch.matmul(q / self.temperature, k.transpose(2, 3)) 16 | 17 | if mask is not None: 18 | attn = attn.masked_fill(mask == 0, -1e9) 19 | 20 | attn = self.dropout(F.softmax(attn, dim=-1)) 21 | output = torch.matmul(attn, v) 22 | 23 | return output, attn 24 | 25 | class MultiHeadAttention(nn.Module): 26 | ''' Multi-Head Attention module ''' 27 | def __init__(self, d_model = -1 ,n_head = 8 , d_k = -1 , d_v = -1 , dropout=0.1): 28 | super().__init__() 29 | self.n_head = n_head 30 | d_k = d_model // n_head 31 | d_v = d_k 32 | self.d_k = d_k 33 | self.d_v = d_v 34 | 35 | self.w_qs = nn.Linear(d_model, n_head * d_k, bias=False) 36 | self.w_ks = nn.Linear(d_model, n_head * d_k, bias=False) 37 | self.w_vs = nn.Linear(d_model, n_head * d_v, bias=False) 38 | self.fc = nn.Linear(n_head * d_v, d_model, bias=False) 39 | 40 | self.attention = ScaledDotProductAttention(temperature=d_k ** 0.5) 41 | 42 | self.dropout = nn.Dropout(dropout) 43 | self.layer_norm = nn.LayerNorm(d_model, eps=1e-6) 44 | 45 | 46 | def forward(self, q, k, v, mask=None): 47 | 48 | d_k, d_v, n_head = self.d_k, self.d_v, self.n_head 49 | sz_b, len_q, len_k, len_v = q.size(0), q.size(1), k.size(1), v.size(1) 50 | 51 | residual = q 52 | 53 | # Pass through the pre-attention projection: b x lq x (n*dv) 54 | # Separate different heads: b x lq x n x dv 55 | q = self.w_qs(q).view(sz_b, len_q, n_head, d_k) 56 | k = self.w_ks(k).view(sz_b, len_k, n_head, d_k) 57 | v = self.w_vs(v).view(sz_b, len_v, n_head, d_v) 58 | 59 | # Transpose for attention dot product: b x n x lq x dv 60 | q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2) 61 | 62 | if mask is not None: 63 | mask = mask.unsqueeze(1) # For head axis broadcasting. 64 | 65 | q, attn = self.attention(q, k, v, mask=mask) 66 | 67 | # Transpose to move the head dimension back: b x lq x n x dv 68 | # Combine the last two dimensions to concatenate all the heads together: b x lq x (n*dv) 69 | q = q.transpose(1, 2).contiguous().view(sz_b, len_q, -1) 70 | q = self.dropout(self.fc(q)) 71 | q += residual 72 | 73 | q = self.layer_norm(q) 74 | 75 | return q, attn -------------------------------------------------------------------------------- /Time-LLM-exp/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/models/__init__.py -------------------------------------------------------------------------------- /Time-LLM-exp/models/__pycache__/Attention.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/models/__pycache__/Attention.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/models/__pycache__/Autoformer.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/models/__pycache__/Autoformer.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/models/__pycache__/DLinear.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/models/__pycache__/DLinear.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/models/__pycache__/TimeLLM.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/models/__pycache__/TimeLLM.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/models/__pycache__/TimeLLM_eval.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/models/__pycache__/TimeLLM_eval.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/models/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/models/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/requirements.txt: -------------------------------------------------------------------------------- 1 | torch==2.2.2 2 | accelerate==0.28.0 3 | einops==0.7.0 4 | matplotlib==3.7.0 5 | numpy==1.23.5 6 | pandas==1.5.3 7 | scikit_learn==1.2.2 8 | scipy==1.12.0 9 | tqdm==4.65.0 10 | peft==0.4.0 11 | transformers==4.31.0 12 | deepspeed==0.14.0 13 | sentencepiece==0.2.0 14 | -------------------------------------------------------------------------------- /Time-LLM-exp/scripts/train_ori/TimeLLM_ECL.sh: -------------------------------------------------------------------------------- 1 | model_name=TimeLLM 2 | train_epochs=10 3 | learning_rate=0.01 4 | llama_layers=32 5 | 6 | master_port=00097 7 | num_process=8 8 | batch_size=24 9 | d_model=16 10 | d_ff=32 11 | 12 | comment='TimeLLM-ECL' 13 | 14 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 15 | --task_name long_term_forecast \ 16 | --is_training 1 \ 17 | --root_path ./dataset/electricity/ \ 18 | --data_path electricity.csv \ 19 | --model_id ECL_512_96 \ 20 | --model $model_name \ 21 | --data ECL \ 22 | --features M \ 23 | --seq_len 512 \ 24 | --label_len 48 \ 25 | --pred_len 96 \ 26 | --e_layers 2 \ 27 | --d_layers 1 \ 28 | --factor 3 \ 29 | --enc_in 321 \ 30 | --dec_in 321 \ 31 | --c_out 321 \ 32 | --batch_size $batch_size \ 33 | --learning_rate $learning_rate \ 34 | --llm_layers $llama_layers \ 35 | --train_epochs $train_epochs \ 36 | --model_comment $comment 37 | 38 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 39 | --task_name long_term_forecast \ 40 | --is_training 1 \ 41 | --root_path ./dataset/electricity/ \ 42 | --data_path electricity.csv \ 43 | --model_id ECL_512_192 \ 44 | --model $model_name \ 45 | --data ECL \ 46 | --features M \ 47 | --seq_len 512 \ 48 | --label_len 48 \ 49 | --pred_len 192 \ 50 | --e_layers 2 \ 51 | --d_layers 1 \ 52 | --factor 3 \ 53 | --enc_in 321 \ 54 | --dec_in 321 \ 55 | --c_out 321 \ 56 | --batch_size $batch_size \ 57 | --learning_rate $learning_rate \ 58 | --llm_layers $llama_layers \ 59 | --train_epochs $train_epochs \ 60 | --model_comment $comment 61 | 62 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 63 | --task_name long_term_forecast \ 64 | --is_training 1 \ 65 | --root_path ./dataset/electricity/ \ 66 | --data_path electricity.csv \ 67 | --model_id ECL_512_336 \ 68 | --model $model_name \ 69 | --data ECL \ 70 | --features M \ 71 | --seq_len 512 \ 72 | --label_len 48 \ 73 | --pred_len 336 \ 74 | --e_layers 2 \ 75 | --d_layers 1 \ 76 | --factor 3 \ 77 | --enc_in 321 \ 78 | --dec_in 321 \ 79 | --c_out 321 \ 80 | --batch_size $batch_size \ 81 | --learning_rate $learning_rate \ 82 | --llm_layers $llama_layers \ 83 | --train_epochs $train_epochs \ 84 | --model_comment $comment 85 | 86 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 87 | --task_name long_term_forecast \ 88 | --is_training 1 \ 89 | --root_path ./dataset/electricity/ \ 90 | --data_path electricity.csv \ 91 | --model_id ECL_512_720 \ 92 | --model $model_name \ 93 | --data ECL \ 94 | --features M \ 95 | --seq_len 512 \ 96 | --label_len 48 \ 97 | --pred_len 720 \ 98 | --e_layers 2 \ 99 | --d_layers 1 \ 100 | --factor 3 \ 101 | --enc_in 321 \ 102 | --dec_in 321 \ 103 | --c_out 321 \ 104 | --batch_size $batch_size \ 105 | --learning_rate $learning_rate \ 106 | --llm_layers $llama_layers \ 107 | --train_epochs $train_epochs \ 108 | --model_comment $comment -------------------------------------------------------------------------------- /Time-LLM-exp/scripts/train_ori/TimeLLM_ETTh1.sh: -------------------------------------------------------------------------------- 1 | model_name=TimeLLM 2 | train_epochs=100 3 | learning_rate=0.01 4 | llama_layers=32 5 | 6 | master_port=00097 7 | num_process=8 8 | batch_size=24 9 | d_model=32 10 | d_ff=128 11 | 12 | comment='TimeLLM-ETTh1' 13 | 14 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 15 | --task_name long_term_forecast \ 16 | --is_training 1 \ 17 | --root_path ./dataset/ETT-small/ \ 18 | --data_path ETTh1.csv \ 19 | --model_id ETTh1_512_96 \ 20 | --model $model_name \ 21 | --data ETTh1 \ 22 | --features M \ 23 | --seq_len 512 \ 24 | --label_len 48 \ 25 | --pred_len 96 \ 26 | --factor 3 \ 27 | --enc_in 7 \ 28 | --dec_in 7 \ 29 | --c_out 7 \ 30 | --des 'Exp' \ 31 | --itr 1 \ 32 | --d_model $d_model \ 33 | --d_ff $d_ff \ 34 | --batch_size $batch_size \ 35 | --learning_rate $learning_rate \ 36 | --llm_layers $llama_layers \ 37 | --train_epochs $train_epochs \ 38 | --model_comment $comment 39 | 40 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 41 | --task_name long_term_forecast \ 42 | --is_training 1 \ 43 | --root_path ./dataset/ETT-small/ \ 44 | --data_path ETTh1.csv \ 45 | --model_id ETTh1_512_192 \ 46 | --model $model_name \ 47 | --data ETTh1 \ 48 | --features M \ 49 | --seq_len 512 \ 50 | --label_len 48 \ 51 | --pred_len 192 \ 52 | --factor 3 \ 53 | --enc_in 7 \ 54 | --dec_in 7 \ 55 | --c_out 7 \ 56 | --des 'Exp' \ 57 | --itr 1 \ 58 | --d_model 32 \ 59 | --d_ff 128 \ 60 | --batch_size $batch_size \ 61 | --learning_rate 0.02 \ 62 | --llm_layers $llama_layers \ 63 | --train_epochs $train_epochs \ 64 | --model_comment $comment 65 | 66 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 67 | --task_name long_term_forecast \ 68 | --is_training 1 \ 69 | --root_path ./dataset/ETT-small/ \ 70 | --data_path ETTh1.csv \ 71 | --model_id ETTh1_512_336 \ 72 | --model $model_name \ 73 | --data ETTh1 \ 74 | --features M \ 75 | --seq_len 512 \ 76 | --label_len 48 \ 77 | --pred_len 336 \ 78 | --factor 3 \ 79 | --enc_in 7 \ 80 | --dec_in 7 \ 81 | --c_out 7 \ 82 | --des 'Exp' \ 83 | --itr 1 \ 84 | --d_model $d_model \ 85 | --d_ff $d_ff \ 86 | --batch_size $batch_size \ 87 | --lradj 'COS'\ 88 | --learning_rate 0.001 \ 89 | --llm_layers $llama_layers \ 90 | --train_epochs $train_epochs \ 91 | --model_comment $comment 92 | 93 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 94 | --task_name long_term_forecast \ 95 | --is_training 1 \ 96 | --root_path ./dataset/ETT-small/ \ 97 | --data_path ETTh1.csv \ 98 | --model_id ETTh1_512_720 \ 99 | --model $model_name \ 100 | --data ETTh1 \ 101 | --features M \ 102 | --seq_len 512 \ 103 | --label_len 48 \ 104 | --pred_len 720 \ 105 | --factor 3 \ 106 | --enc_in 7 \ 107 | --dec_in 7 \ 108 | --c_out 7 \ 109 | --des 'Exp' \ 110 | --itr 1 \ 111 | --d_model $d_model \ 112 | --d_ff $d_ff \ 113 | --batch_size $batch_size \ 114 | --learning_rate $learning_rate \ 115 | --llm_layers $llama_layers \ 116 | --train_epochs $train_epochs \ 117 | --model_comment $comment -------------------------------------------------------------------------------- /Time-LLM-exp/scripts/train_ori/TimeLLM_ETTh1_ETTh2.sh: -------------------------------------------------------------------------------- 1 | model_name=TimeLLM 2 | learning_rate=0.01 3 | llama_layers=32 4 | 5 | master_port=00097 6 | num_process=8 7 | batch_size=24 8 | d_model=32 9 | d_ff=128 10 | 11 | comment='TimeLLM-ETTh1_ETTh2' 12 | 13 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_pretrain.py \ 14 | --task_name long_term_forecast \ 15 | --is_training 1 \ 16 | --root_path ./dataset/ETT-small/ \ 17 | --data_path_pretrain ETTh1.csv \ 18 | --data_path ETTh2.csv \ 19 | --model_id ETTh1_ETTh2_512_96 \ 20 | --model $model_name \ 21 | --data_pretrain ETTh1 \ 22 | --data ETTh2 \ 23 | --features M \ 24 | --seq_len 512 \ 25 | --label_len 48 \ 26 | --pred_len 96 \ 27 | --factor 3 \ 28 | --enc_in 7 \ 29 | --dec_in 7 \ 30 | --c_out 7 \ 31 | --des 'Exp' \ 32 | --itr 1 \ 33 | --d_model $d_model \ 34 | --d_ff $d_ff \ 35 | --batch_size $batch_size \ 36 | --learning_rate $learning_rate \ 37 | --llm_layers $llama_layers \ 38 | --train_epochs 5 \ 39 | --model_comment $comment 40 | 41 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_pretrain.py \ 42 | --task_name long_term_forecast \ 43 | --is_training 1 \ 44 | --root_path ./dataset/ETT-small/ \ 45 | --data_path_pretrain ETTh1.csv \ 46 | --data_path ETTh2.csv \ 47 | --model_id ETTh1_ETTh2_512_192 \ 48 | --model $model_name \ 49 | --data_pretrain ETTh1 \ 50 | --data ETTh2 \ 51 | --features M \ 52 | --seq_len 512 \ 53 | --label_len 48 \ 54 | --pred_len 192 \ 55 | --factor 3 \ 56 | --enc_in 7 \ 57 | --dec_in 7 \ 58 | --c_out 7 \ 59 | --des 'Exp' \ 60 | --itr 1 \ 61 | --d_model 32 \ 62 | --d_ff 128 \ 63 | --batch_size $batch_size \ 64 | --learning_rate 0.02 \ 65 | --llm_layers $llama_layers \ 66 | --train_epochs 5 \ 67 | --model_comment $comment 68 | 69 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_pretrain.py \ 70 | --task_name long_term_forecast \ 71 | --is_training 1 \ 72 | --root_path ./dataset/ETT-small/ \ 73 | --data_path_pretrain ETTh1.csv \ 74 | --data_path ETTh2.csv \ 75 | --model_id ETTh1_ETTh2_512_336 \ 76 | --model $model_name \ 77 | --data_pretrain ETTh1 \ 78 | --data ETTh2 \ 79 | --features M \ 80 | --seq_len 512 \ 81 | --label_len 48 \ 82 | --pred_len 336 \ 83 | --factor 3 \ 84 | --enc_in 7 \ 85 | --dec_in 7 \ 86 | --c_out 7 \ 87 | --des 'Exp' \ 88 | --itr 1 \ 89 | --d_model $d_model \ 90 | --d_ff $d_ff \ 91 | --batch_size $batch_size \ 92 | --lradj 'COS'\ 93 | --learning_rate 0.001 \ 94 | --llm_layers $llama_layers \ 95 | --train_epochs 5 \ 96 | --model_comment $comment 97 | 98 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_pretrain.py \ 99 | --task_name long_term_forecast \ 100 | --is_training 1 \ 101 | --root_path ./dataset/ETT-small/ \ 102 | --data_path_pretrain ETTh1.csv \ 103 | --data_path ETTh2.csv \ 104 | --model_id ETTh1_ETTh2_512_720 \ 105 | --model $model_name \ 106 | --data_pretrain ETTh1 \ 107 | --data ETTh2 \ 108 | --features M \ 109 | --seq_len 512 \ 110 | --label_len 48 \ 111 | --pred_len 720 \ 112 | --factor 3 \ 113 | --enc_in 7 \ 114 | --dec_in 7 \ 115 | --c_out 7 \ 116 | --des 'Exp' \ 117 | --itr 1 \ 118 | --d_model $d_model \ 119 | --d_ff $d_ff \ 120 | --batch_size $batch_size \ 121 | --learning_rate $learning_rate \ 122 | --llm_layers $llama_layers \ 123 | --train_epochs 5 \ 124 | --model_comment $comment -------------------------------------------------------------------------------- /Time-LLM-exp/scripts/train_ori/TimeLLM_ETTh2.sh: -------------------------------------------------------------------------------- 1 | model_name=TimeLLM 2 | train_epochs=10 3 | learning_rate=0.01 4 | llama_layers=32 5 | 6 | master_port=00098 7 | num_process=8 8 | batch_size=24 9 | d_model=32 10 | d_ff=128 11 | 12 | comment='TimeLLM-ETTh2' 13 | 14 | 15 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 16 | --task_name long_term_forecast \ 17 | --is_training 1 \ 18 | --root_path ./dataset/ETT-small/ \ 19 | --data_path ETTh2.csv \ 20 | --model_id ETTh2_512_96 \ 21 | --model $model_name \ 22 | --data ETTh2 \ 23 | --features M \ 24 | --seq_len 512 \ 25 | --label_len 48 \ 26 | --pred_len 96 \ 27 | --factor 3 \ 28 | --enc_in 7 \ 29 | --dec_in 7 \ 30 | --c_out 7 \ 31 | --des 'Exp' \ 32 | --itr 1 \ 33 | --d_model $d_model \ 34 | --d_ff $d_ff \ 35 | --batch_size $batch_size \ 36 | --learning_rate $learning_rate \ 37 | --llm_layers $llama_layers \ 38 | --train_epochs $train_epochs \ 39 | --model_comment $comment 40 | 41 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 42 | --task_name long_term_forecast \ 43 | --is_training 1 \ 44 | --root_path ./dataset/ETT-small/ \ 45 | --data_path ETTh2.csv \ 46 | --model_id ETTh2_512_192 \ 47 | --model $model_name \ 48 | --data ETTh2 \ 49 | --features M \ 50 | --seq_len 512 \ 51 | --label_len 48 \ 52 | --pred_len 192 \ 53 | --factor 3 \ 54 | --enc_in 7 \ 55 | --dec_in 7 \ 56 | --c_out 7 \ 57 | --des 'Exp' \ 58 | --itr 1 \ 59 | --d_model $d_model \ 60 | --d_ff $d_ff \ 61 | --batch_size $batch_size \ 62 | --lradj 'TST'\ 63 | --learning_rate 0.002 \ 64 | --llm_layers $llama_layers \ 65 | --train_epochs $train_epochs \ 66 | --model_comment $comment 67 | 68 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 69 | --task_name long_term_forecast \ 70 | --is_training 1 \ 71 | --root_path ./dataset/ETT-small/ \ 72 | --data_path ETTh2.csv \ 73 | --model_id ETTh2_512_336 \ 74 | --model $model_name \ 75 | --data ETTh2 \ 76 | --features M \ 77 | --seq_len 512 \ 78 | --label_len 48 \ 79 | --pred_len 336 \ 80 | --factor 3 \ 81 | --enc_in 7 \ 82 | --dec_in 7 \ 83 | --c_out 7 \ 84 | --des 'Exp' \ 85 | --itr 1 \ 86 | --d_model $d_model \ 87 | --d_ff $d_ff \ 88 | --batch_size $batch_size \ 89 | --lradj 'TST'\ 90 | --learning_rate 0.005 \ 91 | --llm_layers $llama_layers \ 92 | --train_epochs $train_epochs \ 93 | --model_comment $comment 94 | 95 | 96 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 97 | --task_name long_term_forecast \ 98 | --is_training 1 \ 99 | --root_path ./dataset/ETT-small/ \ 100 | --data_path ETTh2.csv \ 101 | --model_id ETTh2_512_720 \ 102 | --model $model_name \ 103 | --data ETTh2 \ 104 | --features M \ 105 | --seq_len 512 \ 106 | --label_len 48 \ 107 | --pred_len 720 \ 108 | --factor 3 \ 109 | --enc_in 7 \ 110 | --dec_in 7 \ 111 | --c_out 7 \ 112 | --des 'Exp' \ 113 | --itr 1 \ 114 | --d_model 16 \ 115 | --d_ff 128 \ 116 | --batch_size $batch_size \ 117 | --learning_rate 0.005 \ 118 | --lradj 'TST'\ 119 | --llm_layers $llama_layers \ 120 | --train_epochs 20 \ 121 | --patience 10 \ 122 | --model_comment $comment -------------------------------------------------------------------------------- /Time-LLM-exp/scripts/train_ori/TimeLLM_ETTm1.sh: -------------------------------------------------------------------------------- 1 | model_name=TimeLLM 2 | train_epochs=100 3 | learning_rate=0.01 4 | llama_layers=32 5 | 6 | master_port=00097 7 | num_process=8 8 | batch_size=24 9 | d_model=32 10 | d_ff=128 11 | 12 | comment='TimeLLM-ETTm1' 13 | 14 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 15 | --task_name long_term_forecast \ 16 | --is_training 1 \ 17 | --root_path ./dataset/ETT-small/ \ 18 | --data_path ETTm1.csv \ 19 | --model_id ETTm1_512_96 \ 20 | --model $model_name \ 21 | --data ETTm1 \ 22 | --features M \ 23 | --seq_len 512 \ 24 | --label_len 48 \ 25 | --pred_len 96 \ 26 | --factor 3 \ 27 | --enc_in 7 \ 28 | --dec_in 7 \ 29 | --c_out 7 \ 30 | --des 'Exp' \ 31 | --itr 1 \ 32 | --d_model $d_model \ 33 | --d_ff $d_ff \ 34 | --batch_size $batch_size \ 35 | --lradj 'TST'\ 36 | --learning_rate 0.001 \ 37 | --llm_layers $llama_layers \ 38 | --train_epochs $train_epochs \ 39 | --model_comment $comment 40 | 41 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 42 | --task_name long_term_forecast \ 43 | --is_training 1 \ 44 | --root_path ./dataset/ETT-small/ \ 45 | --data_path ETTm1.csv \ 46 | --model_id ETTm1_512_192 \ 47 | --model $model_name \ 48 | --data ETTm1 \ 49 | --features M \ 50 | --seq_len 512 \ 51 | --label_len 48 \ 52 | --pred_len 192 \ 53 | --factor 3 \ 54 | --enc_in 7 \ 55 | --dec_in 7 \ 56 | --c_out 7 \ 57 | --des 'Exp' \ 58 | --itr 1 \ 59 | --d_model $d_model \ 60 | --d_ff $d_ff \ 61 | --batch_size $batch_size \ 62 | --learning_rate $learning_rate \ 63 | --lradj 'TST'\ 64 | --learning_rate 0.001 \ 65 | --llm_layers $llama_layers \ 66 | --train_epochs $train_epochs \ 67 | --patience 20 \ 68 | --model_comment $comment 69 | 70 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 71 | --task_name long_term_forecast \ 72 | --is_training 1 \ 73 | --root_path ./dataset/ETT-small/ \ 74 | --data_path ETTm1.csv \ 75 | --model_id ETTm1_512_336 \ 76 | --model $model_name \ 77 | --data ETTm1 \ 78 | --features M \ 79 | --seq_len 512 \ 80 | --label_len 48 \ 81 | --pred_len 336 \ 82 | --factor 3 \ 83 | --enc_in 7 \ 84 | --dec_in 7 \ 85 | --c_out 7 \ 86 | --des 'Exp' \ 87 | --itr 1 \ 88 | --d_model $d_model \ 89 | --d_ff $d_ff \ 90 | --batch_size $batch_size \ 91 | --learning_rate $learning_rate \ 92 | --lradj 'TST'\ 93 | --learning_rate 0.001 \ 94 | --llm_layers $llama_layers \ 95 | --train_epochs $train_epochs \ 96 | --patience 20 \ 97 | --model_comment $comment 98 | 99 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 100 | --task_name long_term_forecast \ 101 | --is_training 1 \ 102 | --root_path ./dataset/ETT-small/ \ 103 | --data_path ETTm1.csv \ 104 | --model_id ETTm1_512_720 \ 105 | --model $model_name \ 106 | --data ETTm1 \ 107 | --features M \ 108 | --seq_len 512 \ 109 | --label_len 48 \ 110 | --pred_len 720 \ 111 | --factor 3 \ 112 | --enc_in 7 \ 113 | --dec_in 7 \ 114 | --c_out 7 \ 115 | --des 'Exp' \ 116 | --itr 1 \ 117 | --d_model $d_model \ 118 | --d_ff $d_ff \ 119 | --batch_size $batch_size \ 120 | --learning_rate $learning_rate \ 121 | --lradj 'TST'\ 122 | --learning_rate 0.001 \ 123 | --llm_layers $llama_layers \ 124 | --train_epochs $train_epochs \ 125 | --patience 20 \ 126 | --model_comment $comment -------------------------------------------------------------------------------- /Time-LLM-exp/scripts/train_ori/TimeLLM_ETTm2.sh: -------------------------------------------------------------------------------- 1 | model_name=TimeLLM 2 | train_epochs=10 3 | learning_rate=0.01 4 | llama_layers=32 5 | 6 | master_port=00097 7 | num_process=8 8 | batch_size=24 9 | d_model=32 10 | d_ff=128 11 | 12 | comment='TimeLLM-ETTm2' 13 | 14 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 15 | --task_name long_term_forecast \ 16 | --is_training 1 \ 17 | --root_path ./dataset/ETT-small/ \ 18 | --data_path ETTm2.csv \ 19 | --model_id ETTm2_512_96 \ 20 | --model $model_name \ 21 | --data ETTm2 \ 22 | --features M \ 23 | --seq_len 512 \ 24 | --label_len 48 \ 25 | --pred_len 96 \ 26 | --factor 3 \ 27 | --enc_in 7 \ 28 | --dec_in 7 \ 29 | --c_out 7 \ 30 | --des 'Exp' \ 31 | --itr 1 \ 32 | --d_model $d_model \ 33 | --d_ff $d_ff \ 34 | --batch_size 16 \ 35 | --learning_rate $learning_rate \ 36 | --llm_layers $llama_layers \ 37 | --train_epochs $train_epochs \ 38 | --model_comment $comment 39 | 40 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 41 | --task_name long_term_forecast \ 42 | --is_training 1 \ 43 | --root_path ./dataset/ETT-small/ \ 44 | --data_path ETTm2.csv \ 45 | --model_id ETTm2_512_192 \ 46 | --model $model_name \ 47 | --data ETTm2 \ 48 | --features M \ 49 | --seq_len 512 \ 50 | --label_len 48 \ 51 | --pred_len 192 \ 52 | --factor 3 \ 53 | --enc_in 7 \ 54 | --dec_in 7 \ 55 | --c_out 7 \ 56 | --des 'Exp' \ 57 | --itr 1 \ 58 | --d_model $d_model \ 59 | --d_ff $d_ff \ 60 | --batch_size $batch_size \ 61 | --learning_rate $learning_rate \ 62 | --lradj 'TST'\ 63 | --learning_rate 0.002 \ 64 | --llm_layers $llama_layers \ 65 | --train_epochs $train_epochs \ 66 | --model_comment $comment 67 | 68 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 69 | --task_name long_term_forecast \ 70 | --is_training 1 \ 71 | --root_path ./dataset/ETT-small/ \ 72 | --data_path ETTm2.csv \ 73 | --model_id ETTm2_512_336 \ 74 | --model $model_name \ 75 | --data ETTm2 \ 76 | --features M \ 77 | --seq_len 512 \ 78 | --label_len 48 \ 79 | --pred_len 336 \ 80 | --factor 3 \ 81 | --enc_in 7 \ 82 | --dec_in 7 \ 83 | --c_out 7 \ 84 | --des 'Exp' \ 85 | --itr 1 \ 86 | --d_model $d_model \ 87 | --d_ff $d_ff \ 88 | --batch_size $batch_size \ 89 | --learning_rate $learning_rate \ 90 | --lradj 'TST'\ 91 | --learning_rate 0.002 \ 92 | --llm_layers $llama_layers \ 93 | --train_epochs $train_epochs \ 94 | --model_comment $comment 95 | 96 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 97 | --task_name long_term_forecast \ 98 | --is_training 1 \ 99 | --root_path ./dataset/ETT-small/ \ 100 | --data_path ETTm2.csv \ 101 | --model_id ETTm2_512_720 \ 102 | --model $model_name \ 103 | --data ETTm2 \ 104 | --features M \ 105 | --seq_len 512 \ 106 | --label_len 48 \ 107 | --pred_len 720 \ 108 | --factor 3 \ 109 | --enc_in 7 \ 110 | --dec_in 7 \ 111 | --c_out 7 \ 112 | --des 'Exp' \ 113 | --itr 1 \ 114 | --d_model $d_model \ 115 | --d_ff $d_ff \ 116 | --batch_size $batch_size \ 117 | --learning_rate $learning_rate \ 118 | --lradj 'TST'\ 119 | --learning_rate 0.002 \ 120 | --llm_layers $llama_layers \ 121 | --train_epochs $train_epochs \ 122 | --model_comment $comment 123 | 124 | 125 | -------------------------------------------------------------------------------- /Time-LLM-exp/scripts/train_ori/TimeLLM_Traffic.sh: -------------------------------------------------------------------------------- 1 | model_name=TimeLLM 2 | train_epochs=10 3 | learning_rate=0.01 4 | llama_layers=32 5 | 6 | master_port=00097 7 | num_process=8 8 | batch_size=24 9 | d_model=16 10 | d_ff=32 11 | 12 | comment='TimeLLM-Traffic' 13 | 14 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 15 | --task_name long_term_forecast \ 16 | --is_training 1 \ 17 | --root_path ./dataset/traffic/ \ 18 | --data_path traffic.csv \ 19 | --model_id traffic_512_96 \ 20 | --model $model_name \ 21 | --data Traffic \ 22 | --features M \ 23 | --seq_len 512 \ 24 | --label_len 48 \ 25 | --pred_len 96 \ 26 | --e_layers 2 \ 27 | --d_layers 1 \ 28 | --factor 3 \ 29 | --enc_in 862 \ 30 | --dec_in 862 \ 31 | --c_out 862 \ 32 | --batch_size $batch_size \ 33 | --learning_rate $learning_rate \ 34 | --llm_layers $llama_layers \ 35 | --train_epochs $train_epochs \ 36 | --model_comment $comment 37 | 38 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 39 | --task_name long_term_forecast \ 40 | --is_training 1 \ 41 | --root_path ./dataset/traffic/ \ 42 | --data_path traffic.csv \ 43 | --model_id traffic_512_96 \ 44 | --model $model_name \ 45 | --data Traffic \ 46 | --features M \ 47 | --seq_len 512 \ 48 | --label_len 48 \ 49 | --pred_len 192 \ 50 | --e_layers 2 \ 51 | --d_layers 1 \ 52 | --factor 3 \ 53 | --enc_in 862 \ 54 | --dec_in 862 \ 55 | --c_out 862 \ 56 | --batch_size $batch_size \ 57 | --learning_rate $learning_rate \ 58 | --llm_layers $llama_layers \ 59 | --train_epochs $train_epochs \ 60 | --model_comment $comment 61 | 62 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 63 | --task_name long_term_forecast \ 64 | --is_training 1 \ 65 | --root_path ./dataset/traffic/ \ 66 | --data_path traffic.csv \ 67 | --model_id traffic_512_96 \ 68 | --model $model_name \ 69 | --data Traffic \ 70 | --features M \ 71 | --seq_len 512 \ 72 | --label_len 48 \ 73 | --pred_len 336 \ 74 | --e_layers 2 \ 75 | --d_layers 1 \ 76 | --factor 3 \ 77 | --enc_in 862 \ 78 | --dec_in 862 \ 79 | --c_out 862 \ 80 | --batch_size 1 \ 81 | --learning_rate $learning_rate \ 82 | --llm_layers $llama_layers \ 83 | --train_epochs $train_epochs \ 84 | --model_comment $comment 85 | 86 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 87 | --task_name long_term_forecast \ 88 | --is_training 1 \ 89 | --root_path ./dataset/traffic/ \ 90 | --data_path traffic.csv \ 91 | --model_id traffic_512_96 \ 92 | --model $model_name \ 93 | --data Traffic \ 94 | --features M \ 95 | --seq_len 512 \ 96 | --label_len 720 \ 97 | --pred_len 96 \ 98 | --e_layers 2 \ 99 | --d_layers 1 \ 100 | --factor 3 \ 101 | --enc_in 862 \ 102 | --dec_in 862 \ 103 | --c_out 862 \ 104 | --batch_size $batch_size \ 105 | --learning_rate $learning_rate \ 106 | --llm_layers $llama_layers \ 107 | --train_epochs $train_epochs \ 108 | --model_comment $comment -------------------------------------------------------------------------------- /Time-LLM-exp/scripts/train_ori/TimeLLM_Weather.sh: -------------------------------------------------------------------------------- 1 | model_name=TimeLLM 2 | train_epochs=10 3 | learning_rate=0.01 4 | llama_layers=32 5 | 6 | master_port=00097 7 | num_process=8 8 | batch_size=24 9 | d_model=16 10 | d_ff=32 11 | 12 | comment='TimeLLM-Weather' 13 | 14 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 15 | --task_name long_term_forecast \ 16 | --is_training 1 \ 17 | --root_path ./dataset/weather/ \ 18 | --data_path weather.csv \ 19 | --model_id weather_512_96 \ 20 | --model $model_name \ 21 | --data Weather \ 22 | --features M \ 23 | --seq_len 512 \ 24 | --label_len 48 \ 25 | --pred_len 96 \ 26 | --e_layers 2 \ 27 | --d_layers 1 \ 28 | --factor 3 \ 29 | --enc_in 21 \ 30 | --dec_in 21 \ 31 | --c_out 21 \ 32 | --d_model 32 \ 33 | --d_ff 32 \ 34 | --batch_size $batch_size \ 35 | --learning_rate $learning_rate \ 36 | --llm_layers $llama_layers \ 37 | --train_epochs $train_epochs \ 38 | --model_comment $comment 39 | 40 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 41 | --task_name long_term_forecast \ 42 | --is_training 1 \ 43 | --root_path ./dataset/weather/ \ 44 | --data_path weather.csv \ 45 | --model_id weather_512_192 \ 46 | --model $model_name \ 47 | --data Weather \ 48 | --features M \ 49 | --seq_len 512 \ 50 | --label_len 48 \ 51 | --pred_len 192 \ 52 | --e_layers 2 \ 53 | --d_layers 1 \ 54 | --factor 3 \ 55 | --enc_in 21 \ 56 | --dec_in 21 \ 57 | --c_out 21 \ 58 | --d_model 32 \ 59 | --d_ff 32 \ 60 | --batch_size $batch_size \ 61 | --learning_rate $learning_rate \ 62 | --llm_layers $llama_layers \ 63 | --train_epochs $train_epochs \ 64 | --model_comment $comment 65 | 66 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 67 | --task_name long_term_forecast \ 68 | --is_training 1 \ 69 | --root_path ./dataset/weather/ \ 70 | --data_path weather.csv \ 71 | --model_id weather_512_336 \ 72 | --model $model_name \ 73 | --data Weather \ 74 | --features M \ 75 | --seq_len 512 \ 76 | --label_len 48 \ 77 | --pred_len 336 \ 78 | --e_layers 2 \ 79 | --d_layers 1 \ 80 | --factor 3 \ 81 | --enc_in 21 \ 82 | --dec_in 21 \ 83 | --c_out 21 \ 84 | --d_model 32 \ 85 | --d_ff 128 \ 86 | --batch_size $batch_size \ 87 | --learning_rate $learning_rate \ 88 | --llm_layers $llama_layers \ 89 | --train_epochs 10 \ 90 | --model_comment $comment 91 | 92 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 93 | --task_name long_term_forecast \ 94 | --is_training 1 \ 95 | --root_path ./dataset/weather/ \ 96 | --data_path weather.csv \ 97 | --model_id weather_512_720 \ 98 | --model $model_name \ 99 | --data Weather \ 100 | --features M \ 101 | --seq_len 512 \ 102 | --label_len 48 \ 103 | --pred_len 720 \ 104 | --e_layers 2 \ 105 | --d_layers 1 \ 106 | --factor 3 \ 107 | --enc_in 21 \ 108 | --dec_in 21 \ 109 | --c_out 21 \ 110 | --d_model 32 \ 111 | --d_ff 128 \ 112 | --batch_size $batch_size \ 113 | --learning_rate $learning_rate \ 114 | --llm_layers $llama_layers \ 115 | --train_epochs 15 \ 116 | --model_comment $comment -------------------------------------------------------------------------------- /Time-LLM-exp/scripts/train_script/TimeLLM_ECL.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | 3 | model_name=TimeLLM 4 | train_epochs=10 5 | learning_rate=0.05 6 | llama_layers=32 7 | 8 | master_port=8807 9 | num_process=3 10 | batch_size=1024 11 | d_model=16 12 | d_ff=32 13 | 14 | comment='TimeLLM-ECL' 15 | itts='0' 16 | train_epochs=25 17 | methods_h="removeLLM llm_to_trsf llm_to_attn" 18 | for method in $methods_h; 19 | do 20 | for itt in $itts; 21 | do 22 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 23 | --task_name long_term_forecast \ 24 | --is_training 1 \ 25 | --root_path ./dataset/electricity/ \ 26 | --data_path electricity.csv \ 27 | --model_id ECL_512_96_$method \ 28 | --model $model_name \ 29 | --data ECL \ 30 | --features M \ 31 | --seq_len 512 \ 32 | --label_len 48 \ 33 | --pred_len 96 \ 34 | --e_layers 2 \ 35 | --d_layers 1 \ 36 | --factor 3 \ 37 | --enc_in 321 \ 38 | --dec_in 321 \ 39 | --c_out 321 \ 40 | --itr $itt \ 41 | --batch_size $batch_size \ 42 | --learning_rate $learning_rate \ 43 | --llm_layers $llama_layers \ 44 | --train_epochs $train_epochs \ 45 | --model_comment $comment 46 | 47 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 48 | --task_name long_term_forecast \ 49 | --is_training 1 \ 50 | --root_path ./dataset/electricity/ \ 51 | --data_path electricity.csv \ 52 | --model_id ECL_512_192_$method \ 53 | --model $model_name \ 54 | --data ECL \ 55 | --features M \ 56 | --seq_len 512 \ 57 | --label_len 48 \ 58 | --pred_len 192 \ 59 | --e_layers 2 \ 60 | --d_layers 1 \ 61 | --factor 3 \ 62 | --enc_in 321 \ 63 | --dec_in 321 \ 64 | --c_out 321 \ 65 | --itr $itt \ 66 | --batch_size $batch_size \ 67 | --learning_rate $learning_rate \ 68 | --llm_layers $llama_layers \ 69 | --train_epochs $train_epochs \ 70 | --model_comment $comment 71 | 72 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 73 | --task_name long_term_forecast \ 74 | --is_training 1 \ 75 | --root_path ./dataset/electricity/ \ 76 | --data_path electricity.csv \ 77 | --model_id ECL_512_336_$method \ 78 | --model $model_name \ 79 | --data ECL \ 80 | --features M \ 81 | --seq_len 512 \ 82 | --label_len 48 \ 83 | --pred_len 336 \ 84 | --e_layers 2 \ 85 | --d_layers 1 \ 86 | --factor 3 \ 87 | --enc_in 321 \ 88 | --dec_in 321 \ 89 | --c_out 321 \ 90 | --itr $itt \ 91 | --batch_size $batch_size \ 92 | --learning_rate $learning_rate \ 93 | --llm_layers $llama_layers \ 94 | --train_epochs $train_epochs \ 95 | --model_comment $comment 96 | 97 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 98 | --task_name long_term_forecast \ 99 | --is_training 1 \ 100 | --root_path ./dataset/electricity/ \ 101 | --data_path electricity.csv \ 102 | --model_id ECL_512_720_$method \ 103 | --model $model_name \ 104 | --data ECL \ 105 | --features M \ 106 | --seq_len 512 \ 107 | --label_len 48 \ 108 | --pred_len 720 \ 109 | --e_layers 2 \ 110 | --d_layers 1 \ 111 | --factor 3 \ 112 | --enc_in 321 \ 113 | --dec_in 321 \ 114 | --c_out 321 \ 115 | --itr $itt \ 116 | --batch_size $batch_size \ 117 | --learning_rate $learning_rate \ 118 | --llm_layers $llama_layers \ 119 | --train_epochs $train_epochs \ 120 | --model_comment $comment 121 | 122 | done 123 | done -------------------------------------------------------------------------------- /Time-LLM-exp/scripts/train_script/TimeLLM_ETTh1.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | model_name=TimeLLM 3 | learning_rate=0.05 4 | llama_layers=32 5 | master_port=8097 6 | num_process=3 7 | batch_size=256 8 | d_model=32 9 | d_ff=128 10 | count_hyper=0 11 | 12 | comment='TimeLLM-ETTh1' 13 | itts='0' 14 | train_epochs=25 15 | methods_h="removeLLM llm_to_attn llm_to_trsf" 16 | for method in $methods_h; 17 | do 18 | for itt in $itts; 19 | do 20 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 21 | --task_name long_term_forecast \ 22 | --is_training 1 \ 23 | --root_path ./dataset/ETT-small/ \ 24 | --data_path ETTh1.csv \ 25 | --model_id ETTh1_512_96_$method \ 26 | --model $model_name \ 27 | --data ETTh1 \ 28 | --features M \ 29 | --seq_len 512 \ 30 | --label_len 48 \ 31 | --pred_len 96 \ 32 | --factor 3 \ 33 | --enc_in 7 \ 34 | --dec_in 7 \ 35 | --c_out 7 \ 36 | --des 'Exp' \ 37 | --itr $itt \ 38 | --d_model $d_model \ 39 | --d_ff $d_ff \ 40 | --batch_size $batch_size \ 41 | --learning_rate 0.05 \ 42 | --llm_layers $llama_layers \ 43 | --count_hyper $count_hyper \ 44 | --train_epochs $train_epochs \ 45 | --model_comment $comment 46 | 47 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 48 | --task_name long_term_forecast \ 49 | --is_training 1 \ 50 | --root_path ./dataset/ETT-small/ \ 51 | --data_path ETTh1.csv \ 52 | --model_id ETTh1_512_192_$method \ 53 | --model $model_name \ 54 | --data ETTh1 \ 55 | --features M \ 56 | --seq_len 512 \ 57 | --label_len 48 \ 58 | --pred_len 192 \ 59 | --factor 3 \ 60 | --enc_in 7 \ 61 | --dec_in 7 \ 62 | --c_out 7 \ 63 | --des 'Exp' \ 64 | --itr $itt \ 65 | --d_model 32 \ 66 | --d_ff 128 \ 67 | --batch_size $batch_size \ 68 | --learning_rate 0.02 \ 69 | --llm_layers $llama_layers \ 70 | --train_epochs $train_epochs \ 71 | --model_comment $comment 72 | 73 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 74 | --task_name long_term_forecast \ 75 | --is_training 1 \ 76 | --root_path ./dataset/ETT-small/ \ 77 | --data_path ETTh1.csv \ 78 | --model_id ETTh1_512_336_$method \ 79 | --model $model_name \ 80 | --data ETTh1 \ 81 | --features M \ 82 | --seq_len 512 \ 83 | --label_len 48 \ 84 | --pred_len 336 \ 85 | --factor 3 \ 86 | --enc_in 7 \ 87 | --dec_in 7 \ 88 | --c_out 7 \ 89 | --des 'Exp' \ 90 | --itr $itt \ 91 | --d_model $d_model \ 92 | --d_ff $d_ff \ 93 | --batch_size $batch_size \ 94 | --lradj 'COS'\ 95 | --learning_rate 0.001 \ 96 | --llm_layers $llama_layers \ 97 | --train_epochs $train_epochs \ 98 | --model_comment $comment 99 | 100 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 101 | --task_name long_term_forecast \ 102 | --is_training 1 \ 103 | --root_path ./dataset/ETT-small/ \ 104 | --data_path ETTh1.csv \ 105 | --model_id ETTh1_512_720_$method \ 106 | --model $model_name \ 107 | --data ETTh1 \ 108 | --features M \ 109 | --seq_len 512 \ 110 | --label_len 48 \ 111 | --pred_len 720 \ 112 | --factor 3 \ 113 | --enc_in 7 \ 114 | --dec_in 7 \ 115 | --c_out 7 \ 116 | --des 'Exp' \ 117 | --itr $itt \ 118 | --d_model $d_model \ 119 | --d_ff $d_ff \ 120 | --batch_size $batch_size \ 121 | --learning_rate 0.01 \ 122 | --llm_layers $llama_layers \ 123 | --train_epochs $train_epochs \ 124 | --model_comment $comment 125 | done 126 | done -------------------------------------------------------------------------------- /Time-LLM-exp/scripts/train_script/TimeLLM_ETTh2.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | 3 | model_name=TimeLLM 4 | learning_rate=0.05 5 | llama_layers=32 6 | d_model=32 7 | d_ff=128 8 | 9 | comment='TimeLLM-ETTh2' 10 | 11 | master_port=8091 12 | num_process=3 13 | batch_size=256 14 | train_epochs=25 15 | itts='0' 16 | methods_h="removeLLM llm_to_trsf llm_to_attn" 17 | for method in $methods_h; 18 | do 19 | for itt in $itts; 20 | do 21 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 22 | --task_name long_term_forecast \ 23 | --is_training 1 \ 24 | --root_path ./dataset/ETT-small/ \ 25 | --data_path ETTh2.csv \ 26 | --model_id ETTh2_512_96_$method \ 27 | --model $model_name \ 28 | --data ETTh2 \ 29 | --features M \ 30 | --seq_len 512 \ 31 | --label_len 48 \ 32 | --pred_len 96 \ 33 | --factor 3 \ 34 | --enc_in 7 \ 35 | --dec_in 7 \ 36 | --c_out 7 \ 37 | --des 'Exp' \ 38 | --itr $itt \ 39 | --d_model $d_model \ 40 | --d_ff $d_ff \ 41 | --batch_size $batch_size \ 42 | --learning_rate $learning_rate \ 43 | --llm_layers $llama_layers \ 44 | --train_epochs $train_epochs \ 45 | --model_comment $comment 46 | 47 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 48 | --task_name long_term_forecast \ 49 | --is_training 1 \ 50 | --root_path ./dataset/ETT-small/ \ 51 | --data_path ETTh2.csv \ 52 | --model_id ETTh2_512_192_$method \ 53 | --model $model_name \ 54 | --data ETTh2 \ 55 | --features M \ 56 | --seq_len 512 \ 57 | --label_len 48 \ 58 | --pred_len 192 \ 59 | --factor 3 \ 60 | --enc_in 7 \ 61 | --dec_in 7 \ 62 | --c_out 7 \ 63 | --des 'Exp' \ 64 | --itr $itt \ 65 | --d_model $d_model \ 66 | --d_ff $d_ff \ 67 | --batch_size $batch_size \ 68 | --lradj 'TST'\ 69 | --learning_rate 0.002 \ 70 | --llm_layers $llama_layers \ 71 | --train_epochs $train_epochs \ 72 | --model_comment $comment 73 | 74 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 75 | --task_name long_term_forecast \ 76 | --is_training 1 \ 77 | --root_path ./dataset/ETT-small/ \ 78 | --data_path ETTh2.csv \ 79 | --model_id ETTh2_512_336_$method \ 80 | --model $model_name \ 81 | --data ETTh2 \ 82 | --features M \ 83 | --seq_len 512 \ 84 | --label_len 48 \ 85 | --pred_len 336 \ 86 | --factor 3 \ 87 | --enc_in 7 \ 88 | --dec_in 7 \ 89 | --c_out 7 \ 90 | --des 'Exp' \ 91 | --itr $itt \ 92 | --d_model $d_model \ 93 | --d_ff $d_ff \ 94 | --batch_size $batch_size \ 95 | --lradj 'TST'\ 96 | --learning_rate 0.005 \ 97 | --llm_layers $llama_layers \ 98 | --train_epochs $train_epochs \ 99 | --model_comment $comment 100 | 101 | 102 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 103 | --task_name long_term_forecast \ 104 | --is_training 1 \ 105 | --root_path ./dataset/ETT-small/ \ 106 | --data_path ETTh2.csv \ 107 | --model_id ETTh2_512_720_$method \ 108 | --model $model_name \ 109 | --data ETTh2 \ 110 | --features M \ 111 | --seq_len 512 \ 112 | --label_len 48 \ 113 | --pred_len 720 \ 114 | --factor 3 \ 115 | --enc_in 7 \ 116 | --dec_in 7 \ 117 | --c_out 7 \ 118 | --des 'Exp' \ 119 | --itr $itt \ 120 | --d_model 16 \ 121 | --d_ff 128 \ 122 | --batch_size $batch_size \ 123 | --learning_rate 0.005 \ 124 | --lradj 'TST'\ 125 | --llm_layers $llama_layers \ 126 | --train_epochs 20 \ 127 | --patience 10 \ 128 | --model_comment $comment 129 | 130 | done 131 | done -------------------------------------------------------------------------------- /Time-LLM-exp/scripts/train_script/TimeLLM_ETTm1.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | model_name=TimeLLM 3 | learning_rate=0.01 4 | llama_layers=32 5 | d_model=32 6 | d_ff=128 7 | 8 | comment='TimeLLM-ETTm1' 9 | 10 | master_port=8099 11 | num_process=3 12 | batch_size=256 13 | train_epochs=25 14 | itts='0' 15 | methods_h="removeLLM llm_to_trsf llm_to_attn" 16 | for method in $methods_h; 17 | do 18 | for itt in $itts; 19 | do 20 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 21 | --task_name long_term_forecast \ 22 | --is_training 1 \ 23 | --root_path ./dataset/ETT-small/ \ 24 | --data_path ETTm1.csv \ 25 | --model_id ETTm1_512_96_$method \ 26 | --model $model_name \ 27 | --data ETTm1 \ 28 | --features M \ 29 | --seq_len 512 \ 30 | --label_len 48 \ 31 | --pred_len 96 \ 32 | --factor 3 \ 33 | --enc_in 7 \ 34 | --dec_in 7 \ 35 | --c_out 7 \ 36 | --des 'Exp' \ 37 | --itr $itt \ 38 | --d_model $d_model \ 39 | --d_ff $d_ff \ 40 | --batch_size $batch_size \ 41 | --lradj 'TST'\ 42 | --learning_rate 0.001 \ 43 | --llm_layers $llama_layers \ 44 | --train_epochs $train_epochs \ 45 | --model_comment $comment 46 | 47 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 48 | --task_name long_term_forecast \ 49 | --is_training 1 \ 50 | --root_path ./dataset/ETT-small/ \ 51 | --data_path ETTm1.csv \ 52 | --model_id ETTm1_512_192_$method \ 53 | --model $model_name \ 54 | --data ETTm1 \ 55 | --features M \ 56 | --seq_len 512 \ 57 | --label_len 48 \ 58 | --pred_len 192 \ 59 | --factor 3 \ 60 | --enc_in 7 \ 61 | --dec_in 7 \ 62 | --c_out 7 \ 63 | --des 'Exp' \ 64 | --itr $itt \ 65 | --d_model $d_model \ 66 | --d_ff $d_ff \ 67 | --batch_size $batch_size \ 68 | --learning_rate $learning_rate \ 69 | --lradj 'TST'\ 70 | --learning_rate 0.001 \ 71 | --llm_layers $llama_layers \ 72 | --train_epochs $train_epochs \ 73 | --patience 20 \ 74 | --model_comment $comment 75 | 76 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 77 | --task_name long_term_forecast \ 78 | --is_training 1 \ 79 | --root_path ./dataset/ETT-small/ \ 80 | --data_path ETTm1.csv \ 81 | --model_id ETTm1_512_336_$method \ 82 | --model $model_name \ 83 | --data ETTm1 \ 84 | --features M \ 85 | --seq_len 512 \ 86 | --label_len 48 \ 87 | --pred_len 336 \ 88 | --factor 3 \ 89 | --enc_in 7 \ 90 | --dec_in 7 \ 91 | --c_out 7 \ 92 | --des 'Exp' \ 93 | --itr $itt \ 94 | --d_model $d_model \ 95 | --d_ff $d_ff \ 96 | --batch_size $batch_size \ 97 | --learning_rate $learning_rate \ 98 | --lradj 'TST'\ 99 | --learning_rate 0.001 \ 100 | --llm_layers $llama_layers \ 101 | --train_epochs $train_epochs \ 102 | --patience 20 \ 103 | --model_comment $comment 104 | 105 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 106 | --task_name long_term_forecast \ 107 | --is_training 1 \ 108 | --root_path ./dataset/ETT-small/ \ 109 | --data_path ETTm1.csv \ 110 | --model_id ETTm1_512_720_$method \ 111 | --model $model_name \ 112 | --data ETTm1 \ 113 | --features M \ 114 | --seq_len 512 \ 115 | --label_len 48 \ 116 | --pred_len 720 \ 117 | --factor 3 \ 118 | --enc_in 7 \ 119 | --dec_in 7 \ 120 | --c_out 7 \ 121 | --des 'Exp' \ 122 | --itr $itt \ 123 | --d_model $d_model \ 124 | --d_ff $d_ff \ 125 | --batch_size $batch_size \ 126 | --learning_rate $learning_rate \ 127 | --lradj 'TST'\ 128 | --learning_rate 0.001 \ 129 | --llm_layers $llama_layers \ 130 | --train_epochs $train_epochs \ 131 | --patience 20 \ 132 | --model_comment $comment 133 | 134 | done 135 | done -------------------------------------------------------------------------------- /Time-LLM-exp/scripts/train_script/TimeLLM_ETTm2.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | 3 | model_name=TimeLLM 4 | learning_rate=0.01 5 | llama_layers=32 6 | 7 | d_model=32 8 | d_ff=128 9 | 10 | comment='TimeLLM-ETTm2' 11 | 12 | master_port=8099 13 | num_process=3 14 | batch_size=256 15 | train_epochs=25 16 | itts='0' 17 | methods_h="removeLLM llm_to_trsf llm_to_attn" 18 | 19 | for method in $methods_h; 20 | do 21 | for itt in $itts; 22 | do 23 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 24 | --task_name long_term_forecast \ 25 | --is_training 1 \ 26 | --root_path ./dataset/ETT-small/ \ 27 | --data_path ETTm2.csv \ 28 | --model_id ETTm2_512_96_$method \ 29 | --model $model_name \ 30 | --data ETTm2 \ 31 | --features M \ 32 | --seq_len 512 \ 33 | --label_len 48 \ 34 | --pred_len 96 \ 35 | --factor 3 \ 36 | --enc_in 7 \ 37 | --dec_in 7 \ 38 | --c_out 7 \ 39 | --des 'Exp' \ 40 | --itr $itt \ 41 | --d_model $d_model \ 42 | --d_ff $d_ff \ 43 | --batch_size 16 \ 44 | --learning_rate $learning_rate \ 45 | --llm_layers $llama_layers \ 46 | --train_epochs $train_epochs \ 47 | --model_comment $comment 48 | 49 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 50 | --task_name long_term_forecast \ 51 | --is_training 1 \ 52 | --root_path ./dataset/ETT-small/ \ 53 | --data_path ETTm2.csv \ 54 | --model_id ETTm2_512_192_$method \ 55 | --model $model_name \ 56 | --data ETTm2 \ 57 | --features M \ 58 | --seq_len 512 \ 59 | --label_len 48 \ 60 | --pred_len 192 \ 61 | --factor 3 \ 62 | --enc_in 7 \ 63 | --dec_in 7 \ 64 | --c_out 7 \ 65 | --des 'Exp' \ 66 | --itr $itt \ 67 | --d_model $d_model \ 68 | --d_ff $d_ff \ 69 | --batch_size $batch_size \ 70 | --learning_rate $learning_rate \ 71 | --lradj 'TST'\ 72 | --learning_rate 0.002 \ 73 | --llm_layers $llama_layers \ 74 | --train_epochs $train_epochs \ 75 | --model_comment $comment 76 | 77 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 78 | --task_name long_term_forecast \ 79 | --is_training 1 \ 80 | --root_path ./dataset/ETT-small/ \ 81 | --data_path ETTm2.csv \ 82 | --model_id ETTm2_512_336_$method \ 83 | --model $model_name \ 84 | --data ETTm2 \ 85 | --features M \ 86 | --seq_len 512 \ 87 | --label_len 48 \ 88 | --pred_len 336 \ 89 | --factor 3 \ 90 | --enc_in 7 \ 91 | --dec_in 7 \ 92 | --c_out 7 \ 93 | --des 'Exp' \ 94 | --itr $itt \ 95 | --d_model $d_model \ 96 | --d_ff $d_ff \ 97 | --batch_size $batch_size \ 98 | --learning_rate $learning_rate \ 99 | --lradj 'TST'\ 100 | --learning_rate 0.002 \ 101 | --llm_layers $llama_layers \ 102 | --train_epochs $train_epochs \ 103 | --model_comment $comment 104 | 105 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 106 | --task_name long_term_forecast \ 107 | --is_training 1 \ 108 | --root_path ./dataset/ETT-small/ \ 109 | --data_path ETTm2.csv \ 110 | --model_id ETTm2_512_720_$method \ 111 | --model $model_name \ 112 | --data ETTm2 \ 113 | --features M \ 114 | --seq_len 512 \ 115 | --label_len 48 \ 116 | --pred_len 720 \ 117 | --factor 3 \ 118 | --enc_in 7 \ 119 | --dec_in 7 \ 120 | --c_out 7 \ 121 | --des 'Exp' \ 122 | --itr $itt \ 123 | --d_model $d_model \ 124 | --d_ff $d_ff \ 125 | --batch_size $batch_size \ 126 | --learning_rate $learning_rate \ 127 | --lradj 'TST'\ 128 | --learning_rate 0.002 \ 129 | --llm_layers $llama_layers \ 130 | --train_epochs $train_epochs \ 131 | --model_comment $comment 132 | 133 | done 134 | done -------------------------------------------------------------------------------- /Time-LLM-exp/scripts/train_script/TimeLLM_Traffic.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | model_name=TimeLLM 3 | train_epochs=10 4 | learning_rate=0.05 5 | llama_layers=32 6 | 7 | master_port=8872 8 | num_process=3 9 | batch_size=4096 10 | d_model=16 11 | d_ff=32 12 | 13 | comment='TimeLLM-Traffic' 14 | 15 | itts='0' 16 | train_epochs=25 17 | methods_h="removeLLM llm_to_trsf llm_to_attn" 18 | 19 | for method in $methods_h; 20 | do 21 | for itt in $itts; 22 | do 23 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 24 | --task_name long_term_forecast \ 25 | --is_training 1 \ 26 | --root_path ./dataset/traffic/ \ 27 | --data_path traffic.csv \ 28 | --model_id traffic_512_96_$method \ 29 | --model $model_name \ 30 | --data Traffic \ 31 | --features M \ 32 | --seq_len 512 \ 33 | --label_len 48 \ 34 | --pred_len 96 \ 35 | --e_layers 2 \ 36 | --d_layers 1 \ 37 | --factor 3 \ 38 | --enc_in 862 \ 39 | --dec_in 862 \ 40 | --c_out 862 \ 41 | --itr $itt \ 42 | --batch_size $batch_size \ 43 | --learning_rate $learning_rate \ 44 | --llm_layers $llama_layers \ 45 | --train_epochs $train_epochs \ 46 | --model_comment $comment 47 | 48 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 49 | --task_name long_term_forecast \ 50 | --is_training 1 \ 51 | --root_path ./dataset/traffic/ \ 52 | --data_path traffic.csv \ 53 | --model_id traffic_512_192_$method \ 54 | --model $model_name \ 55 | --data Traffic \ 56 | --features M \ 57 | --seq_len 512 \ 58 | --label_len 48 \ 59 | --pred_len 192 \ 60 | --e_layers 2 \ 61 | --d_layers 1 \ 62 | --factor 3 \ 63 | --enc_in 862 \ 64 | --dec_in 862 \ 65 | --c_out 862 \ 66 | --itr $itt \ 67 | --batch_size $batch_size \ 68 | --learning_rate $learning_rate \ 69 | --llm_layers $llama_layers \ 70 | --train_epochs $train_epochs \ 71 | --model_comment $comment 72 | 73 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 74 | --task_name long_term_forecast \ 75 | --is_training 1 \ 76 | --root_path ./dataset/traffic/ \ 77 | --data_path traffic.csv \ 78 | --model_id traffic_512_336_$method \ 79 | --model $model_name \ 80 | --data Traffic \ 81 | --features M \ 82 | --seq_len 512 \ 83 | --label_len 48 \ 84 | --pred_len 336 \ 85 | --e_layers 2 \ 86 | --d_layers 1 \ 87 | --factor 3 \ 88 | --enc_in 862 \ 89 | --dec_in 862 \ 90 | --c_out 862 \ 91 | --batch_size $batch_size \ 92 | --itr $itt \ 93 | --learning_rate $learning_rate \ 94 | --llm_layers $llama_layers \ 95 | --train_epochs $train_epochs \ 96 | --model_comment $comment 97 | 98 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 99 | --task_name long_term_forecast \ 100 | --is_training 1 \ 101 | --root_path ./dataset/traffic/ \ 102 | --data_path traffic.csv \ 103 | --model_id traffic_512_720_$method \ 104 | --model $model_name \ 105 | --data Traffic \ 106 | --features M \ 107 | --seq_len 512 \ 108 | --label_len 0 \ 109 | --pred_len 720 \ 110 | --e_layers 2 \ 111 | --d_layers 1 \ 112 | --factor 3 \ 113 | --enc_in 862 \ 114 | --dec_in 862 \ 115 | --c_out 862 \ 116 | --itr $itt \ 117 | --batch_size $batch_size \ 118 | --learning_rate $learning_rate \ 119 | --llm_layers $llama_layers \ 120 | --train_epochs $train_epochs \ 121 | --model_comment $comment 122 | 123 | done 124 | done -------------------------------------------------------------------------------- /Time-LLM-exp/scripts/train_script/TimeLLM_Weather.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | 3 | model_name=TimeLLM 4 | train_epochs=10 5 | learning_rate=0.05 6 | llama_layers=32 7 | 8 | master_port=8867 9 | num_process=3 10 | batch_size=512 11 | d_model=16 12 | d_ff=32 13 | itts='0' 14 | comment='TimeLLM-Weather' 15 | methods_h="removeLLM llm_to_trsf llm_to_attn" 16 | for method in $methods_h; 17 | do 18 | for itt in $itts; 19 | do 20 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 21 | --task_name long_term_forecast \ 22 | --is_training 1 \ 23 | --root_path ./dataset/weather/ \ 24 | --data_path weather.csv \ 25 | --model_id weather_512_96_$method \ 26 | --model $model_name \ 27 | --data Weather \ 28 | --features M \ 29 | --seq_len 512 \ 30 | --label_len 48 \ 31 | --pred_len 96 \ 32 | --e_layers 2 \ 33 | --d_layers 1 \ 34 | --factor 3 \ 35 | --enc_in 21 \ 36 | --dec_in 21 \ 37 | --c_out 21 \ 38 | --d_model 32 \ 39 | --d_ff 32 \ 40 | --itr $itt \ 41 | --batch_size $batch_size \ 42 | --learning_rate $learning_rate \ 43 | --llm_layers $llama_layers \ 44 | --train_epochs $train_epochs \ 45 | --model_comment $comment 46 | 47 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 48 | --task_name long_term_forecast \ 49 | --is_training 1 \ 50 | --root_path ./dataset/weather/ \ 51 | --data_path weather.csv \ 52 | --model_id weather_512_192_$method \ 53 | --model $model_name \ 54 | --data Weather \ 55 | --features M \ 56 | --seq_len 512 \ 57 | --label_len 48 \ 58 | --pred_len 192 \ 59 | --e_layers 2 \ 60 | --d_layers 1 \ 61 | --factor 3 \ 62 | --enc_in 21 \ 63 | --dec_in 21 \ 64 | --c_out 21 \ 65 | --d_model 32 \ 66 | --d_ff 32 \ 67 | --itr $itt \ 68 | --batch_size $batch_size \ 69 | --learning_rate $learning_rate \ 70 | --llm_layers $llama_layers \ 71 | --train_epochs $train_epochs \ 72 | --model_comment $comment 73 | 74 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 75 | --task_name long_term_forecast \ 76 | --is_training 1 \ 77 | --root_path ./dataset/weather/ \ 78 | --data_path weather.csv \ 79 | --model_id weather_512_336_$method \ 80 | --model $model_name \ 81 | --data Weather \ 82 | --features M \ 83 | --seq_len 512 \ 84 | --label_len 48 \ 85 | --pred_len 336 \ 86 | --e_layers 2 \ 87 | --d_layers 1 \ 88 | --factor 3 \ 89 | --enc_in 21 \ 90 | --dec_in 21 \ 91 | --c_out 21 \ 92 | --d_model 32 \ 93 | --d_ff 128 \ 94 | --itr $itt \ 95 | --batch_size $batch_size \ 96 | --learning_rate $learning_rate \ 97 | --llm_layers $llama_layers \ 98 | --train_epochs 10 \ 99 | --model_comment $comment 100 | 101 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 102 | --task_name long_term_forecast \ 103 | --is_training 1 \ 104 | --root_path ./dataset/weather/ \ 105 | --data_path weather.csv \ 106 | --model_id weather_512_720_$method \ 107 | --model $model_name \ 108 | --data Weather \ 109 | --features M \ 110 | --seq_len 512 \ 111 | --label_len 48 \ 112 | --pred_len 720 \ 113 | --e_layers 2 \ 114 | --d_layers 1 \ 115 | --factor 3 \ 116 | --enc_in 21 \ 117 | --dec_in 21 \ 118 | --c_out 21 \ 119 | --d_model 32 \ 120 | --d_ff 128 \ 121 | --itr $itt \ 122 | --batch_size $batch_size \ 123 | --learning_rate $learning_rate \ 124 | --llm_layers $llama_layers \ 125 | --train_epochs 15 \ 126 | --model_comment $comment 127 | 128 | done 129 | done -------------------------------------------------------------------------------- /Time-LLM-exp/scripts/train_script/illness.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES="0,1,2" 2 | model_name=TimeLLM 3 | train_epochs=10 4 | learning_rate=0.1 5 | llama_layers=32 6 | 7 | master_port=8871 8 | num_process=3 9 | batch_size=16 10 | d_model=16 11 | d_ff=32 12 | 13 | comment='TimeLLM-illness' 14 | 15 | itts='0' 16 | train_epochs=25 17 | methods_h="removeLLM llm_to_trsf llm_to_attn" 18 | for method in $methods_h; 19 | do 20 | for itt in $itts; 21 | do 22 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 23 | --task_name long_term_forecast \ 24 | --is_training 1 \ 25 | --root_path ./dataset/illness/ \ 26 | --data_path national_illness.csv \ 27 | --model_id Illness_104_24_$method \ 28 | --model $model_name \ 29 | --data Illness \ 30 | --features M \ 31 | --seq_len 104 \ 32 | --label_len 0 \ 33 | --pred_len 24 \ 34 | --e_layers 2 \ 35 | --d_layers 1 \ 36 | --factor 3 \ 37 | --enc_in 7 \ 38 | --dec_in 7 \ 39 | --c_out 7 \ 40 | --itr $itt \ 41 | --batch_size $batch_size \ 42 | --learning_rate $learning_rate \ 43 | --llm_layers $llama_layers \ 44 | --train_epochs $train_epochs \ 45 | --model_comment $comment 46 | 47 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 48 | --task_name long_term_forecast \ 49 | --is_training 1 \ 50 | --root_path ./dataset/illness/ \ 51 | --data_path national_illness.csv \ 52 | --model_id Illness_104_36_$method \ 53 | --model $model_name \ 54 | --data Illness \ 55 | --features M \ 56 | --seq_len 104 \ 57 | --label_len 0 \ 58 | --pred_len 36 \ 59 | --e_layers 2 \ 60 | --d_layers 1 \ 61 | --factor 3 \ 62 | --enc_in 7 \ 63 | --dec_in 7 \ 64 | --c_out 7 \ 65 | --itr $itt \ 66 | --batch_size $batch_size \ 67 | --learning_rate $learning_rate \ 68 | --llm_layers $llama_layers \ 69 | --train_epochs $train_epochs \ 70 | --model_comment $comment 71 | 72 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 73 | --task_name long_term_forecast \ 74 | --is_training 1 \ 75 | --root_path ./dataset/illness/ \ 76 | --data_path national_illness.csv \ 77 | --model_id Illness_104_48_$method \ 78 | --model $model_name \ 79 | --data Illness \ 80 | --features M \ 81 | --seq_len 104 \ 82 | --label_len 0 \ 83 | --pred_len 48 \ 84 | --e_layers 2 \ 85 | --d_layers 1 \ 86 | --factor 3 \ 87 | --enc_in 7 \ 88 | --dec_in 7 \ 89 | --c_out 7 \ 90 | --batch_size 1 \ 91 | --itr $itt \ 92 | --learning_rate $learning_rate \ 93 | --llm_layers $llama_layers \ 94 | --train_epochs $train_epochs \ 95 | --model_comment $comment 96 | 97 | accelerate launch --multi_gpu --mixed_precision bf16 --num_processes $num_process --main_process_port $master_port run_main.py \ 98 | --task_name long_term_forecast \ 99 | --is_training 1 \ 100 | --root_path ./dataset/illness/ \ 101 | --data_path national_illness.csv \ 102 | --model_id Illness_104_60_$method \ 103 | --model $model_name \ 104 | --data Illness \ 105 | --features M \ 106 | --seq_len 104 \ 107 | --label_len 0 \ 108 | --pred_len 60 \ 109 | --e_layers 2 \ 110 | --d_layers 1 \ 111 | --factor 3 \ 112 | --enc_in 7 \ 113 | --dec_in 7 \ 114 | --c_out 7 \ 115 | --itr $itt \ 116 | --batch_size $batch_size \ 117 | --learning_rate $learning_rate \ 118 | --llm_layers $llama_layers \ 119 | --train_epochs $train_epochs \ 120 | --model_comment $comment 121 | done 122 | done -------------------------------------------------------------------------------- /Time-LLM-exp/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/utils/__init__.py -------------------------------------------------------------------------------- /Time-LLM-exp/utils/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/utils/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/utils/__pycache__/timefeatures.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/utils/__pycache__/timefeatures.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/utils/__pycache__/tools.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/Time-LLM-exp/utils/__pycache__/tools.cpython-39.pyc -------------------------------------------------------------------------------- /Time-LLM-exp/utils/losses.py: -------------------------------------------------------------------------------- 1 | # This source code is provided for the purposes of scientific reproducibility 2 | # under the following limited license from Element AI Inc. The code is an 3 | # implementation of the N-BEATS model (Oreshkin et al., N-BEATS: Neural basis 4 | # expansion analysis for interpretable time series forecasting, 5 | # https://arxiv.org/abs/1905.10437). The copyright to the source code is 6 | # licensed under the Creative Commons - Attribution-NonCommercial 4.0 7 | # International license (CC BY-NC 4.0): 8 | # https://creativecommons.org/licenses/by-nc/4.0/. Any commercial use (whether 9 | # for the benefit of third parties or internally in production) requires an 10 | # explicit license. The subject-matter of the N-BEATS model and associated 11 | # materials are the property of Element AI Inc. and may be subject to patent 12 | # protection. No license to patents is granted hereunder (whether express or 13 | # implied). Copyright © 2020 Element AI Inc. All rights reserved. 14 | 15 | """ 16 | Loss functions for PyTorch. 17 | """ 18 | 19 | import torch as t 20 | import torch.nn as nn 21 | import numpy as np 22 | import pdb 23 | 24 | 25 | def divide_no_nan(a, b): 26 | """ 27 | a/b where the resulted NaN or Inf are replaced by 0. 28 | """ 29 | result = a / b 30 | result[result != result] = .0 31 | result[result == np.inf] = .0 32 | return result 33 | 34 | 35 | class mape_loss(nn.Module): 36 | def __init__(self): 37 | super(mape_loss, self).__init__() 38 | 39 | def forward(self, insample: t.Tensor, freq: int, 40 | forecast: t.Tensor, target: t.Tensor, mask: t.Tensor) -> t.float: 41 | """ 42 | MAPE loss as defined in: https://en.wikipedia.org/wiki/Mean_absolute_percentage_error 43 | 44 | :param forecast: Forecast values. Shape: batch, time 45 | :param target: Target values. Shape: batch, time 46 | :param mask: 0/1 mask. Shape: batch, time 47 | :return: Loss value 48 | """ 49 | weights = divide_no_nan(mask, target) 50 | return t.mean(t.abs((forecast - target) * weights)) 51 | 52 | 53 | class smape_loss(nn.Module): 54 | def __init__(self): 55 | super(smape_loss, self).__init__() 56 | 57 | def forward(self, insample: t.Tensor, freq: int, 58 | forecast: t.Tensor, target: t.Tensor, mask: t.Tensor) -> t.float: 59 | """ 60 | sMAPE loss as defined in https://robjhyndman.com/hyndsight/smape/ (Makridakis 1993) 61 | 62 | :param forecast: Forecast values. Shape: batch, time 63 | :param target: Target values. Shape: batch, time 64 | :param mask: 0/1 mask. Shape: batch, time 65 | :return: Loss value 66 | """ 67 | return 200 * t.mean(divide_no_nan(t.abs(forecast - target), 68 | t.abs(forecast.data) + t.abs(target.data)) * mask) 69 | 70 | 71 | class mase_loss(nn.Module): 72 | def __init__(self): 73 | super(mase_loss, self).__init__() 74 | 75 | def forward(self, insample: t.Tensor, freq: int, 76 | forecast: t.Tensor, target: t.Tensor, mask: t.Tensor) -> t.float: 77 | """ 78 | MASE loss as defined in "Scaled Errors" https://robjhyndman.com/papers/mase.pdf 79 | 80 | :param insample: Insample values. Shape: batch, time_i 81 | :param freq: Frequency value 82 | :param forecast: Forecast values. Shape: batch, time_o 83 | :param target: Target values. Shape: batch, time_o 84 | :param mask: 0/1 mask. Shape: batch, time_o 85 | :return: Loss value 86 | """ 87 | masep = t.mean(t.abs(insample[:, freq:] - insample[:, :-freq]), dim=1) 88 | masked_masep_inv = divide_no_nan(mask, masep[:, None]) 89 | return t.mean(t.abs(target - forecast) * masked_masep_inv) 90 | -------------------------------------------------------------------------------- /Time-LLM-exp/utils/masking.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class TriangularCausalMask(): 5 | def __init__(self, B, L, device="cpu"): 6 | mask_shape = [B, 1, L, L] 7 | with torch.no_grad(): 8 | self._mask = torch.triu(torch.ones(mask_shape, dtype=torch.bool), diagonal=1).to(device) 9 | 10 | @property 11 | def mask(self): 12 | return self._mask 13 | 14 | 15 | class ProbMask(): 16 | def __init__(self, B, H, L, index, scores, device="cpu"): 17 | _mask = torch.ones(L, scores.shape[-1], dtype=torch.bool).to(device).triu(1) 18 | _mask_ex = _mask[None, None, :].expand(B, H, L, scores.shape[-1]) 19 | indicator = _mask_ex[torch.arange(B)[:, None, None], 20 | torch.arange(H)[None, :, None], 21 | index, :].to(device) 22 | self._mask = indicator.view(scores.shape).to(device) 23 | 24 | @property 25 | def mask(self): 26 | return self._mask -------------------------------------------------------------------------------- /Time-LLM-exp/utils/metrics.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def RSE(pred, true): 5 | return np.sqrt(np.sum((true - pred) ** 2)) / np.sqrt(np.sum((true - true.mean()) ** 2)) 6 | 7 | 8 | def CORR(pred, true): 9 | u = ((true - true.mean(0)) * (pred - pred.mean(0))).sum(0) 10 | d = np.sqrt(((true - true.mean(0)) ** 2 * (pred - pred.mean(0)) ** 2).sum(0)) 11 | return (u / d).mean(-1) 12 | 13 | 14 | def MAE(pred, true): 15 | return np.mean(np.abs(pred - true)) 16 | 17 | 18 | def MSE(pred, true): 19 | return np.mean((pred - true) ** 2) 20 | 21 | 22 | def RMSE(pred, true): 23 | return np.sqrt(MSE(pred, true)) 24 | 25 | 26 | def MAPE(pred, true): 27 | return np.mean(np.abs((pred - true) / true)) 28 | 29 | 30 | def MSPE(pred, true): 31 | return np.mean(np.square((pred - true) / true)) 32 | 33 | 34 | def metric(pred, true): 35 | mae = MAE(pred, true) 36 | mse = MSE(pred, true) 37 | rmse = RMSE(pred, true) 38 | mape = MAPE(pred, true) 39 | mspe = MSPE(pred, true) 40 | 41 | return mae, mse, rmse, mape, mspe 42 | -------------------------------------------------------------------------------- /pic/ablations.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BennyTMT/LLMsForTimeSeries/65a8897ce06abc9db4aa60ae7eb677661a58c48a/pic/ablations.png --------------------------------------------------------------------------------