├── softclt_ts2vec ├── models │ ├── __init__.py │ ├── __pycache__ │ │ ├── encoder.cpython-310.pyc │ │ ├── losses.cpython-310.pyc │ │ ├── __init__.cpython-310.pyc │ │ ├── losses_KL.cpython-310.pyc │ │ ├── dilated_conv.cpython-310.pyc │ │ ├── losses_erase.cpython-310.pyc │ │ └── soft_labels.cpython-310.pyc │ ├── timelags.py │ ├── hard_losses.py │ ├── dilated_conv.py │ ├── encoder.py │ └── soft_losses.py ├── tasks │ ├── __pycache__ │ │ ├── __init__.cpython-310.pyc │ │ ├── forecasting.cpython-310.pyc │ │ ├── _eval_protocols.cpython-310.pyc │ │ ├── classification.cpython-310.pyc │ │ └── anomaly_detection.cpython-310.pyc │ ├── __init__.py │ ├── classification.py │ ├── _eval_protocols.py │ ├── forecasting_separate.py │ └── anomaly_detection.py ├── hyperparameters │ ├── ad_hyperparams.csv │ ├── semi_cls_1p_hyperparams.csv │ ├── semi_cls_5p_hyperparams.csv │ └── cls_hyperparams.csv ├── ts2vec.yml ├── scripts │ ├── electricity.sh │ ├── kpi.sh │ ├── yahoo.sh │ ├── ett.sh │ └── uea.sh ├── utils.py ├── utils_distance_matrix.py └── train.py ├── softclt_catcc ├── models │ ├── __pycache__ │ │ ├── TC.cpython-310.pyc │ │ ├── TC.cpython-38.pyc │ │ ├── loss.cpython-38.pyc │ │ ├── loss.cpython-310.pyc │ │ ├── model.cpython-310.pyc │ │ ├── model.cpython-38.pyc │ │ ├── attention.cpython-38.pyc │ │ ├── soft_loss.cpython-38.pyc │ │ ├── attention.cpython-310.pyc │ │ ├── soft_labels.cpython-38.pyc │ │ ├── soft_loss.cpython-310.pyc │ │ └── soft_labels.cpython-310.pyc │ ├── timelags.py │ ├── hard_losses.py │ ├── TC.py │ ├── attention.py │ ├── model.py │ ├── loss.py │ └── soft_losses.py ├── trainer │ ├── __pycache__ │ │ ├── trainer.cpython-310.pyc │ │ ├── trainer.cpython-38.pyc │ │ ├── train_utils.cpython-310.pyc │ │ └── train_utils.cpython-38.pyc │ ├── train_utils.py │ └── trainer.py ├── dataloader │ ├── __pycache__ │ │ ├── dataloader.cpython-38.pyc │ │ ├── dataloader.cpython-310.pyc │ │ ├── augmentations.cpython-310.pyc │ │ └── augmentations.cpython-38.pyc │ ├── augmentations.py │ └── dataloader.py ├── config_files │ ├── __pycache__ │ │ ├── EMG_Configs.cpython-310.pyc │ │ ├── EMG_Configs.cpython-38.pyc │ │ ├── FD_B_Configs.cpython-310.pyc │ │ ├── FD_B_Configs.cpython-38.pyc │ │ ├── HAR_Configs.cpython-310.pyc │ │ ├── POC_Configs.cpython-310.pyc │ │ ├── PPOC_Configs.cpython-310.pyc │ │ ├── FordA_Configs.cpython-310.pyc │ │ ├── FordB_Configs.cpython-310.pyc │ │ ├── Gesture_Configs.cpython-38.pyc │ │ ├── Wafer_Configs.cpython-310.pyc │ │ ├── Epilepsy_Configs.cpython-310.pyc │ │ ├── Epilepsy_Configs.cpython-38.pyc │ │ ├── Gesture_Configs.cpython-310.pyc │ │ ├── SleepEEG_Configs.cpython-310.pyc │ │ ├── SleepEEG_Configs.cpython-38.pyc │ │ ├── ElectricDevices_Configs.cpython-310.pyc │ │ └── StarLightCurves_Configs.cpython-310.pyc │ ├── HAR_Configs.py │ ├── EMG_Configs.py │ ├── Epilepsy_Configs.py │ ├── FD_B_Configs.py │ ├── FordA_Configs.py │ ├── FordB_Configs.py │ ├── Gesture_Configs.py │ ├── POC_Configs.py │ ├── PPOC_Configs.py │ ├── Wafer_Configs.py │ ├── SleepEEG_Configs.py │ ├── ElectricDevices_Configs.py │ ├── StarLightCurves_Configs.py │ ├── pFD_Configs.py │ └── sleepEDF_Configs.py ├── utils.py ├── main_pretrain_TL.py ├── main_finetune_TL.py └── main_semi_classification.py └── README.md /softclt_ts2vec/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .encoder import TSEncoder, TSEncoder_all_repr -------------------------------------------------------------------------------- /softclt_catcc/models/__pycache__/TC.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/models/__pycache__/TC.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/models/__pycache__/TC.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/models/__pycache__/TC.cpython-38.pyc -------------------------------------------------------------------------------- /softclt_catcc/models/__pycache__/loss.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/models/__pycache__/loss.cpython-38.pyc -------------------------------------------------------------------------------- /softclt_catcc/models/__pycache__/loss.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/models/__pycache__/loss.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/models/__pycache__/model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/models/__pycache__/model.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/models/__pycache__/model.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/models/__pycache__/model.cpython-38.pyc -------------------------------------------------------------------------------- /softclt_catcc/models/__pycache__/attention.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/models/__pycache__/attention.cpython-38.pyc -------------------------------------------------------------------------------- /softclt_catcc/models/__pycache__/soft_loss.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/models/__pycache__/soft_loss.cpython-38.pyc -------------------------------------------------------------------------------- /softclt_catcc/trainer/__pycache__/trainer.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/trainer/__pycache__/trainer.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/trainer/__pycache__/trainer.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/trainer/__pycache__/trainer.cpython-38.pyc -------------------------------------------------------------------------------- /softclt_ts2vec/models/__pycache__/encoder.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_ts2vec/models/__pycache__/encoder.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_ts2vec/models/__pycache__/losses.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_ts2vec/models/__pycache__/losses.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_ts2vec/tasks/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_ts2vec/tasks/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/models/__pycache__/attention.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/models/__pycache__/attention.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/models/__pycache__/soft_labels.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/models/__pycache__/soft_labels.cpython-38.pyc -------------------------------------------------------------------------------- /softclt_catcc/models/__pycache__/soft_loss.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/models/__pycache__/soft_loss.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_ts2vec/models/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_ts2vec/models/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_ts2vec/models/__pycache__/losses_KL.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_ts2vec/models/__pycache__/losses_KL.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/dataloader/__pycache__/dataloader.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/dataloader/__pycache__/dataloader.cpython-38.pyc -------------------------------------------------------------------------------- /softclt_catcc/models/__pycache__/soft_labels.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/models/__pycache__/soft_labels.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/trainer/__pycache__/train_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/trainer/__pycache__/train_utils.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/trainer/__pycache__/train_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/trainer/__pycache__/train_utils.cpython-38.pyc -------------------------------------------------------------------------------- /softclt_ts2vec/models/__pycache__/dilated_conv.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_ts2vec/models/__pycache__/dilated_conv.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_ts2vec/models/__pycache__/losses_erase.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_ts2vec/models/__pycache__/losses_erase.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_ts2vec/models/__pycache__/soft_labels.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_ts2vec/models/__pycache__/soft_labels.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_ts2vec/tasks/__pycache__/forecasting.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_ts2vec/tasks/__pycache__/forecasting.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/dataloader/__pycache__/dataloader.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/dataloader/__pycache__/dataloader.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_ts2vec/tasks/__pycache__/_eval_protocols.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_ts2vec/tasks/__pycache__/_eval_protocols.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_ts2vec/tasks/__pycache__/classification.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_ts2vec/tasks/__pycache__/classification.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/EMG_Configs.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/EMG_Configs.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/EMG_Configs.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/EMG_Configs.cpython-38.pyc -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/FD_B_Configs.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/FD_B_Configs.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/FD_B_Configs.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/FD_B_Configs.cpython-38.pyc -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/HAR_Configs.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/HAR_Configs.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/POC_Configs.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/POC_Configs.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/PPOC_Configs.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/PPOC_Configs.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/dataloader/__pycache__/augmentations.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/dataloader/__pycache__/augmentations.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/dataloader/__pycache__/augmentations.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/dataloader/__pycache__/augmentations.cpython-38.pyc -------------------------------------------------------------------------------- /softclt_ts2vec/tasks/__pycache__/anomaly_detection.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_ts2vec/tasks/__pycache__/anomaly_detection.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/FordA_Configs.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/FordA_Configs.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/FordB_Configs.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/FordB_Configs.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/Gesture_Configs.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/Gesture_Configs.cpython-38.pyc -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/Wafer_Configs.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/Wafer_Configs.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_ts2vec/hyperparameters/ad_hyperparams.csv: -------------------------------------------------------------------------------- 1 | data,settings,tau_inst,tau_temp,bs 2 | yahoo,w/o inst,,8.0,64 3 | yahoo,w inst,0.0,5.0,16 4 | kpi,w/o inst,,1.5,8 5 | kpi,w inst,0.0,1.0,4 6 | -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/Epilepsy_Configs.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/Epilepsy_Configs.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/Epilepsy_Configs.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/Epilepsy_Configs.cpython-38.pyc -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/Gesture_Configs.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/Gesture_Configs.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/SleepEEG_Configs.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/SleepEEG_Configs.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/SleepEEG_Configs.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/SleepEEG_Configs.cpython-38.pyc -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/ElectricDevices_Configs.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/ElectricDevices_Configs.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_catcc/config_files/__pycache__/StarLightCurves_Configs.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seunghan96/softclt/HEAD/softclt_catcc/config_files/__pycache__/StarLightCurves_Configs.cpython-310.pyc -------------------------------------------------------------------------------- /softclt_ts2vec/hyperparameters/semi_cls_1p_hyperparams.csv: -------------------------------------------------------------------------------- 1 | data,tau_inst,tau_temp,bs 2 | HAR,20.0,1.0,4 3 | Epilepsy,40.0,2.0,16 4 | Wafer,1.0,0.5,4 5 | FordA,40.0,2.5,8 6 | FordB,30.0,1.5,4 7 | POC,1.0,1.0,16 8 | StarLightCurves,3.0,0.5,4 9 | ElectricDevices,40.0,2.5,8 10 | -------------------------------------------------------------------------------- /softclt_ts2vec/hyperparameters/semi_cls_5p_hyperparams.csv: -------------------------------------------------------------------------------- 1 | data,tau_inst,tau_temp,bs 2 | HAR,3.0,1.0,4 3 | Epilepsy,40.0,1.0,16 4 | Wafer,30.0,1.0,4 5 | FordA,40.0,2.5,8 6 | FordB,20.0,2.5,4 7 | POC,40.0,1.5,8 8 | StarLightCurves,1.0,2.5,4 9 | ElectricDevices,30.0,1.5,8 10 | -------------------------------------------------------------------------------- /softclt_ts2vec/ts2vec.yml: -------------------------------------------------------------------------------- 1 | name: ts2vec 2 | channels: 3 | - defaults 4 | - pytorch 5 | dependencies: 6 | - python=3.8 7 | - bottleneck==1.3.2 8 | # - torch==1.8.1 9 | - scipy==1.6.1 10 | - numpy==1.19.2 11 | - statsmodels==0.12.2 12 | - pandas==1.0.1 13 | - scikit-learn==0.24.2 -------------------------------------------------------------------------------- /softclt_ts2vec/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | from .classification import eval_classification, eval_semi_classification 2 | from .forecasting import eval_forecasting, eval_forecasting2,eval_forecasting_norm, eval_forecasting2_norm 3 | from .anomaly_detection import eval_anomaly_detection, eval_anomaly_detection_coldstart 4 | -------------------------------------------------------------------------------- /softclt_ts2vec/scripts/electricity.sh: -------------------------------------------------------------------------------- 1 | # multivar 2 | python -u train.py electricity forecast_multivar --loader forecast_csv --repr-dims 320 --max-threads 8 --seed 42 --eval 3 | 4 | # univar 5 | python -u train.py electricity forecast_univar --loader forecast_csv_univar --repr-dims 320 --max-threads 8 --seed 42 --eval 6 | -------------------------------------------------------------------------------- /softclt_ts2vec/scripts/kpi.sh: -------------------------------------------------------------------------------- 1 | python -u train.py kpi anomaly_0 --loader anomaly --repr-dims 320 --max-threads 8 --seed 1 --eval 2 | python -u train.py kpi anomaly_1 --loader anomaly --repr-dims 320 --max-threads 8 --seed 2 --eval 3 | python -u train.py kpi anomaly_2 --loader anomaly --repr-dims 320 --max-threads 8 --seed 3 --eval 4 | 5 | python -u train.py kpi anomaly_coldstart_0 --loader anomaly_coldstart --repr-dims 320 --max-threads 8 --seed 1 --eval 6 | python -u train.py kpi anomaly_coldstart_1 --loader anomaly_coldstart --repr-dims 320 --max-threads 8 --seed 2 --eval 7 | python -u train.py kpi anomaly_coldstart_2 --loader anomaly_coldstart --repr-dims 320 --max-threads 8 --seed 3 --eval 8 | -------------------------------------------------------------------------------- /softclt_ts2vec/scripts/yahoo.sh: -------------------------------------------------------------------------------- 1 | python -u train.py yahoo anomaly_0 --loader anomaly --repr-dims 320 --max-threads 8 --seed 1 --eval 2 | python -u train.py yahoo anomaly_1 --loader anomaly --repr-dims 320 --max-threads 8 --seed 2 --eval 3 | python -u train.py yahoo anomaly_2 --loader anomaly --repr-dims 320 --max-threads 8 --seed 3 --eval 4 | 5 | python -u train.py yahoo anomaly_coldstart_0 --loader anomaly_coldstart --repr-dims 320 --max-threads 8 --seed 1 --eval 6 | python -u train.py yahoo anomaly_coldstart_1 --loader anomaly_coldstart --repr-dims 320 --max-threads 8 --seed 2 --eval 7 | python -u train.py yahoo anomaly_coldstart_2 --loader anomaly_coldstart --repr-dims 320 --max-threads 8 --seed 3 --eval 8 | -------------------------------------------------------------------------------- /softclt_ts2vec/scripts/ett.sh: -------------------------------------------------------------------------------- 1 | # multivar 2 | python -u train.py ETTh1 forecast_multivar --loader forecast_csv --repr-dims 320 --max-threads 8 --seed 42 --eval 3 | python -u train.py ETTh2 forecast_multivar --loader forecast_csv --repr-dims 320 --max-threads 8 --seed 42 --eval 4 | python -u train.py ETTm1 forecast_multivar --loader forecast_csv --repr-dims 320 --max-threads 8 --seed 42 --eval 5 | 6 | # univar 7 | python -u train.py ETTh1 forecast_univar --loader forecast_csv_univar --repr-dims 320 --max-threads 8 --seed 42 --eval 8 | python -u train.py ETTh2 forecast_univar --loader forecast_csv_univar --repr-dims 320 --max-threads 8 --seed 42 --eval 9 | python -u train.py ETTm1 forecast_univar --loader forecast_csv_univar --repr-dims 320 --max-threads 8 --seed 42 --eval 10 | -------------------------------------------------------------------------------- /softclt_catcc/config_files/HAR_Configs.py: -------------------------------------------------------------------------------- 1 | class Config(object): 2 | def __init__(self): 3 | # model configs 4 | self.input_channels = 9 5 | self.kernel_size = 8 6 | self.stride = 1 7 | self.final_out_channels = 128 8 | 9 | self.num_classes = 6 10 | self.dropout = 0.35 11 | self.features_len = 18 12 | 13 | # training configs 14 | self.num_epoch = 2 15 | 16 | # optimizer parameters 17 | self.beta1 = 0.9 18 | self.beta2 = 0.99 19 | self.lr = 3e-4 20 | 21 | # data parameters 22 | self.drop_last = True 23 | self.batch_size = 128 24 | 25 | self.Context_Cont = Context_Cont_configs() 26 | self.TC = TC() 27 | self.augmentation = augmentations() 28 | 29 | 30 | class augmentations(object): 31 | def __init__(self): 32 | self.jitter_scale_ratio = 1.1 33 | self.jitter_ratio = 0.8 34 | self.max_seg = 8 35 | 36 | 37 | class Context_Cont_configs(object): 38 | def __init__(self): 39 | self.temperature = 0.2 40 | self.use_cosine_similarity = True 41 | 42 | 43 | class TC(object): 44 | def __init__(self): 45 | self.hidden_dim = 100 46 | self.timesteps = 6 47 | -------------------------------------------------------------------------------- /softclt_catcc/config_files/EMG_Configs.py: -------------------------------------------------------------------------------- 1 | class Config(object): 2 | def __init__(self): 3 | # model configs 4 | self.input_channels = 1 5 | self.kernel_size = 8 6 | self.stride = 1 7 | self.final_out_channels = 128 8 | 9 | self.num_classes = 3 10 | self.dropout = 0.35 11 | self.features_len = 190 12 | 13 | # training configs 14 | self.num_epoch = 40 15 | 16 | # optimizer parameters 17 | self.beta1 = 0.9 18 | self.beta2 = 0.99 19 | self.lr = 3e-4 20 | 21 | # data parameters 22 | self.drop_last = True 23 | self.batch_size = 128 24 | 25 | self.Context_Cont = Context_Cont_configs() 26 | self.TC = TC() 27 | self.augmentation = augmentations() 28 | 29 | 30 | class augmentations(object): 31 | def __init__(self): 32 | self.jitter_scale_ratio = 0.001 33 | self.jitter_ratio = 0.001 34 | self.max_seg = 10 35 | 36 | 37 | class Context_Cont_configs(object): 38 | def __init__(self): 39 | self.temperature = 0.2 40 | self.use_cosine_similarity = True 41 | 42 | 43 | class TC(object): 44 | def __init__(self): 45 | self.hidden_dim = 100 46 | self.timesteps = 10 47 | -------------------------------------------------------------------------------- /softclt_catcc/config_files/Epilepsy_Configs.py: -------------------------------------------------------------------------------- 1 | class Config(object): 2 | def __init__(self): 3 | # model configs 4 | self.input_channels = 1 5 | self.kernel_size = 8 6 | self.stride = 1 7 | self.final_out_channels = 128 8 | 9 | self.num_classes = 2 10 | self.dropout = 0.35 11 | self.features_len = 24 12 | 13 | # training configs 14 | self.num_epoch = 40 15 | 16 | # optimizer parameters 17 | self.beta1 = 0.9 18 | self.beta2 = 0.99 19 | self.lr = 3e-4 20 | 21 | # data parameters 22 | self.drop_last = True 23 | self.batch_size = 128 24 | 25 | self.Context_Cont = Context_Cont_configs() 26 | self.TC = TC() 27 | self.augmentation = augmentations() 28 | 29 | 30 | class augmentations(object): 31 | def __init__(self): 32 | self.jitter_scale_ratio = 0.001 33 | self.jitter_ratio = 0.001 34 | self.max_seg = 5 35 | 36 | 37 | class Context_Cont_configs(object): 38 | def __init__(self): 39 | self.temperature = 0.2 40 | self.use_cosine_similarity = True 41 | 42 | 43 | class TC(object): 44 | def __init__(self): 45 | self.hidden_dim = 100 46 | self.timesteps = 10 47 | -------------------------------------------------------------------------------- /softclt_catcc/config_files/FD_B_Configs.py: -------------------------------------------------------------------------------- 1 | class Config(object): 2 | def __init__(self): 3 | # model configs 4 | self.input_channels = 1 5 | self.kernel_size = 8 6 | self.stride = 1 7 | self.final_out_channels = 128 8 | 9 | self.num_classes = 3 10 | self.dropout = 0.35 11 | self.features_len = 642 12 | 13 | # training configs 14 | self.num_epoch = 40 15 | 16 | # optimizer parameters 17 | self.beta1 = 0.9 18 | self.beta2 = 0.99 19 | self.lr = 3e-4 20 | 21 | # data parameters 22 | self.drop_last = True 23 | self.batch_size = 128 24 | 25 | self.Context_Cont = Context_Cont_configs() 26 | self.TC = TC() 27 | self.augmentation = augmentations() 28 | 29 | 30 | class augmentations(object): 31 | def __init__(self): 32 | self.jitter_scale_ratio = 0.001 33 | self.jitter_ratio = 0.001 34 | self.max_seg = 10 35 | 36 | 37 | class Context_Cont_configs(object): 38 | def __init__(self): 39 | self.temperature = 0.2 40 | self.use_cosine_similarity = True 41 | 42 | 43 | class TC(object): 44 | def __init__(self): 45 | self.hidden_dim = 100 46 | self.timesteps = 10 47 | -------------------------------------------------------------------------------- /softclt_catcc/config_files/FordA_Configs.py: -------------------------------------------------------------------------------- 1 | class Config(object): 2 | def __init__(self): 3 | # model configs 4 | self.input_channels = 1 5 | self.kernel_size = 8 6 | self.stride = 1 7 | self.final_out_channels = 128 8 | 9 | self.num_classes = 2 10 | self.dropout = 0.35 11 | self.features_len = 65 12 | 13 | # training configs 14 | self.num_epoch = 40 15 | 16 | # optimizer parameters 17 | self.beta1 = 0.9 18 | self.beta2 = 0.99 19 | self.lr = 3e-4 20 | 21 | # data parameters 22 | self.drop_last = True 23 | self.batch_size = 128 24 | 25 | self.Context_Cont = Context_Cont_configs() 26 | self.TC = TC() 27 | self.augmentation = augmentations() 28 | 29 | 30 | class augmentations(object): 31 | def __init__(self): 32 | self.jitter_scale_ratio = 0.001 33 | self.jitter_ratio = 0.001 34 | self.max_seg = 10 35 | 36 | 37 | class Context_Cont_configs(object): 38 | def __init__(self): 39 | self.temperature = 0.2 40 | self.use_cosine_similarity = True 41 | 42 | 43 | class TC(object): 44 | def __init__(self): 45 | self.hidden_dim = 100 46 | self.timesteps = 10 47 | -------------------------------------------------------------------------------- /softclt_catcc/config_files/FordB_Configs.py: -------------------------------------------------------------------------------- 1 | class Config(object): 2 | def __init__(self): 3 | # model configs 4 | self.input_channels = 1 5 | self.kernel_size = 8 6 | self.stride = 1 7 | self.final_out_channels = 128 8 | 9 | self.num_classes = 2 10 | self.dropout = 0.35 11 | self.features_len = 65 12 | 13 | # training configs 14 | self.num_epoch = 40 15 | 16 | # optimizer parameters 17 | self.beta1 = 0.9 18 | self.beta2 = 0.99 19 | self.lr = 3e-4 20 | 21 | # data parameters 22 | self.drop_last = True 23 | self.batch_size = 128 24 | 25 | self.Context_Cont = Context_Cont_configs() 26 | self.TC = TC() 27 | self.augmentation = augmentations() 28 | 29 | 30 | class augmentations(object): 31 | def __init__(self): 32 | self.jitter_scale_ratio = 0.001 33 | self.jitter_ratio = 0.001 34 | self.max_seg = 10 35 | 36 | 37 | class Context_Cont_configs(object): 38 | def __init__(self): 39 | self.temperature = 0.2 40 | self.use_cosine_similarity = True 41 | 42 | 43 | class TC(object): 44 | def __init__(self): 45 | self.hidden_dim = 100 46 | self.timesteps = 10 47 | -------------------------------------------------------------------------------- /softclt_catcc/config_files/Gesture_Configs.py: -------------------------------------------------------------------------------- 1 | class Config(object): 2 | def __init__(self): 3 | # model configs 4 | self.input_channels = 3 5 | self.kernel_size = 8 6 | self.stride = 1 7 | self.final_out_channels = 128 8 | 9 | self.num_classes = 8 10 | self.dropout = 0.35 11 | self.features_len = 42 12 | 13 | # training configs 14 | self.num_epoch = 40 15 | 16 | # optimizer parameters 17 | self.beta1 = 0.9 18 | self.beta2 = 0.99 19 | self.lr = 3e-4 20 | 21 | # data parameters 22 | self.drop_last = True 23 | self.batch_size = 128 24 | 25 | self.Context_Cont = Context_Cont_configs() 26 | self.TC = TC() 27 | self.augmentation = augmentations() 28 | 29 | 30 | class augmentations(object): 31 | def __init__(self): 32 | self.jitter_scale_ratio = 0.001 33 | self.jitter_ratio = 0.001 34 | self.max_seg = 10 35 | 36 | 37 | class Context_Cont_configs(object): 38 | def __init__(self): 39 | self.temperature = 0.2 40 | self.use_cosine_similarity = True 41 | 42 | 43 | class TC(object): 44 | def __init__(self): 45 | self.hidden_dim = 100 46 | self.timesteps = 10 47 | -------------------------------------------------------------------------------- /softclt_catcc/config_files/POC_Configs.py: -------------------------------------------------------------------------------- 1 | class Config(object): 2 | def __init__(self): 3 | # model configs 4 | self.input_channels = 1 5 | self.kernel_size = 8 6 | self.stride = 1 7 | self.final_out_channels = 128 8 | 9 | self.num_classes = 2 10 | self.dropout = 0.35 11 | self.features_len = 12 12 | 13 | # training configs 14 | self.num_epoch = 40 15 | 16 | # optimizer parameters 17 | self.beta1 = 0.9 18 | self.beta2 = 0.99 19 | self.lr = 3e-4 20 | 21 | # data parameters 22 | self.drop_last = True 23 | self.batch_size = 128 24 | 25 | self.Context_Cont = Context_Cont_configs() 26 | self.TC = TC() 27 | self.augmentation = augmentations() 28 | 29 | 30 | class augmentations(object): 31 | def __init__(self): 32 | self.jitter_scale_ratio = 0.001 33 | self.jitter_ratio = 0.001 34 | self.max_seg = 10 35 | 36 | 37 | class Context_Cont_configs(object): 38 | def __init__(self): 39 | self.temperature = 0.2 40 | self.use_cosine_similarity = True 41 | 42 | 43 | class TC(object): 44 | def __init__(self): 45 | self.hidden_dim = 100 46 | self.timesteps = 10 47 | -------------------------------------------------------------------------------- /softclt_catcc/config_files/PPOC_Configs.py: -------------------------------------------------------------------------------- 1 | class Config(object): 2 | def __init__(self): 3 | # model configs 4 | self.input_channels = 1 5 | self.kernel_size = 8 6 | self.stride = 1 7 | self.final_out_channels = 128 8 | 9 | self.num_classes = 2 10 | self.dropout = 0.35 11 | self.features_len = 12 12 | 13 | # training configs 14 | self.num_epoch = 40 15 | 16 | # optimizer parameters 17 | self.beta1 = 0.9 18 | self.beta2 = 0.99 19 | self.lr = 3e-4 20 | 21 | # data parameters 22 | self.drop_last = True 23 | self.batch_size = 128 24 | 25 | self.Context_Cont = Context_Cont_configs() 26 | self.TC = TC() 27 | self.augmentation = augmentations() 28 | 29 | 30 | class augmentations(object): 31 | def __init__(self): 32 | self.jitter_scale_ratio = 0.001 33 | self.jitter_ratio = 0.001 34 | self.max_seg = 10 35 | 36 | 37 | class Context_Cont_configs(object): 38 | def __init__(self): 39 | self.temperature = 0.2 40 | self.use_cosine_similarity = True 41 | 42 | 43 | class TC(object): 44 | def __init__(self): 45 | self.hidden_dim = 100 46 | self.timesteps = 10 47 | -------------------------------------------------------------------------------- /softclt_catcc/config_files/Wafer_Configs.py: -------------------------------------------------------------------------------- 1 | class Config(object): 2 | def __init__(self): 3 | # model configs 4 | self.input_channels = 1 5 | self.kernel_size = 8 6 | self.stride = 1 7 | self.final_out_channels = 128 8 | 9 | self.num_classes = 2 10 | self.dropout = 0.35 11 | self.features_len = 21 12 | 13 | # training configs 14 | self.num_epoch = 40 15 | 16 | # optimizer parameters 17 | self.beta1 = 0.9 18 | self.beta2 = 0.99 19 | self.lr = 3e-4 20 | 21 | # data parameters 22 | self.drop_last = True 23 | self.batch_size = 128 24 | 25 | self.Context_Cont = Context_Cont_configs() 26 | self.TC = TC() 27 | self.augmentation = augmentations() 28 | 29 | 30 | class augmentations(object): 31 | def __init__(self): 32 | self.jitter_scale_ratio = 0.001 33 | self.jitter_ratio = 0.001 34 | self.max_seg = 10 35 | 36 | 37 | class Context_Cont_configs(object): 38 | def __init__(self): 39 | self.temperature = 0.2 40 | self.use_cosine_similarity = True 41 | 42 | 43 | class TC(object): 44 | def __init__(self): 45 | self.hidden_dim = 100 46 | self.timesteps = 10 47 | -------------------------------------------------------------------------------- /softclt_catcc/config_files/SleepEEG_Configs.py: -------------------------------------------------------------------------------- 1 | class Config(object): 2 | def __init__(self): 3 | # model configs 4 | self.input_channels = 1 5 | self.kernel_size = 8 6 | self.stride = 1 7 | self.final_out_channels = 128 8 | 9 | self.num_classes = 5 10 | self.dropout = 0.35 11 | self.features_len = 24 12 | 13 | # training configs 14 | self.num_epoch = 40 15 | 16 | # optimizer parameters 17 | self.beta1 = 0.9 18 | self.beta2 = 0.99 19 | self.lr = 3e-4 20 | 21 | # data parameters 22 | self.drop_last = True 23 | self.batch_size = 128 24 | 25 | self.Context_Cont = Context_Cont_configs() 26 | self.TC = TC() 27 | self.augmentation = augmentations() 28 | 29 | 30 | class augmentations(object): 31 | def __init__(self): 32 | self.jitter_scale_ratio = 0.001 33 | self.jitter_ratio = 0.001 34 | self.max_seg = 10 35 | 36 | 37 | class Context_Cont_configs(object): 38 | def __init__(self): 39 | self.temperature = 0.2 40 | self.use_cosine_similarity = True 41 | 42 | 43 | class TC(object): 44 | def __init__(self): 45 | self.hidden_dim = 100 46 | self.timesteps = 10 47 | -------------------------------------------------------------------------------- /softclt_catcc/config_files/ElectricDevices_Configs.py: -------------------------------------------------------------------------------- 1 | class Config(object): 2 | def __init__(self): 3 | # model configs 4 | self.input_channels = 1 5 | self.kernel_size = 8 6 | self.stride = 1 7 | self.final_out_channels = 128 8 | 9 | self.num_classes = 7 10 | self.dropout = 0.35 11 | self.features_len = 14 12 | 13 | # training configs 14 | self.num_epoch = 40 15 | 16 | # optimizer parameters 17 | self.beta1 = 0.9 18 | self.beta2 = 0.99 19 | self.lr = 3e-4 20 | 21 | # data parameters 22 | self.drop_last = True 23 | self.batch_size = 128 24 | 25 | self.Context_Cont = Context_Cont_configs() 26 | self.TC = TC() 27 | self.augmentation = augmentations() 28 | 29 | 30 | class augmentations(object): 31 | def __init__(self): 32 | self.jitter_scale_ratio = 0.001 33 | self.jitter_ratio = 0.001 34 | self.max_seg = 10 35 | 36 | 37 | class Context_Cont_configs(object): 38 | def __init__(self): 39 | self.temperature = 0.2 40 | self.use_cosine_similarity = True 41 | 42 | 43 | class TC(object): 44 | def __init__(self): 45 | self.hidden_dim = 100 46 | self.timesteps = 10 47 | -------------------------------------------------------------------------------- /softclt_catcc/config_files/StarLightCurves_Configs.py: -------------------------------------------------------------------------------- 1 | class Config(object): 2 | def __init__(self): 3 | # model configs 4 | self.input_channels = 1 5 | self.kernel_size = 8 6 | self.stride = 1 7 | self.final_out_channels = 128 8 | 9 | self.num_classes = 3 10 | self.dropout = 0.35 11 | self.features_len = 130 12 | 13 | # training configs 14 | self.num_epoch = 40 15 | 16 | # optimizer parameters 17 | self.beta1 = 0.9 18 | self.beta2 = 0.99 19 | self.lr = 3e-4 20 | 21 | # data parameters 22 | self.drop_last = True 23 | self.batch_size = 128 24 | 25 | self.Context_Cont = Context_Cont_configs() 26 | self.TC = TC() 27 | self.augmentation = augmentations() 28 | 29 | 30 | class augmentations(object): 31 | def __init__(self): 32 | self.jitter_scale_ratio = 0.001 33 | self.jitter_ratio = 0.001 34 | self.max_seg = 10 35 | 36 | 37 | class Context_Cont_configs(object): 38 | def __init__(self): 39 | self.temperature = 0.2 40 | self.use_cosine_similarity = True 41 | 42 | 43 | class TC(object): 44 | def __init__(self): 45 | self.hidden_dim = 100 46 | self.timesteps = 10 47 | -------------------------------------------------------------------------------- /softclt_catcc/config_files/pFD_Configs.py: -------------------------------------------------------------------------------- 1 | class Config(object): 2 | def __init__(self): 3 | # model configs 4 | self.input_channels = 1 5 | self.kernel_size = 32 6 | self.stride = 4 7 | self.final_out_channels = 128 8 | self.features_len = 162 9 | 10 | self.num_classes = 3 11 | self.dropout = 0.35 12 | 13 | # training configs 14 | self.num_epoch = 40 15 | self.batch_size = 64 16 | 17 | # optimizer parameters 18 | self.optimizer = 'adam' 19 | self.beta1 = 0.9 20 | self.beta2 = 0.99 21 | self.lr = 3e-4 22 | 23 | # data parameters 24 | self.drop_last = True 25 | 26 | self.Context_Cont = Context_Cont_configs() 27 | self.TC = TC() 28 | self.augmentation = augmentations() 29 | 30 | 31 | class augmentations(object): 32 | def __init__(self): 33 | self.jitter_scale_ratio = 2 34 | self.jitter_ratio = 0.1 35 | self.max_seg = 5 36 | 37 | 38 | class Context_Cont_configs(object): 39 | def __init__(self): 40 | self.temperature = 0.2 41 | self.use_cosine_similarity = True 42 | 43 | 44 | class TC(object): 45 | def __init__(self): 46 | self.hidden_dim = 64 47 | self.timesteps = 50 48 | -------------------------------------------------------------------------------- /softclt_catcc/config_files/sleepEDF_Configs.py: -------------------------------------------------------------------------------- 1 | class Config(object): 2 | def __init__(self): 3 | # model configs 4 | self.input_channels = 1 5 | self.final_out_channels = 128 6 | self.num_classes = 5 7 | self.dropout = 0.35 8 | 9 | self.kernel_size = 25 10 | self.stride = 3 11 | self.features_len = 127 12 | 13 | # training configs 14 | self.num_epoch = 40 15 | 16 | # optimizer parameters 17 | self.optimizer = 'adam' 18 | self.beta1 = 0.9 19 | self.beta2 = 0.99 20 | self.lr = 3e-4 21 | 22 | # data parameters 23 | self.drop_last = True 24 | self.batch_size = 128 25 | 26 | self.Context_Cont = Context_Cont_configs() 27 | self.TC = TC() 28 | self.augmentation = augmentations() 29 | 30 | 31 | class augmentations(object): 32 | def __init__(self): 33 | self.jitter_scale_ratio = 1.5 34 | self.jitter_ratio = 2 35 | self.max_seg = 12 36 | 37 | 38 | class Context_Cont_configs(object): 39 | def __init__(self): 40 | self.temperature = 0.2 41 | self.use_cosine_similarity = True 42 | 43 | 44 | class TC(object): 45 | def __init__(self): 46 | self.hidden_dim = 64 47 | self.timesteps = 50 48 | -------------------------------------------------------------------------------- /softclt_catcc/dataloader/augmentations.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | 5 | def DataTransform(sample, config): 6 | weak_aug = scaling(sample, sigma=config.augmentation.jitter_scale_ratio) 7 | strong_aug = jitter(permutation(sample, max_segments=config.augmentation.max_seg), config.augmentation.jitter_ratio) 8 | return weak_aug, strong_aug 9 | 10 | 11 | def jitter(x, sigma=0.8): 12 | # https://arxiv.org/pdf/1706.00527.pdf 13 | return x + np.random.normal(loc=0., scale=sigma, size=x.shape) 14 | 15 | 16 | def scaling(x, sigma=1.1): 17 | # https://arxiv.org/pdf/1706.00527.pdf 18 | x = x.cpu().numpy() 19 | factor = np.random.normal(loc=2., scale=sigma, size=(x.shape[0], x.shape[2])) 20 | ai = [] 21 | for i in range(x.shape[1]): 22 | xi = x[:, i, :] 23 | ai.append(np.multiply(xi, factor[:, :])[:, np.newaxis, :]) 24 | return np.concatenate((ai), axis=1) 25 | 26 | 27 | def permutation(x, max_segments=5, seg_mode="random"): 28 | orig_steps = np.arange(x.shape[2]) 29 | x = x.cpu().numpy() 30 | num_segs = np.random.randint(1, max_segments, size=(x.shape[0])) 31 | 32 | ret = np.zeros_like(x) 33 | for i, pat in enumerate(x): 34 | if num_segs[i] > 1: 35 | if seg_mode == "random": 36 | split_points = np.random.choice(x.shape[2] - 2, num_segs[i] - 1, replace=False) 37 | split_points.sort() 38 | splits = np.split(orig_steps, split_points) 39 | else: 40 | splits = np.array_split(orig_steps, num_segs[i]) 41 | warp = np.concatenate(np.random.permutation(splits)).ravel() 42 | ret[i] = pat[0, warp] 43 | else: 44 | ret[i] = pat 45 | return torch.from_numpy(ret) 46 | -------------------------------------------------------------------------------- /softclt_catcc/models/timelags.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | def dup_matrix(mat): 5 | mat0 = torch.tril(mat, diagonal=-1)[:, :-1] 6 | mat0 += torch.triu(mat, diagonal=1)[:, 1:] 7 | mat1 = torch.cat([mat0,mat],dim=1) 8 | mat2 = torch.cat([mat,mat0],dim=1) 9 | return mat1, mat2 10 | 11 | ############################################################################## 12 | ## 6 Different ways of generating time lags 13 | ############################################################################## 14 | def timelag_sigmoid(T,sigma=1): 15 | dist = np.arange(T) 16 | dist = np.abs(dist - dist[:, np.newaxis]) 17 | matrix = 2 / (1 +np.exp(dist*sigma)) 18 | matrix = np.where(matrix < 1e-6, 0, matrix) # set very small values to 0 19 | return matrix 20 | 21 | def timelag_gaussian(T,sigma): 22 | dist = np.arange(T) 23 | dist = np.abs(dist - dist[:, np.newaxis]) 24 | matrix = np.exp(-(dist**2)/(2 * sigma ** 2)) 25 | matrix = np.where(matrix < 1e-6, 0, matrix) 26 | return matrix 27 | 28 | def timelag_same_interval(T): 29 | d = np.arange(T) 30 | X, Y = np.meshgrid(d, d) 31 | matrix = 1 - np.abs(X - Y) / T 32 | return matrix 33 | 34 | def timelag_sigmoid_window(T, sigma=1, window_ratio=1.0): 35 | dist = np.arange(T) 36 | dist = np.abs(dist - dist[:, np.newaxis]) 37 | matrix = 2 / (1 +np.exp(dist*sigma)) 38 | matrix = np.where(matrix < 1e-6, 0, matrix) 39 | dist_from_diag = np.abs(np.subtract.outer(np.arange(dist.shape[0]), np.arange(dist.shape[1]))) 40 | matrix[dist_from_diag > T*window_ratio] = 0 41 | return matrix 42 | 43 | def timelag_sigmoid_threshold(T, threshold=1.0): 44 | dist = np.ones((T,T)) 45 | dist_from_diag = np.abs(np.subtract.outer(np.arange(dist.shape[0]), np.arange(dist.shape[1]))) 46 | dist[dist_from_diag > T*threshold] = 0 47 | return dist 48 | 49 | ############################################################################## 50 | 51 | -------------------------------------------------------------------------------- /softclt_ts2vec/models/timelags.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | def dup_matrix(mat): 5 | mat0 = torch.tril(mat, diagonal=-1)[:, :-1] 6 | mat0 += torch.triu(mat, diagonal=1)[:, 1:] 7 | mat1 = torch.cat([mat0,mat],dim=1) 8 | mat2 = torch.cat([mat,mat0],dim=1) 9 | return mat1, mat2 10 | 11 | ############################################################################## 12 | ## 6 Different ways of generating time lags 13 | ############################################################################## 14 | def timelag_sigmoid(T,sigma=1): 15 | dist = np.arange(T) 16 | dist = np.abs(dist - dist[:, np.newaxis]) 17 | matrix = 2 / (1 +np.exp(dist*sigma)) 18 | matrix = np.where(matrix < 1e-6, 0, matrix) # set very small values to 0 19 | return matrix 20 | 21 | def timelag_gaussian(T,sigma): 22 | dist = np.arange(T) 23 | dist = np.abs(dist - dist[:, np.newaxis]) 24 | matrix = np.exp(-(dist**2)/(2 * sigma ** 2)) 25 | matrix = np.where(matrix < 1e-6, 0, matrix) 26 | return matrix 27 | 28 | def timelag_same_interval(T): 29 | d = np.arange(T) 30 | X, Y = np.meshgrid(d, d) 31 | matrix = 1 - np.abs(X - Y) / T 32 | return matrix 33 | 34 | def timelag_sigmoid_window(T, sigma=1, window_ratio=1.0): 35 | dist = np.arange(T) 36 | dist = np.abs(dist - dist[:, np.newaxis]) 37 | matrix = 2 / (1 +np.exp(dist*sigma)) 38 | matrix = np.where(matrix < 1e-6, 0, matrix) 39 | dist_from_diag = np.abs(np.subtract.outer(np.arange(dist.shape[0]), np.arange(dist.shape[1]))) 40 | matrix[dist_from_diag > T*window_ratio] = 0 41 | return matrix 42 | 43 | def timelag_sigmoid_threshold(T, threshold=1.0): 44 | dist = np.ones((T,T)) 45 | dist_from_diag = np.abs(np.subtract.outer(np.arange(dist.shape[0]), np.arange(dist.shape[1]))) 46 | dist[dist_from_diag > T*threshold] = 0 47 | return dist 48 | 49 | ############################################################################## 50 | 51 | -------------------------------------------------------------------------------- /softclt_catcc/models/hard_losses.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | 4 | ######################################################################################################## 5 | ## Hard Contrastive Losses 6 | ######################################################################################################## 7 | 8 | def inst_CL_hard(z1, z2): 9 | B, T = z1.size(0), z1.size(1) 10 | if B == 1: 11 | return z1.new_tensor(0.) 12 | z = torch.cat([z1, z2], dim=0) # 2B x T x C 13 | z = z.transpose(0, 1) # T x 2B x C 14 | sim = torch.matmul(z, z.transpose(1, 2)) # T x 2B x 2B 15 | logits = torch.tril(sim, diagonal=-1)[:, :, :-1] # T x 2B x (2B-1) 16 | logits += torch.triu(sim, diagonal=1)[:, :, 1:] 17 | logits = -F.log_softmax(logits, dim=-1) 18 | 19 | i = torch.arange(B, device=z1.device) 20 | loss = (logits[:, i, B + i - 1].mean() + logits[:, B + i, i].mean()) / 2 21 | return loss 22 | 23 | def temp_CL_hard(z1, z2): 24 | B, T = z1.size(0), z1.size(1) 25 | if T == 1: 26 | return z1.new_tensor(0.) 27 | z = torch.cat([z1, z2], dim=1) # B x 2T x C 28 | sim = torch.matmul(z, z.transpose(1, 2)) # B x 2T x 2T 29 | logits = torch.tril(sim, diagonal=-1)[:, :, :-1] # B x 2T x (2T-1) 30 | logits += torch.triu(sim, diagonal=1)[:, :, 1:] 31 | logits = -F.log_softmax(logits, dim=-1) 32 | 33 | t = torch.arange(T, device=z1.device) 34 | loss = (logits[:, t, T + t - 1].mean() + logits[:, T + t, t].mean()) / 2 35 | return loss 36 | 37 | def hier_CL_hard(z1, z2, lambda_=0.5, temporal_unit=0): 38 | loss = torch.tensor(0., device=z1.device) 39 | d = 0 40 | while z1.size(1) > 1: 41 | if lambda_ != 0: 42 | loss += lambda_ * inst_CL_hard(z1, z2) 43 | if d >= temporal_unit: 44 | if 1 - lambda_ != 0: 45 | loss += (1 - lambda_) * temp_CL_hard(z1, z2) 46 | d += 1 47 | z1 = F.max_pool1d(z1.transpose(1, 2), kernel_size=2).transpose(1, 2) 48 | z2 = F.max_pool1d(z2.transpose(1, 2), kernel_size=2).transpose(1, 2) 49 | 50 | if z1.size(1) == 1: 51 | if lambda_ != 0: 52 | loss += lambda_ * inst_CL_hard(z1, z2) 53 | d += 1 54 | return loss / d 55 | 56 | -------------------------------------------------------------------------------- /softclt_ts2vec/models/hard_losses.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | 4 | ######################################################################################################## 5 | ## Hard Contrastive Losses 6 | ######################################################################################################## 7 | 8 | def inst_CL_hard(z1, z2): 9 | B, T = z1.size(0), z1.size(1) 10 | if B == 1: 11 | return z1.new_tensor(0.) 12 | z = torch.cat([z1, z2], dim=0) # 2B x T x C 13 | z = z.transpose(0, 1) # T x 2B x C 14 | sim = torch.matmul(z, z.transpose(1, 2)) # T x 2B x 2B 15 | logits = torch.tril(sim, diagonal=-1)[:, :, :-1] # T x 2B x (2B-1) 16 | logits += torch.triu(sim, diagonal=1)[:, :, 1:] 17 | logits = -F.log_softmax(logits, dim=-1) 18 | 19 | i = torch.arange(B, device=z1.device) 20 | loss = (logits[:, i, B + i - 1].mean() + logits[:, B + i, i].mean()) / 2 21 | return loss 22 | 23 | def temp_CL_hard(z1, z2): 24 | B, T = z1.size(0), z1.size(1) 25 | if T == 1: 26 | return z1.new_tensor(0.) 27 | z = torch.cat([z1, z2], dim=1) # B x 2T x C 28 | sim = torch.matmul(z, z.transpose(1, 2)) # B x 2T x 2T 29 | logits = torch.tril(sim, diagonal=-1)[:, :, :-1] # B x 2T x (2T-1) 30 | logits += torch.triu(sim, diagonal=1)[:, :, 1:] 31 | logits = -F.log_softmax(logits, dim=-1) 32 | 33 | t = torch.arange(T, device=z1.device) 34 | loss = (logits[:, t, T + t - 1].mean() + logits[:, T + t, t].mean()) / 2 35 | return loss 36 | 37 | def hier_CL_hard(z1, z2, lambda_=0.5, temporal_unit=0): 38 | loss = torch.tensor(0., device=z1.device) 39 | d = 0 40 | while z1.size(1) > 1: 41 | if lambda_ != 0: 42 | loss += lambda_ * inst_CL_hard(z1, z2) 43 | if d >= temporal_unit: 44 | if 1 - lambda_ != 0: 45 | loss += (1 - lambda_) * temp_CL_hard(z1, z2) 46 | d += 1 47 | z1 = F.max_pool1d(z1.transpose(1, 2), kernel_size=2).transpose(1, 2) 48 | z2 = F.max_pool1d(z2.transpose(1, 2), kernel_size=2).transpose(1, 2) 49 | 50 | if z1.size(1) == 1: 51 | if lambda_ != 0: 52 | loss += lambda_ * inst_CL_hard(z1, z2) 53 | d += 1 54 | return loss / d 55 | 56 | -------------------------------------------------------------------------------- /softclt_catcc/models/TC.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | 5 | from .attention import Seq_Transformer 6 | 7 | 8 | class TC(nn.Module): 9 | def __init__(self, configs, device): 10 | super(TC, self).__init__() 11 | self.num_channels = configs.final_out_channels 12 | self.timestep = configs.TC.timesteps 13 | self.Wk = nn.ModuleList([nn.Linear(configs.TC.hidden_dim, self.num_channels) for i in range(self.timestep)]) 14 | self.lsoftmax = nn.LogSoftmax() 15 | self.device = device 16 | 17 | self.projection_head = nn.Sequential( 18 | nn.Linear(configs.TC.hidden_dim, configs.final_out_channels // 2), 19 | nn.BatchNorm1d(configs.final_out_channels // 2), 20 | nn.ReLU(inplace=True), 21 | nn.Linear(configs.final_out_channels // 2, configs.final_out_channels // 4), 22 | ) 23 | 24 | self.seq_transformer = Seq_Transformer(patch_size=self.num_channels, dim=configs.TC.hidden_dim, depth=4, 25 | heads=4, mlp_dim=64) 26 | 27 | def forward(self, z_aug1, z_aug2): 28 | seq_len = z_aug1.shape[2] 29 | 30 | z_aug1 = z_aug1.transpose(1, 2) 31 | z_aug2 = z_aug2.transpose(1, 2) 32 | 33 | batch = z_aug1.shape[0] 34 | t_samples = torch.randint(seq_len - self.timestep, size=(1,)).long().to( 35 | self.device) # randomly pick time stamps 36 | 37 | nce = 0 # average over timestep and batch 38 | encode_samples = torch.empty((self.timestep, batch, self.num_channels)).float().to(self.device) 39 | 40 | for i in np.arange(1, self.timestep + 1): 41 | encode_samples[i - 1] = z_aug2[:, t_samples + i, :].view(batch, self.num_channels) 42 | forward_seq = z_aug1[:, :t_samples + 1, :] 43 | 44 | c_t = self.seq_transformer(forward_seq) 45 | 46 | pred = torch.empty((self.timestep, batch, self.num_channels)).float().to(self.device) 47 | for i in np.arange(0, self.timestep): 48 | linear = self.Wk[i] 49 | pred[i] = linear(c_t) 50 | for i in np.arange(0, self.timestep): 51 | total = torch.mm(encode_samples[i], torch.transpose(pred[i], 0, 1)) 52 | nce += torch.sum(torch.diag(self.lsoftmax(total))) 53 | nce /= -1. * batch * self.timestep 54 | return nce, self.projection_head(c_t) 55 | -------------------------------------------------------------------------------- /softclt_ts2vec/models/dilated_conv.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | import numpy as np 5 | 6 | class SamePadConv(nn.Module): 7 | def __init__(self, in_channels, out_channels, kernel_size, dilation=1, groups=1): 8 | super().__init__() 9 | self.receptive_field = (kernel_size - 1) * dilation + 1 10 | padding = self.receptive_field // 2 11 | self.conv = nn.Conv1d( 12 | in_channels, out_channels, kernel_size, 13 | padding=padding, 14 | dilation=dilation, 15 | groups=groups 16 | ) 17 | self.remove = 1 if self.receptive_field % 2 == 0 else 0 18 | 19 | def forward(self, x): 20 | out = self.conv(x) 21 | if self.remove > 0: 22 | out = out[:, :, : -self.remove] 23 | return out 24 | 25 | class ConvBlock(nn.Module): 26 | def __init__(self, in_channels, out_channels, kernel_size, dilation, final=False): 27 | super().__init__() 28 | self.conv1 = SamePadConv(in_channels, out_channels, kernel_size, dilation=dilation) 29 | self.conv2 = SamePadConv(out_channels, out_channels, kernel_size, dilation=dilation) 30 | self.projector = nn.Conv1d(in_channels, out_channels, 1) if in_channels != out_channels or final else None 31 | 32 | def forward(self, x): 33 | residual = x if self.projector is None else self.projector(x) 34 | x = F.gelu(x) 35 | x = self.conv1(x) 36 | x = F.gelu(x) 37 | x = self.conv2(x) 38 | return x + residual 39 | 40 | class DilatedConvEncoder(nn.Module): 41 | def __init__(self, in_channels, channels, kernel_size): 42 | super().__init__() 43 | self.net = nn.Sequential(*[ 44 | ConvBlock( 45 | channels[i-1] if i > 0 else in_channels, 46 | channels[i], 47 | kernel_size=kernel_size, 48 | dilation=2**i, 49 | final=(i == len(channels)-1) 50 | ) 51 | for i in range(len(channels)) 52 | ]) 53 | 54 | def forward(self, x): 55 | return self.net(x) 56 | 57 | class DilatedConvEncoder_all_repr(nn.Module): 58 | def __init__(self, in_channels, channels, kernel_size): 59 | super().__init__() 60 | self.net = nn.ModuleList([ 61 | ConvBlock( 62 | channels[i-1] if i > 0 else in_channels, 63 | channels[i], 64 | kernel_size=kernel_size, 65 | dilation=2**i, 66 | final=(i == len(channels)-1) 67 | ) 68 | for i in range(len(channels)) 69 | ]) 70 | 71 | def forward(self, x): 72 | out = [] 73 | out.append(x) 74 | for n in self.net: 75 | x = n(x) 76 | out.append(x) 77 | return x, out 78 | -------------------------------------------------------------------------------- /softclt_ts2vec/scripts/uea.sh: -------------------------------------------------------------------------------- 1 | python -u train.py ERing UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 2 | python -u train.py Libras UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 3 | python -u train.py AtrialFibrillation UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 4 | python -u train.py BasicMotions UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 5 | python -u train.py RacketSports UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 6 | python -u train.py Handwriting UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 7 | python -u train.py Epilepsy UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 8 | python -u train.py JapaneseVowels UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 9 | python -u train.py UWaveGestureLibrary UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 10 | python -u train.py PenDigits UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 11 | python -u train.py StandWalkJump UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 12 | python -u train.py NATOPS UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 13 | python -u train.py ArticularyWordRecognition UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 14 | python -u train.py FingerMovements UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 15 | python -u train.py LSST UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 16 | python -u train.py HandMovementDirection UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 17 | python -u train.py Cricket UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 18 | python -u train.py CharacterTrajectories UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 19 | python -u train.py EthanolConcentration UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 20 | python -u train.py SelfRegulationSCP1 UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 21 | python -u train.py SelfRegulationSCP2 UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 22 | python -u train.py Heartbeat UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 23 | python -u train.py PhonemeSpectra UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 24 | python -u train.py SpokenArabicDigits UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 25 | python -u train.py EigenWorms UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 26 | python -u train.py DuckDuckGeese UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 27 | python -u train.py PEMS-SF UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 28 | python -u train.py FaceDetection UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 29 | python -u train.py MotorImagery UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 30 | python -u train.py InsectWingbeat UEA --loader UEA --batch-size 8 --repr-dims 320 --max-threads 8 --seed 42 --eval 31 | -------------------------------------------------------------------------------- /softclt_ts2vec/hyperparameters/cls_hyperparams.csv: -------------------------------------------------------------------------------- 1 | ,0 2 | ACSF1,20.0-2.0 3 | Adiac,1.0-0.1 4 | AllGestureWiimoteX,30.0-1.5 5 | AllGestureWiimoteY,30.0-2.5 6 | AllGestureWiimoteZ,40.0-2.0 7 | ArrowHead,3.0-2.5 8 | BME,0.1-0.1 9 | Beef,20.0-0.1 10 | BeetleFly,0.1-1.0 11 | BirdChicken,30.0-2.5 12 | CBF,10.0-0.5 13 | Car,30.0-0.5 14 | Chinatown,0.1-2.5 15 | ChlorineConcentration,5.0-0.1 16 | CinCECGTorso,3.0-0.5 17 | Coffee,0.1-0.1 18 | Computers,0.1-1.0 19 | CricketX,10.0-2.0 20 | CricketY,1.0-0.1 21 | CricketZ,5.0-2.5 22 | Crop,1.0-0.5 23 | DiatomSizeReduction,3.0-2.5 24 | DistalPhalanxOutlineAgeGroup,0.5-0.1 25 | DistalPhalanxOutlineCorrect,0.5-2.5 26 | DistalPhalanxTW,5.0-0.5 27 | DodgerLoopDay,0.1-0.5 28 | DodgerLoopGame,10.0-1.0 29 | DodgerLoopWeekend,20.0-2.5 30 | ECG200,20.0-2.0 31 | ECG5000,5.0-2.5 32 | ECGFiveDays,0.1-0.1 33 | EOGHorizontalSignal,3.0-1.0 34 | EOGVerticalSignal,20.0-1.0 35 | Earthquakes,0.1-0.1 36 | ElectricDevices,30.0-2.0 37 | EthanolLevel,0.5-2.0 38 | FaceAll,0.5-0.5 39 | FaceFour,30.0-0.5 40 | FacesUCR,40.0-0.5 41 | FiftyWords,0.5-2.5 42 | Fish,20.0-1.0 43 | FordA,10.0-2.0 44 | FordB,40.0-2.5 45 | FreezerRegularTrain,0.1-0.5 46 | FreezerSmallTrain,0.1-1.0 47 | Fungi,3.0-2.5 48 | GestureMidAirD1,0.1-0.5 49 | GestureMidAirD2,20.0-2.0 50 | GestureMidAirD3,3.0-0.1 51 | GesturePebbleZ1,10.0-0.1 52 | GesturePebbleZ2,10.0-0.1 53 | GunPoint,0.1-1.0 54 | GunPointAgeSpan,0.5-2.5 55 | GunPointMaleVersusFemale,0.1-0.1 56 | GunPointOldVersusYoung,0.1-0.1 57 | Ham,10.0-0.1 58 | HandOutlines,10.0-0.5 59 | Haptics,40.0-2.0 60 | Herring,20.0-0.5 61 | HouseTwenty,3.0-0.5 62 | InlineSkate,5.0-2.5 63 | InsectEPGRegularTrain,0.1-0.1 64 | InsectEPGSmallTrain,0.1-0.1 65 | InsectWingbeatSound,0.1-1.0 66 | ItalyPowerDemand,0.1-0.1 67 | LargeKitchenAppliances,10.0-2.5 68 | Lightning2,1.0-1.0 69 | Lightning7,30.0-2.0 70 | Mallat,30.0-2.5 71 | Meat,5.0-0.5 72 | MedicalImages,40.0-2.0 73 | MelbournePedestrian,30.0-1.0 74 | MiddlePhalanxOutlineAgeGroup,0.1-2.5 75 | MiddlePhalanxOutlineCorrect,0.5-2.5 76 | MiddlePhalanxTW,0.1-2.5 77 | MixedShapesRegularTrain,5.0-2.5 78 | MixedShapesSmallTrain,1.0-1.0 79 | MoteStrain,10.0-0.5 80 | NonInvasiveFetalECGThorax1,5.0-2.5 81 | NonInvasiveFetalECGThorax2,5.0-1.5 82 | OSULeaf,40.0-1.5 83 | OliveOil,0.1-0.1 84 | PLAID,3.0-2.0 85 | PhalangesOutlinesCorrect,0.1-1.5 86 | Phoneme,10.0-2.5 87 | PickupGestureWiimoteZ,5.0-0.5 88 | PigAirwayPressure,5.0-0.1 89 | PigArtPressure,30.0-0.5 90 | PigCVP,30.0-2.0 91 | Plane,0.1-0.1 92 | PowerCons,5.0-0.1 93 | ProximalPhalanxOutlineAgeGroup,0.1-0.1 94 | ProximalPhalanxOutlineCorrect,0.5-0.1 95 | ProximalPhalanxTW,10.0-2.5 96 | RefrigerationDevices,0.1-1.5 97 | Rock,20.0-0.1 98 | ScreenType,1.0-2.0 99 | SemgHandGenderCh2,40.0-0.1 100 | SemgHandMovementCh2,40.0-2.5 101 | SemgHandSubjectCh2,5.0-0.1 102 | ShakeGestureWiimoteZ,20.0-2.5 103 | ShapeletSim,20.0-0.5 104 | ShapesAll,20.0-2.5 105 | SmallKitchenAppliances,5.0-1.0 106 | SmoothSubspace,20.0-1.0 107 | SonyAIBORobotSurface1,3.0-0.1 108 | SonyAIBORobotSurface2,0.5-2.5 109 | StarLightCurves,10.0-2.5 110 | Strawberry,0.1-0.5 111 | SwedishLeaf,3.0-1.5 112 | Symbols,10.0-0.5 113 | SyntheticControl,1.0-0.5 114 | ToeSegmentation1,0.5-0.5 115 | ToeSegmentation2,30.0-0.5 116 | Trace,0.1-0.1 117 | TwoLeadECG,20.0-0.5 118 | TwoPatterns,0.1-0.5 119 | UMD,0.1-0.1 120 | UWaveGestureLibraryAll,0.1-0.1 121 | UWaveGestureLibraryX,3.0-2.0 122 | UWaveGestureLibraryY,3.0-0.5 123 | UWaveGestureLibraryZ,5.0-2.0 124 | Wafer,0.1-1.0 125 | Wine,0.1-1.0 126 | WordSynonyms,10.0-0.5 127 | Worms,10.0-1.5 128 | WormsTwoClass,40.0-0.5 129 | Yoga,10.0-1.5 130 | -------------------------------------------------------------------------------- /softclt_catcc/models/attention.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from einops import rearrange, repeat 5 | 6 | 7 | ######################################################################################## 8 | 9 | class Residual(nn.Module): 10 | def __init__(self, fn): 11 | super().__init__() 12 | self.fn = fn 13 | 14 | def forward(self, x, **kwargs): 15 | return self.fn(x, **kwargs) + x 16 | 17 | 18 | class PreNorm(nn.Module): 19 | def __init__(self, dim, fn): 20 | super().__init__() 21 | self.norm = nn.LayerNorm(dim) 22 | self.fn = fn 23 | 24 | def forward(self, x, **kwargs): 25 | return self.fn(self.norm(x), **kwargs) 26 | 27 | 28 | class FeedForward(nn.Module): 29 | def __init__(self, dim, hidden_dim, dropout=0.): 30 | super().__init__() 31 | self.net = nn.Sequential( 32 | nn.Linear(dim, hidden_dim), 33 | nn.ReLU(), 34 | nn.Dropout(dropout), 35 | nn.Linear(hidden_dim, dim), 36 | nn.Dropout(dropout) 37 | ) 38 | 39 | def forward(self, x): 40 | return self.net(x) 41 | 42 | 43 | class Attention(nn.Module): 44 | def __init__(self, dim, heads=8, dropout=0.): 45 | super().__init__() 46 | self.heads = heads 47 | self.scale = dim ** -0.5 48 | 49 | self.to_qkv = nn.Linear(dim, dim * 3, bias=False) 50 | self.to_out = nn.Sequential( 51 | nn.Linear(dim, dim), 52 | nn.Dropout(dropout) 53 | ) 54 | 55 | def forward(self, x, mask=None): 56 | b, n, _, h = *x.shape, self.heads 57 | qkv = self.to_qkv(x).chunk(3, dim=-1) 58 | q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h=h), qkv) 59 | 60 | dots = torch.einsum('bhid,bhjd->bhij', q, k) * self.scale 61 | 62 | if mask is not None: 63 | mask = F.pad(mask.flatten(1), (1, 0), value=True) 64 | assert mask.shape[-1] == dots.shape[-1], 'mask has incorrect dimensions' 65 | mask = mask[:, None, :] * mask[:, :, None] 66 | dots.masked_fill_(~mask, float('-inf')) 67 | del mask 68 | 69 | attn = dots.softmax(dim=-1) 70 | 71 | out = torch.einsum('bhij,bhjd->bhid', attn, v) 72 | out = rearrange(out, 'b h n d -> b n (h d)') 73 | out = self.to_out(out) 74 | return out 75 | 76 | 77 | class Transformer(nn.Module): 78 | def __init__(self, dim, depth, heads, mlp_dim, dropout): 79 | super().__init__() 80 | self.layers = nn.ModuleList([]) 81 | for _ in range(depth): 82 | self.layers.append(nn.ModuleList([ 83 | Residual(PreNorm(dim, Attention(dim, heads=heads, dropout=dropout))), 84 | Residual(PreNorm(dim, FeedForward(dim, mlp_dim, dropout=dropout))) 85 | ])) 86 | 87 | def forward(self, x, mask=None): 88 | for attn, ff in self.layers: 89 | x = attn(x, mask=mask) 90 | x = ff(x) 91 | return x 92 | 93 | 94 | class Seq_Transformer(nn.Module): 95 | def __init__(self, *, patch_size, dim, depth, heads, mlp_dim, channels=1, dropout=0.1): 96 | super().__init__() 97 | patch_dim = channels * patch_size 98 | self.patch_to_embedding = nn.Linear(patch_dim, dim) 99 | self.c_token = nn.Parameter(torch.randn(1, 1, dim)) 100 | self.transformer = Transformer(dim, depth, heads, mlp_dim, dropout) 101 | self.to_c_token = nn.Identity() 102 | 103 | def forward(self, forward_seq): 104 | x = self.patch_to_embedding(forward_seq) 105 | b, n, _ = x.shape 106 | c_tokens = repeat(self.c_token, '() n d -> b n d', b=b) 107 | x = torch.cat((c_tokens, x), dim=1) 108 | x = self.transformer(x) 109 | c_t = self.to_c_token(x[:, 0]) 110 | return c_t 111 | -------------------------------------------------------------------------------- /softclt_catcc/utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import random 4 | import sys 5 | 6 | import numpy as np 7 | import pandas as pd 8 | import torch 9 | from sklearn.metrics import classification_report, cohen_kappa_score, confusion_matrix, accuracy_score 10 | from torch import nn 11 | 12 | 13 | def set_requires_grad(model, dict_, requires_grad=True): 14 | for param in model.named_parameters(): 15 | if param[0] in dict_: 16 | param[1].requires_grad = requires_grad 17 | 18 | 19 | def loop_iterable(iterable): 20 | while True: 21 | yield from iterable 22 | 23 | 24 | def fix_randomness(SEED): 25 | random.seed(SEED) 26 | np.random.seed(SEED) 27 | torch.manual_seed(SEED) 28 | torch.cuda.manual_seed(SEED) 29 | torch.backends.cudnn.deterministic = True 30 | 31 | 32 | def init_weights(m): 33 | for name, param in m.named_parameters(): 34 | nn.init.uniform_(param.data, -0.08, 0.08) 35 | # if name=='weight': 36 | # nn.init.kaiming_uniform_(param.data) 37 | # else: 38 | # torch.nn.init.zeros_(param.data) 39 | 40 | 41 | def count_parameters(model): 42 | return sum(p.numel() for p in model.parameters() if p.requires_grad) 43 | 44 | 45 | def epoch_time(start_time, end_time): 46 | elapsed_time = end_time - start_time 47 | elapsed_mins = int(elapsed_time / 60) 48 | elapsed_secs = int(elapsed_time - (elapsed_mins * 60)) 49 | return elapsed_mins, elapsed_secs 50 | 51 | 52 | def _calc_metrics(pred_labels, true_labels, fname, home_path): 53 | pred_labels = np.array(pred_labels).astype(int) 54 | true_labels = np.array(true_labels).astype(int) 55 | 56 | # save targets 57 | #labels_save_path = os.path.join(log_dir, "labels") 58 | #os.makedirs(labels_save_path, exist_ok=True) 59 | #np.save(os.path.join(labels_save_path, "predicted_labels.npy"), pred_labels) 60 | #np.save(os.path.join(labels_save_path, "true_labels.npy"), true_labels) 61 | 62 | r = classification_report(true_labels, pred_labels, digits=6, output_dict=True) 63 | cm = confusion_matrix(true_labels, pred_labels) 64 | df = pd.DataFrame(r) 65 | df["cohen"] = cohen_kappa_score(true_labels, pred_labels) 66 | df["accuracy"] = accuracy_score(true_labels, pred_labels) 67 | df = df * 100 68 | 69 | # save classification report 70 | report_Save_path = os.path.join(home_path, f'{fname}.xlsx') 71 | df.to_excel(report_Save_path) 72 | 73 | # save confusion matrix 74 | #cm_file_name = f"{exp_name}_{training_mode}_confusion_matrix.torch" 75 | #cm_Save_path = os.path.join(home_path, log_dir, cm_file_name) 76 | #torch.save(cm, cm_Save_path) 77 | 78 | 79 | def _logger(logger_name, level=logging.DEBUG): 80 | """ 81 | Method to return a custom logger with the given name and level 82 | """ 83 | logger = logging.getLogger(logger_name) 84 | logger.setLevel(level) 85 | # format_string = ("%(asctime)s — %(name)s — %(levelname)s — %(funcName)s:" 86 | # "%(lineno)d — %(message)s") 87 | format_string = "%(message)s" 88 | log_format = logging.Formatter(format_string) 89 | # Creating and adding the console handler 90 | console_handler = logging.StreamHandler(sys.stdout) 91 | console_handler.setFormatter(log_format) 92 | logger.addHandler(console_handler) 93 | # Creating and adding the file handler 94 | file_handler = logging.FileHandler(logger_name, mode='a') 95 | file_handler.setFormatter(log_format) 96 | logger.addHandler(file_handler) 97 | return logger 98 | 99 | 100 | from shutil import copy 101 | 102 | 103 | def copy_Files(destination, data_type): 104 | destination_dir = os.path.join(destination, "model_files") 105 | os.makedirs(destination_dir, exist_ok=True) 106 | #copy("main.py", os.path.join(destination_dir, "main.py")) 107 | # copy("args.py", os.path.join(destination_dir, "args.py")) 108 | copy("trainer/trainer.py", os.path.join(destination_dir, "trainer.py")) 109 | copy(f"config_files/{data_type}_Configs.py", os.path.join(destination_dir, f"{data_type}_Configs.py")) 110 | copy("dataloader/augmentations.py", os.path.join(destination_dir, "augmentations.py")) 111 | copy("dataloader/dataloader.py", os.path.join(destination_dir, "dataloader.py")) 112 | copy(f"models/model.py", os.path.join(destination_dir, f"model.py")) 113 | copy("models/loss.py", os.path.join(destination_dir, "loss.py")) 114 | copy("models/TC.py", os.path.join(destination_dir, "TC.py")) 115 | -------------------------------------------------------------------------------- /softclt_catcc/main_pretrain_TL.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | warnings.filterwarnings("ignore") 3 | warnings.filterwarnings("ignore", category=UserWarning) 4 | import argparse 5 | import os 6 | from datetime import datetime 7 | 8 | import numpy as np 9 | import torch 10 | 11 | from dataloader.dataloader import * 12 | from models.TC import TC 13 | from models.model import base_Model 14 | from trainer.trainer import * 15 | from utils import _logger 16 | 17 | start_time = datetime.now() 18 | 19 | ######################## Model parameters ######################## 20 | parser = argparse.ArgumentParser() 21 | home_dir = os.getcwd() 22 | parser.add_argument('--seed', default=1, type=int, help='seed value') 23 | parser.add_argument('--selected_dataset', default='SleepEEG', type=str) 24 | parser.add_argument('--data_path', default=r'data/', type=str, help='Path containing dataset') 25 | parser.add_argument('--logs_save_dir', default='experiments_logs', type=str, help='saving directory') 26 | parser.add_argument('--device', default='7', type=str, help='cpu or cuda') 27 | parser.add_argument('--home_path', default=home_dir, type=str, help='Project home directory') 28 | 29 | parser.add_argument('--lambda_', default=0.5, type=float) 30 | parser.add_argument('--lambda_aux', default=0.5, type=float) 31 | parser.add_argument('--alpha', type=float, default=0.5) 32 | 33 | parser.add_argument('--tau_temp', type=float, default=1) 34 | parser.add_argument('--tau_inst', type=float, default=1) 35 | 36 | parser.add_argument('--num_epochs', type=int, default=40) 37 | parser.add_argument('--save_epoch', type=int, default=10) 38 | 39 | parser.add_argument('--dist', type=str, default='cos') 40 | ############################################################################################################################################ 41 | 42 | args = parser.parse_args() 43 | assert args.selected_dataset == 'SleepEEG' 44 | args.soft_instance = (args.tau_inst > 0) 45 | args.soft_temporal = (args.tau_temp > 0) 46 | 47 | device = torch.device(f'cuda:{args.device}') 48 | data_type = args.selected_dataset 49 | training_mode = 'self_supervised' 50 | 51 | logs_save_dir = args.logs_save_dir 52 | os.makedirs(logs_save_dir, exist_ok=True) 53 | 54 | exec(f'from config_files.{data_type}_Configs import Config as Configs') 55 | configs = Configs() 56 | 57 | # ##### fix random seeds for reproducibility ######## 58 | SEED = args.seed 59 | torch.manual_seed(SEED) 60 | torch.backends.cudnn.deterministic = False 61 | torch.backends.cudnn.benchmark = False 62 | np.random.seed(SEED) 63 | ##################################################### 64 | 65 | settings = f"INST{int(args.soft_instance)}_TEMP{int(args.soft_temporal)}_" 66 | settings += f"inst{args.tau_inst}_temp{args.tau_temp}_" 67 | settings += f"lambda{args.lambda_}_lambda_aux{args.lambda_aux}_{args.dist}" 68 | 69 | log_dir = os.path.join(logs_save_dir, args.selected_dataset, 70 | settings, training_mode + f"_seed_{SEED}") 71 | os.makedirs(os.path.join(log_dir,'saved_models'), exist_ok=True) 72 | print(log_dir) 73 | 74 | counter = 0 75 | src_counter = 0 76 | 77 | log_file_name = os.path.join(log_dir, f"logs_{datetime.now().strftime('%d_%m_%Y_%H_%M_%S')}.log") 78 | logger = _logger(log_file_name) 79 | logger.debug("=" * 45) 80 | logger.debug(f'Dataset: {data_type}') 81 | logger.debug(f'Mode: {training_mode}') 82 | logger.debug("=" * 45) 83 | 84 | # Load datasets 85 | data_path = os.path.join(args.data_path, data_type) 86 | label_pct = 100 87 | train_dl, test_dl = data_generator_wo_val(data_path, configs, training_mode, 88 | label_pct, configs.batch_size) 89 | 90 | logger.debug("Data loaded ...") 91 | 92 | # Load Model 93 | model = base_Model(configs, args).to(device) 94 | temporal_contr_model = TC(configs, device).to(device) 95 | 96 | model_optimizer = torch.optim.Adam(model.parameters(), lr=configs.lr, betas=(configs.beta1, configs.beta2), 97 | weight_decay=3e-4) 98 | 99 | temporal_contr_optimizer = torch.optim.Adam(temporal_contr_model.parameters(), lr=configs.lr, 100 | betas=(configs.beta1, configs.beta2), weight_decay=3e-4) 101 | 102 | Trainer_wo_DTW_wo_val(args, model, temporal_contr_model, model_optimizer, 103 | temporal_contr_optimizer, train_dl, test_dl, device, 104 | logger, configs, log_dir, training_mode) 105 | 106 | logger.debug(f"Training time is : {datetime.now() - start_time}") 107 | 108 | -------------------------------------------------------------------------------- /softclt_ts2vec/tasks/classification.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | from . import _eval_protocols as eval_protocols 4 | from sklearn.preprocessing import label_binarize 5 | from sklearn.metrics import average_precision_score 6 | from sklearn.metrics import f1_score 7 | 8 | def eval_classification(model, train_data, train_labels, test_data, test_labels, eval_protocol='linear'): 9 | assert train_labels.ndim == 1 or train_labels.ndim == 2 10 | train_repr = model.encode(train_data, encoding_window='full_series' if train_labels.ndim == 1 else None) 11 | test_repr = model.encode(test_data, encoding_window='full_series' if train_labels.ndim == 1 else None) 12 | 13 | if eval_protocol == 'linear': 14 | fit_clf = eval_protocols.fit_lr 15 | elif eval_protocol == 'svm': 16 | fit_clf = eval_protocols.fit_svm 17 | elif eval_protocol == 'knn': 18 | fit_clf = eval_protocols.fit_knn 19 | else: 20 | assert False, 'unknown evaluation protocol' 21 | 22 | def merge_dim01(array): 23 | return array.reshape(array.shape[0]*array.shape[1], *array.shape[2:]) 24 | 25 | if train_labels.ndim == 2: 26 | train_repr = merge_dim01(train_repr) 27 | train_labels = merge_dim01(train_labels) 28 | test_repr = merge_dim01(test_repr) 29 | test_labels = merge_dim01(test_labels) 30 | 31 | clf = fit_clf(train_repr, train_labels) 32 | 33 | acc = clf.score(test_repr, test_labels) 34 | if eval_protocol == 'linear': 35 | y_score = clf.predict_proba(test_repr) 36 | else: 37 | y_score = clf.decision_function(test_repr) 38 | test_labels_onehot = label_binarize(test_labels, classes=np.arange(train_labels.max()+1)) 39 | auprc = average_precision_score(test_labels_onehot, y_score) 40 | 41 | return y_score, { 'acc': acc, 'auprc': auprc } 42 | 43 | def eval_semi_classification(model, 44 | train_data1, train_labels1, 45 | train_data5, train_labels5, 46 | train_labels, 47 | test_data, test_labels, eval_protocol='linear'): 48 | assert train_labels.ndim == 1 or train_labels.ndim == 2 49 | 50 | def merge_dim01(array): 51 | return array.reshape(array.shape[0]*array.shape[1], *array.shape[2:]) 52 | 53 | test_repr = model.encode(test_data, encoding_window='full_series' if train_labels.ndim == 1 else None) 54 | test_labels_onehot = label_binarize(test_labels, classes=np.arange(train_labels.max()+1)) 55 | 56 | if train_labels1.ndim == 2: 57 | test_repr = merge_dim01(test_repr) 58 | test_labels = merge_dim01(test_labels) 59 | 60 | train_repr1 = model.encode(train_data1, encoding_window='full_series' if train_labels.ndim == 1 else None) 61 | train_repr5 = model.encode(train_data5, encoding_window='full_series' if train_labels.ndim == 1 else None) 62 | 63 | if eval_protocol == 'linear': 64 | fit_clf = eval_protocols.fit_lr 65 | elif eval_protocol == 'svm': 66 | fit_clf = eval_protocols.fit_svm 67 | elif eval_protocol == 'knn': 68 | fit_clf = eval_protocols.fit_knn 69 | else: 70 | assert False, 'unknown evaluation protocol' 71 | 72 | if train_labels.ndim == 2: 73 | train_repr1 = merge_dim01(train_repr1) 74 | train_labels1 = merge_dim01(train_labels1) 75 | train_repr5 = merge_dim01(train_repr5) 76 | train_labels5 = merge_dim01(train_labels5) 77 | 78 | clf1 = fit_clf(train_repr1, train_labels1) 79 | clf5 = fit_clf(train_repr5, train_labels5) 80 | 81 | acc1 = clf1.score(test_repr, test_labels) 82 | acc5 = clf5.score(test_repr, test_labels) 83 | 84 | if eval_protocol == 'linear': 85 | y_score1 = clf1.predict_proba(test_repr) 86 | y_score5 = clf5.predict_proba(test_repr) 87 | else: 88 | y_score1 = clf1.decision_function(test_repr) 89 | y_score5 = clf5.decision_function(test_repr) 90 | 91 | y_pred1 = clf1.predict(test_repr) 92 | y_pred5 = clf5.predict(test_repr) 93 | 94 | if y_score1.ndim==1: 95 | y_score1 = y_score1.reshape(-1, 1) 96 | if y_score5.ndim==1: 97 | y_score5 = y_score5.reshape(-1, 1) 98 | 99 | auprc1 = average_precision_score(test_labels_onehot, y_score1) 100 | auprc5 = average_precision_score(test_labels_onehot, y_score5) 101 | 102 | 103 | f1_1 = f1_score(test_labels, y_pred1, average='macro') 104 | f1_5 = f1_score(test_labels, y_pred5, average='macro') 105 | 106 | 107 | return [y_score1,y_score5], { 'acc': [acc1,acc5], 'auprc': [auprc1,auprc5], 108 | 'f1': [f1_1,f1_5]} -------------------------------------------------------------------------------- /softclt_ts2vec/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import pickle 4 | import torch 5 | import random 6 | from datetime import datetime 7 | from torch.utils.data import Dataset 8 | 9 | def pkl_save(name, var): 10 | with open(name, 'wb') as f: 11 | pickle.dump(var, f) 12 | 13 | def pkl_load(name): 14 | with open(name, 'rb') as f: 15 | return pickle.load(f) 16 | 17 | def torch_pad_nan(arr, left=0, right=0, dim=0): 18 | if left > 0: 19 | padshape = list(arr.shape) 20 | padshape[dim] = left 21 | arr = torch.cat((torch.full(padshape, np.nan), arr), dim=dim) 22 | if right > 0: 23 | padshape = list(arr.shape) 24 | padshape[dim] = right 25 | arr = torch.cat((arr, torch.full(padshape, np.nan)), dim=dim) 26 | return arr 27 | 28 | def pad_nan_to_target(array, target_length, axis=0, both_side=False): 29 | assert array.dtype in [np.float16, np.float32, np.float64] 30 | pad_size = target_length - array.shape[axis] 31 | if pad_size <= 0: 32 | return array 33 | npad = [(0, 0)] * array.ndim 34 | if both_side: 35 | npad[axis] = (pad_size // 2, pad_size - pad_size//2) 36 | else: 37 | npad[axis] = (0, pad_size) 38 | return np.pad(array, pad_width=npad, mode='constant', constant_values=np.nan) 39 | 40 | def split_with_nan(x, sections, axis=0): 41 | assert x.dtype in [np.float16, np.float32, np.float64] 42 | arrs = np.array_split(x, sections, axis=axis) 43 | target_length = arrs[0].shape[axis] 44 | for i in range(len(arrs)): 45 | arrs[i] = pad_nan_to_target(arrs[i], target_length, axis=axis) 46 | return arrs 47 | 48 | def take_per_row(A, indx, num_elem): 49 | all_indx = indx[:,None] + np.arange(num_elem) 50 | return A[torch.arange(all_indx.shape[0])[:,None], all_indx] 51 | 52 | def centerize_vary_length_series(x): 53 | prefix_zeros = np.argmax(~np.isnan(x).all(axis=-1), axis=1) 54 | suffix_zeros = np.argmax(~np.isnan(x[:, ::-1]).all(axis=-1), axis=1) 55 | offset = (prefix_zeros + suffix_zeros) // 2 - prefix_zeros 56 | rows, column_indices = np.ogrid[:x.shape[0], :x.shape[1]] 57 | offset[offset < 0] += x.shape[1] 58 | column_indices = column_indices - offset[:, np.newaxis] 59 | return x[rows, column_indices] 60 | 61 | def data_dropout(arr, p): 62 | B, T = arr.shape[0], arr.shape[1] 63 | mask = np.full(B*T, False, dtype=np.bool) 64 | ele_sel = np.random.choice( 65 | B*T, 66 | size=int(B*T*p), 67 | replace=False 68 | ) 69 | mask[ele_sel] = True 70 | res = arr.copy() 71 | res[mask.reshape(B, T)] = np.nan 72 | return res 73 | 74 | def name_with_datetime(prefix='default'): 75 | now = datetime.now() 76 | return prefix + '_' + now.strftime("%Y%m%d_%H%M%S") 77 | 78 | def init_dl_program( 79 | device_name, 80 | seed=None, 81 | use_cudnn=True, 82 | deterministic=False, 83 | benchmark=False, 84 | use_tf32=False, 85 | max_threads=None 86 | ): 87 | import torch 88 | if max_threads is not None: 89 | torch.set_num_threads(max_threads) # intraop 90 | if torch.get_num_interop_threads() != max_threads: 91 | torch.set_num_interop_threads(max_threads) # interop 92 | try: 93 | import mkl 94 | except: 95 | pass 96 | else: 97 | mkl.set_num_threads(max_threads) 98 | 99 | if seed is not None: 100 | random.seed(seed) 101 | seed += 1 102 | np.random.seed(seed) 103 | seed += 1 104 | torch.manual_seed(seed) 105 | 106 | if isinstance(device_name, (str, int)): 107 | device_name = [device_name] 108 | 109 | devices = [] 110 | for t in reversed(device_name): 111 | t_device = torch.device(t) 112 | devices.append(t_device) 113 | if t_device.type == 'cuda': 114 | assert torch.cuda.is_available() 115 | torch.cuda.set_device(t_device) 116 | if seed is not None: 117 | seed += 1 118 | torch.cuda.manual_seed(seed) 119 | devices.reverse() 120 | torch.backends.cudnn.enabled = use_cudnn 121 | torch.backends.cudnn.deterministic = deterministic 122 | torch.backends.cudnn.benchmark = benchmark 123 | 124 | if hasattr(torch.backends.cudnn, 'allow_tf32'): 125 | torch.backends.cudnn.allow_tf32 = use_tf32 126 | torch.backends.cuda.matmul.allow_tf32 = use_tf32 127 | 128 | return devices if len(devices) > 1 else devices[0] 129 | 130 | class custom_dataset(Dataset): 131 | def __init__(self, X): 132 | self.X = X 133 | 134 | def __len__(self): 135 | return len(self.X) 136 | 137 | def __getitem__(self, idx): 138 | X = torch.FloatTensor(self.X[idx]) 139 | return X, idx 140 | 141 | -------------------------------------------------------------------------------- /softclt_ts2vec/models/encoder.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | import numpy as np 5 | from .dilated_conv import DilatedConvEncoder, DilatedConvEncoder_all_repr 6 | 7 | def generate_continuous_mask(B, T, n=5, l=0.1): 8 | res = torch.full((B, T), True, dtype=torch.bool) 9 | if isinstance(n, float): 10 | n = int(n * T) 11 | n = max(min(n, T // 2), 1) 12 | 13 | if isinstance(l, float): 14 | l = int(l * T) 15 | l = max(l, 1) 16 | 17 | for i in range(B): 18 | for _ in range(n): 19 | t = np.random.randint(T-l+1) 20 | res[i, t:t+l] = False 21 | return res 22 | 23 | def generate_binomial_mask(B, T, p=0.5): 24 | return torch.from_numpy(np.random.binomial(1, p, size=(B, T))).to(torch.bool) 25 | 26 | class TSEncoder(nn.Module): 27 | def __init__(self, input_dims, output_dims, hidden_dims=64, depth=10, mask_mode='binomial'): 28 | super().__init__() 29 | self.input_dims = input_dims 30 | self.output_dims = output_dims 31 | self.hidden_dims = hidden_dims 32 | self.mask_mode = mask_mode 33 | self.input_fc = nn.Linear(input_dims, hidden_dims) 34 | self.feature_extractor = DilatedConvEncoder( 35 | hidden_dims, 36 | [hidden_dims] * depth + [output_dims], 37 | kernel_size=3 38 | ) 39 | self.repr_dropout = nn.Dropout(p=0.1) 40 | 41 | def forward(self, x, mask=None): # x: B x T x input_dims 42 | nan_mask = ~x.isnan().any(axis=-1) 43 | x[~nan_mask] = 0 44 | x = self.input_fc(x) # B x T x Ch 45 | 46 | # generate & apply mask 47 | if mask is None: 48 | if self.training: 49 | mask = self.mask_mode 50 | else: 51 | mask = 'all_true' 52 | 53 | if mask == 'binomial': 54 | mask = generate_binomial_mask(x.size(0), x.size(1)).to(x.device) 55 | elif mask == 'continuous': 56 | mask = generate_continuous_mask(x.size(0), x.size(1)).to(x.device) 57 | elif mask == 'all_true': 58 | mask = x.new_full((x.size(0), x.size(1)), True, dtype=torch.bool) 59 | elif mask == 'all_false': 60 | mask = x.new_full((x.size(0), x.size(1)), False, dtype=torch.bool) 61 | elif mask == 'mask_last': 62 | mask = x.new_full((x.size(0), x.size(1)), True, dtype=torch.bool) 63 | mask[:, -1] = False 64 | 65 | mask &= nan_mask 66 | x[~mask] = 0 67 | 68 | # conv encoder 69 | x = x.transpose(1, 2) # B x Ch x T 70 | x = self.repr_dropout(self.feature_extractor(x)) # B x Co x T 71 | x = x.transpose(1, 2) # B x T x Co 72 | 73 | return x 74 | 75 | 76 | class TSEncoder_all_repr(nn.Module): 77 | def __init__(self, input_dims, output_dims, hidden_dims=64, depth=10, mask_mode='binomial'): 78 | super().__init__() 79 | self.input_dims = input_dims 80 | self.output_dims = output_dims 81 | self.hidden_dims = hidden_dims 82 | self.mask_mode = mask_mode 83 | self.input_fc = nn.Linear(input_dims, hidden_dims) 84 | self.feature_extractor = DilatedConvEncoder_all_repr( 85 | hidden_dims, 86 | [hidden_dims] * depth + [output_dims], 87 | kernel_size=3 88 | ) 89 | self.repr_dropout = nn.Dropout(p=0.1) 90 | 91 | def forward(self, x, mask=None): # x: B x T x input_dims 92 | nan_mask = ~x.isnan().any(axis=-1) 93 | x[~nan_mask] = 0 94 | x = self.input_fc(x) # B x T x Ch 95 | 96 | # generate & apply mask 97 | if mask is None: 98 | if self.training: 99 | mask = self.mask_mode 100 | else: 101 | mask = 'all_true' 102 | 103 | if mask == 'binomial': 104 | mask = generate_binomial_mask(x.size(0), x.size(1)).to(x.device) 105 | elif mask == 'continuous': 106 | mask = generate_continuous_mask(x.size(0), x.size(1)).to(x.device) 107 | elif mask == 'all_true': 108 | mask = x.new_full((x.size(0), x.size(1)), True, dtype=torch.bool) 109 | elif mask == 'all_false': 110 | mask = x.new_full((x.size(0), x.size(1)), False, dtype=torch.bool) 111 | elif mask == 'mask_last': 112 | mask = x.new_full((x.size(0), x.size(1)), True, dtype=torch.bool) 113 | mask[:, -1] = False 114 | 115 | mask &= nan_mask 116 | x[~mask] = 0 117 | 118 | # conv encoder 119 | x = x.transpose(1, 2) # B x Ch x T 120 | x,outs = self.feature_extractor(x) 121 | #print(x.shape) 122 | #for o in outs: 123 | # print(o.shape) 124 | #print('haha') 125 | #fadsfads 126 | x = self.repr_dropout(x) # B x Co x T 127 | x = x.transpose(1, 2) # B x T x Co 128 | 129 | return x, outs -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Soft Contrastive Learning for Time Series 2 | 3 | ### Seunghan Lee, Taeyoung Park, Kibok Lee 4 | 5 |
6 | 7 | This repository contains the official implementation for the paper [Soft Contrastive Learning for Time Series]([https://arxiv.org/abs/xxxxx](https://arxiv.org/abs/2312.16424)) 8 | 9 | This work is accepted in 10 | 11 | - [ICLR 2024](https://openreview.net/forum?id=pAsQSWlDUf) 12 | - [NeurIPS 2023 Workshop: Self-Supervised Learning - Theory and Practice](https://sslneurips23.github.io/). 13 | 14 |
15 | 16 | # 1. SoftCLT + TS2Vec 17 | 18 | ```bash 19 | chdir softclt_ts2vec 20 | ``` 21 | 22 | Please refer to https://github.com/yuezhihan/ts2vec for 23 | 24 | - (1) Requirements 25 | - (2) Dataset preparation 26 | 27 |
28 | 29 | ## Code Example 30 | 31 | ### 1) Standard Classification 32 | 33 | **UCR 128 datasets** for univariate TS classification 34 | 35 | ```python 36 | bs = 8 37 | data = 'BeetleFly' 38 | # tau_inst = xx 39 | # tau_temp = xx 40 | 41 | !python train.py {data} --loader='UCR' --batch-size {bs} --eval \ 42 | --tau_inst {tau_inst} --tau_temp {tau_temp} 43 | ``` 44 | 45 |
46 | 47 | **UEA 30 datasets** for multivariate TS classification 48 | 49 | ```python 50 | bs = 8 51 | data = 'Cricket' 52 | # tau_inst = xx 53 | # tau_temp = xx 54 | 55 | !python train.py {data} --loader='UEA' --batch-size {bs} --eval \ 56 | --tau_inst {tau_inst} --tau_temp {tau_temp} 57 | ``` 58 | 59 |
60 | 61 | For optimal hyperparameter setting for each dataset, please refer to `hyperparameters/cls_hyperparams.csv` 62 | 63 |
64 | 65 | ## 2) Semi-supervised Classification 66 | 67 | ```python 68 | data = 'Epilepsy' 69 | # bs = xx 70 | # tau_inst = xx 71 | # tau_temp = xx 72 | 73 | !python train.py {data} --loader='semi' --batch-size {bs} --eval \ 74 | --tau_inst {tau_inst} --tau_temp {tau_temp} 75 | ``` 76 | 77 |
78 | 79 | For optimal hyperparameter setting for each dataset, please refer to 80 | 81 | - `hyperparameters/semi_cls_1p_hyperparams.csv` 82 | - `hyperparameters/semi_cls_5p_hyperparams.csv` 83 | 84 |
85 | 86 | ## 3) Anomaly Detection 87 | 88 | ( Note that we ***only use temporal CL*** for anomaly detection task ) 89 | 90 | ```python 91 | data = 'yahoo' 92 | # bs = xxx 93 | # tau_temp = xxx 94 | 95 | !python train.py {data} --loader='anomaly' --batch-size {bs} --eval \ 96 | --lambda_=0 --tau_temp={tau_temp} 97 | ``` 98 | 99 |
100 | 101 | For optimal hyperparameter setting for each dataset, please refer to 102 | 103 | - `hyperparameters/ad_hyperparams.csv` 104 | 105 |
106 | 107 | # 2. SoftCLT + TS-TCC/CA-TCC 108 | 109 | ```bash 110 | chdir softclt_catcc 111 | ``` 112 | 113 | Please refer to https://github.com/emadeldeen24/TS-TCC and https://github.com/emadeldeen24/CA-TCC for 114 | 115 | - (1) Requirements 116 | - (2) Dataset preparation 117 | 118 |
119 | 120 | ## 1) Semi-supervised Classification 121 | 122 | ```python 123 | dataset = 'Epilepsy' 124 | # tau_inst = xxx 125 | # tau_temp = xxx 126 | # lambda_ = xxx 127 | # lambda_aux = xxx 128 | 129 | ############################################################# 130 | # TS-TCC : (1)~(2) 131 | # CA-TCC : (1)~(7) 132 | ############################################################# 133 | # (1) Pretrain 134 | !python main_semi_classification.py --selected_dataset {dataset} --training_mode "self_supervised" \ 135 | --tau_temp {tau_temp} --tau_inst {tau_inst} \ 136 | --lambda_ {lambda_} --lambda_aux {lambda_aux 137 | 138 | # (2) Finetune Classifier 139 | !python3 main_semi_classification.py --selected_dataset {dataset} --training_mode "train_linear" \ 140 | --tau_temp {tau_temp} --tau_inst {tau_inst} \ 141 | --lambda_ {lambda_} --lambda_aux {lambda_aux} 142 | 143 | # (3) Finetune Classifier ( with partially labeled datasets ) 144 | # (4) Finetune Encoder ( with partially labeled datasets ) 145 | # (5) Generate Pseudo-labels 146 | # (6) Supervised CL 147 | # (7) Finetune Classifier 148 | labeled_pc = 1 149 | 150 | for mode_ in ['ft_linear','ft','gen_pseudo_labels','SupCon','train_linear_SupCon']: 151 | !python3 main_semi_classification.py --selected_dataset {dataset} --training_mode {mode_} \ 152 | --tau_temp {tau_temp} --tau_inst {tau_inst} \ 153 | --lambda_ {lambda_} --lambda_aux {lambda_aux} \ 154 | --data_perc {labeled_pc} 155 | ``` 156 | 157 |
158 | 159 | ## 2) Transfer Learning 160 | 161 | - Source dataset: SleepEEG 162 | - Target dataset: Epilepsy, FD-B, Gesture, EMG 163 | 164 | ```python 165 | source_data = 'SleepEEG' 166 | target_data = 'Epilepsy' 167 | epoch_pretrain = 40 168 | # tau_inst = xx 169 | # tau_temp = xx 170 | # lambda = xx 171 | # lambda_aux = xx 172 | 173 | !python3 main_pretrain_TL.py --selected_dataset {source_data} \ 174 | --tau_temp {tau_temp} --tau_inst {tau_inst} \ 175 | --num_epochs {epoch_pretrain} \ 176 | --lambda_ {lambda_} --lambda_aux {lambda_aux} 177 | ``` 178 | 179 |
180 | 181 | ```python 182 | tm = 'fine_tune' # 'linear_probing' 183 | finetune_epoch = 50 # 100,200,300,400 184 | 185 | !python3 main_finetune_TL.py --training_mode {tm} \ 186 | --source_dataset {source_data} --target_dataset {target_data} \ 187 | --tau_temp {tau_temp} --tau_inst {tau_inst} \ 188 | --load_epoch {load_epoch} \ 189 | --num_epochs_finetune {ft_epoch}\ 190 | --lambda_ {lambda_} --lambda_aux {lambda_aux} 191 | ``` 192 | 193 |
194 | 195 | # Contact 196 | 197 | If you have any questions, please contact **seunghan9613@yonsei.ac.kr** 198 | 199 |
200 | 201 | # Acknowledgement 202 | 203 | We appreciate the following github repositories for their valuable code base & datasets: 204 | 205 | - https://github.com/yuezhihan/ts2vec 206 | - https://github.com/emadeldeen24/TS-TCC 207 | - https://github.com/emadeldeen24/CA-TCC 208 | -------------------------------------------------------------------------------- /softclt_ts2vec/tasks/_eval_protocols.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn.linear_model import Ridge 3 | from sklearn.svm import SVC 4 | from sklearn.linear_model import LogisticRegression 5 | from sklearn.neighbors import KNeighborsClassifier 6 | from sklearn.preprocessing import StandardScaler 7 | from sklearn.pipeline import make_pipeline 8 | from sklearn.model_selection import GridSearchCV, train_test_split 9 | 10 | def fit_svm(features, y, MAX_SAMPLES=10000): 11 | nb_classes = np.unique(y, return_counts=True)[1].shape[0] 12 | train_size = features.shape[0] 13 | 14 | svm = SVC(C=10000, gamma='scale') 15 | if train_size // nb_classes < 5 or train_size < 50: 16 | return svm.fit(features, y) 17 | else: 18 | grid_search = GridSearchCV( 19 | svm, { 20 | 'C': [ 21 | 0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000 22 | ], 23 | 'kernel': ['rbf'], 24 | 'degree': [3], 25 | 'gamma': ['scale'], 26 | 'coef0': [0], 27 | 'shrinking': [True], 28 | 'probability': [False], 29 | 'tol': [0.001], 30 | 'cache_size': [200], 31 | 'class_weight': [None], 32 | 'verbose': [False], 33 | 'max_iter': [10000000], 34 | 'decision_function_shape': ['ovr'], 35 | 'random_state': [None] 36 | }, 37 | cv=5, n_jobs=5 38 | ) 39 | # If the training set is too large, subsample MAX_SAMPLES examples 40 | if train_size > MAX_SAMPLES: 41 | split = train_test_split( 42 | features, y, 43 | train_size=MAX_SAMPLES, random_state=0, stratify=y, 44 | ) 45 | features = split[0] 46 | y = split[2] 47 | 48 | grid_search.fit(features, y) 49 | return grid_search.best_estimator_ 50 | 51 | def fit_lr(features, y, MAX_SAMPLES=100000): 52 | # If the training set is too large, subsample MAX_SAMPLES examples 53 | if features.shape[0] > MAX_SAMPLES: 54 | split = train_test_split( 55 | features, y, 56 | train_size=MAX_SAMPLES, random_state=0, stratify=y, 57 | ) 58 | features = split[0] 59 | y = split[2] 60 | 61 | pipe = make_pipeline( 62 | StandardScaler(), 63 | LogisticRegression( 64 | random_state=0, 65 | max_iter=1000000, 66 | multi_class='ovr' 67 | ) 68 | ) 69 | pipe.fit(features, y) 70 | return pipe 71 | 72 | def fit_knn(features, y): 73 | pipe = make_pipeline( 74 | StandardScaler(), 75 | KNeighborsClassifier(n_neighbors=1) 76 | ) 77 | pipe.fit(features, y) 78 | return pipe 79 | 80 | def fit_ridge(train_features, train_y, valid_features, valid_y, MAX_SAMPLES=100000): 81 | # If the training set is too large, subsample MAX_SAMPLES examples 82 | if train_features.shape[0] > MAX_SAMPLES: 83 | split = train_test_split( 84 | train_features, train_y, 85 | train_size=MAX_SAMPLES, random_state=0 86 | ) 87 | train_features = split[0] 88 | train_y = split[2] 89 | if valid_features.shape[0] > MAX_SAMPLES: 90 | split = train_test_split( 91 | valid_features, valid_y, 92 | train_size=MAX_SAMPLES, random_state=0 93 | ) 94 | valid_features = split[0] 95 | valid_y = split[2] 96 | 97 | alphas = [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000] 98 | valid_results = [] 99 | for alpha in alphas: 100 | lr = Ridge(alpha=alpha).fit(train_features, train_y) 101 | valid_pred = lr.predict(valid_features) 102 | score = np.sqrt(((valid_pred - valid_y) ** 2).mean()) + np.abs(valid_pred - valid_y).mean() 103 | valid_results.append(score) 104 | best_alpha = alphas[np.argmin(valid_results)] 105 | 106 | lr = Ridge(alpha=best_alpha) 107 | lr.fit(train_features, train_y) 108 | return lr 109 | 110 | def fit_ridge_norm(train_features, train_y, train_y_last, valid_features, valid_y, valid_y_last, MAX_SAMPLES=100000): 111 | # If the training set is too large, subsample MAX_SAMPLES examples 112 | print('train_y',train_y.shape) 113 | print('train_y_last',train_y_last.shape) 114 | 115 | if train_y.shape[1]!=train_y_last.shape[1]: 116 | k = train_y.shape[1]//train_y_last.shape[1] 117 | train_y_last = np.repeat(train_y_last,k, axis=1) 118 | valid_y_last = np.repeat(valid_y_last,k, axis=1) 119 | 120 | if train_features.shape[0] > MAX_SAMPLES: 121 | split = train_test_split( 122 | train_features, train_y, 123 | train_size=MAX_SAMPLES, random_state=0 124 | ) 125 | train_features = split[0] 126 | train_y = split[2] 127 | if valid_features.shape[0] > MAX_SAMPLES: 128 | split = train_test_split( 129 | valid_features, valid_y, 130 | train_size=MAX_SAMPLES, random_state=0 131 | ) 132 | valid_features = split[0] 133 | valid_y = split[2] 134 | 135 | alphas = [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000] 136 | valid_results = [] 137 | for alpha in alphas: 138 | lr = Ridge(alpha=alpha).fit(train_features, train_y-train_y_last) 139 | valid_pred = lr.predict(valid_features) 140 | valid_pred = valid_pred +valid_y_last 141 | score = np.sqrt(((valid_pred - valid_y) ** 2).mean()) + np.abs(valid_pred - valid_y).mean() 142 | valid_results.append(score) 143 | best_alpha = alphas[np.argmin(valid_results)] 144 | 145 | lr = Ridge(alpha=best_alpha) 146 | lr.fit(train_features, train_y-train_y_last) 147 | return lr 148 | -------------------------------------------------------------------------------- /softclt_ts2vec/utils_distance_matrix.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import pandas as pd 4 | from scipy.spatial.distance import euclidean 5 | # from fastdtw import fastdtw 6 | from tslearn.metrics import dtw, dtw_path,gak 7 | import tqdm 8 | 9 | from sklearn.preprocessing import MinMaxScaler 10 | from sklearn.metrics.pairwise import cosine_similarity, euclidean_distances 11 | 12 | def find(condition): 13 | res, = np.nonzero(np.ravel(condition)) 14 | return res 15 | 16 | def tam(path, report='full'): 17 | # Delay and advance counting 18 | delay = len(find(np.diff(path[0]) == 0)) 19 | advance = len(find(np.diff(path[1]) == 0)) 20 | 21 | # Phase counting 22 | incumbent = find((np.diff(path[0]) == 1) * (np.diff(path[1]) == 1)) 23 | phase = len(incumbent) 24 | 25 | # Estimated and reference time series duration. 26 | len_estimation = path[1][-1] 27 | len_ref = path[0][-1] 28 | 29 | p_advance = advance * 1. / len_ref 30 | p_delay = delay * 1. / len_estimation 31 | p_phase = phase * 1. / np.min([len_ref, len_estimation]) 32 | 33 | return p_advance + p_delay + (1 - p_phase) 34 | 35 | def get_DTW(UTS_tr): 36 | N = len(UTS_tr) 37 | dist_mat = np.zeros((N,N)) 38 | for i in tqdm.tqdm(range(N)): 39 | for j in range(N): 40 | if i>j: 41 | dist = dtw(UTS_tr[i].reshape(-1,1), UTS_tr[j].reshape(-1,1)) 42 | dist_mat[i,j] = dist 43 | dist_mat[j,i] = dist 44 | elif i==j: 45 | dist_mat[i,j] = 0 46 | else : 47 | pass 48 | return dist_mat 49 | 50 | def get_TAM(UTS_tr): 51 | N = len(UTS_tr) 52 | dist_mat = np.zeros((N,N)) 53 | for i in tqdm.tqdm(range(N)): 54 | for j in range(N): 55 | if i>j: 56 | k = dtw_path(UTS_tr[i].reshape(-1,1), 57 | UTS_tr[j].reshape(-1,1))[0] 58 | p = [np.array([i[0] for i in k]), 59 | np.array([i[1] for i in k])] 60 | dist = tam(p) 61 | dist_mat[i,j] = dist 62 | dist_mat[j,i] = dist 63 | elif i==j: 64 | dist_mat[i,j] = 0 65 | else : 66 | pass 67 | return dist_mat 68 | 69 | def get_GAK(UTS_tr): 70 | N = len(UTS_tr) 71 | dist_mat = np.zeros((N,N)) 72 | for i in tqdm.tqdm(range(N)): 73 | for j in range(N): 74 | if i>j: 75 | dist = gak(UTS_tr[i].reshape(-1,1), 76 | UTS_tr[j].reshape(-1,1)) 77 | dist_mat[i,j] = dist 78 | dist_mat[j,i] = dist 79 | elif i==j: 80 | dist_mat[i,j] = 0 81 | else : 82 | pass 83 | return dist_mat 84 | 85 | 86 | def get_MDTW(MTS_tr): 87 | N = MTS_tr.shape[0] 88 | dist_mat = np.zeros((N,N)) 89 | for i in tqdm.tqdm(range(N)): 90 | for j in range(N): 91 | if i>j: 92 | mdtw_dist = dtw(MTS_tr[i], MTS_tr[j]) 93 | dist_mat[i,j] = mdtw_dist 94 | dist_mat[j,i] = mdtw_dist 95 | elif i==j: 96 | dist_mat[i,j] = 0 97 | else : 98 | pass 99 | return dist_mat 100 | 101 | def get_COS(MTS_tr): 102 | cos_sim_matrix = -cosine_similarity(MTS_tr) 103 | return cos_sim_matrix 104 | 105 | def get_EUC(MTS_tr): 106 | return euclidean_distances(MTS_tr) 107 | 108 | def save_sim_mat(X_tr, min_ = 0, max_ = 1, multivariate=False, type_='DTW'): 109 | if multivariate: 110 | assert type=='DTW' 111 | dist_mat = get_MDTW(X_tr) 112 | else: 113 | if type_=='DTW': 114 | dist_mat = get_DTW(X_tr) 115 | elif type_=='TAM': 116 | dist_mat = get_TAM(X_tr) 117 | elif type_=='COS': 118 | dist_mat = get_COS(X_tr) 119 | elif type_=='EUC': 120 | dist_mat = get_EUC(X_tr) 121 | elif type_=='GAK': 122 | dist_mat = get_GAK(X_tr) 123 | N = dist_mat.shape[0] 124 | 125 | # (1) distance matrix 126 | diag_indices = np.diag_indices(N) 127 | mask = np.ones(dist_mat.shape, dtype=bool) 128 | mask[diag_indices] = False 129 | temp = dist_mat[mask].reshape(N, N-1) 130 | dist_mat[diag_indices] = temp.min() 131 | 132 | # (2) normalized distance matrix 133 | scaler = MinMaxScaler(feature_range=(min_, max_)) 134 | dist_mat = scaler.fit_transform(dist_mat) 135 | 136 | # (3) normalized similarity matrix 137 | return 1 - dist_mat 138 | 139 | def get_example_data(data_name): 140 | ex_data_path = f'./data/UCR/{data_name}/{data_name}_TRAIN.tsv' 141 | data = pd.read_csv(ex_data_path, delimiter='\t', keep_default_na=False, header=None) 142 | data_X = data.iloc[:,1:] 143 | data_y = data.iloc[:,0] 144 | return data_X,data_y 145 | 146 | def set_nan_to_zero(a): 147 | where_are_NaNs = np.isnan(a) 148 | a[where_are_NaNs] = 0 149 | return a 150 | 151 | def densify(x, tau, alpha): 152 | return ((2*alpha) / (1 + np.exp(-tau*x))) + (1-alpha)*np.eye(x.shape[0]) 153 | 154 | def topK_one_else_zero(matrix, k): 155 | """ 156 | Convert the off-diagonal elements to one if they are in the top-k values of each row, and zero if not 157 | """ 158 | new_matrix = matrix.copy() 159 | top_k_indices = np.argpartition(new_matrix, -(k+1), axis=1)[:, -(k+1):] 160 | np.fill_diagonal(new_matrix, 0) 161 | 162 | mask = np.zeros_like(new_matrix) 163 | mask[np.repeat(np.arange(new_matrix.shape[0]), (k+1)), 164 | top_k_indices.flatten()] = 1 165 | return mask 166 | 167 | def convert_hard_matrix(soft_matrix, pos_ratio): 168 | N = soft_matrix.shape[0] 169 | num_pos = int((N-1) * pos_ratio) 170 | hard_matrix = topK_one_else_zero(soft_matrix, num_pos) 171 | return hard_matrix -------------------------------------------------------------------------------- /softclt_catcc/models/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | 4 | from models.timelags import * 5 | from models.soft_losses import * 6 | from models.hard_losses import * 7 | 8 | class base_Model(nn.Module): 9 | def __init__(self, configs, args): 10 | super(base_Model, self).__init__() 11 | 12 | self.conv_block1 = nn.Sequential( 13 | nn.Conv1d(configs.input_channels, 32, kernel_size=configs.kernel_size, 14 | stride=configs.stride, bias=False, padding=(configs.kernel_size//2)), 15 | nn.BatchNorm1d(32), 16 | nn.ReLU(), 17 | nn.MaxPool1d(kernel_size=2, stride=2, padding=1), 18 | nn.Dropout(configs.dropout) 19 | ) 20 | 21 | self.conv_block2 = nn.Sequential( 22 | nn.Conv1d(32, 64, kernel_size=8, stride=1, bias=False, padding=4), 23 | nn.BatchNorm1d(64), 24 | nn.ReLU(), 25 | nn.MaxPool1d(kernel_size=2, stride=2, padding=1) 26 | ) 27 | 28 | self.conv_block3 = nn.Sequential( 29 | nn.Conv1d(64, configs.final_out_channels, kernel_size=8, stride=1, bias=False, padding=4), 30 | nn.BatchNorm1d(configs.final_out_channels), 31 | nn.ReLU(), 32 | nn.MaxPool1d(kernel_size=2, stride=2, padding=1), 33 | ) 34 | 35 | model_output_dim = configs.features_len 36 | self.logits = nn.Linear(model_output_dim * configs.final_out_channels, configs.num_classes) 37 | self.lambda_ = args.lambda_ # 0.5 38 | self.soft_temporal = args.soft_temporal #True 39 | self.soft_instance = args.soft_instance #True 40 | 41 | self.tau_temp = args.tau_temp 42 | 43 | 44 | def forward(self, aug1, aug2, soft_labels, train=True): 45 | if train: 46 | if self.soft_instance: 47 | soft_labels_L, soft_labels_R = dup_matrix(soft_labels) 48 | del soft_labels 49 | 50 | temporal_loss = torch.tensor(0., device=aug1.device) 51 | instance_loss = torch.tensor(0., device=aug1.device) 52 | 53 | #-------------------------------------------------# 54 | # DEPTH = 1 55 | #-------------------------------------------------# 56 | aug1 = self.conv_block1(aug1) 57 | aug2 = self.conv_block1(aug2) 58 | if self.soft_temporal: 59 | d=0 60 | timelag = timelag_sigmoid(aug1.shape[2], self.tau_temp*(2**d)) 61 | timelag = torch.tensor(timelag, device=aug1.device) 62 | timelag_L, timelag_R = dup_matrix(timelag) 63 | temporal_loss += (1-self.lambda_) * temp_CL_soft(aug1, aug2, timelag_L, timelag_R) 64 | else: 65 | temporal_loss += (1-self.lambda_) * temp_CL_hard(aug1, aug2) 66 | if self.soft_instance: 67 | instance_loss += self.lambda_ * inst_CL_soft(aug1, aug2, soft_labels_L, soft_labels_R) 68 | else: 69 | instance_loss += self.lambda_ * inst_CL_hard(aug1, aug2) 70 | 71 | #-------------------------------------------------# 72 | # DEPTH = 2 73 | #-------------------------------------------------# 74 | aug1 = self.conv_block2(aug1) 75 | aug2 = self.conv_block2(aug2) 76 | 77 | if self.soft_temporal: 78 | d=1 79 | timelag = timelag_sigmoid(aug1.shape[2],self.tau_temp*(2**d)) 80 | timelag = torch.tensor(timelag, device=aug1.device) 81 | timelag_L, timelag_R = dup_matrix(timelag) 82 | temporal_loss += (1-self.lambda_) * temp_CL_soft(aug1, aug2, timelag_L, timelag_R) 83 | else: 84 | temporal_loss += (1-self.lambda_) * temp_CL_hard(aug1, aug2) 85 | 86 | if self.soft_instance: 87 | instance_loss += self.lambda_ * inst_CL_soft(aug1, aug2, soft_labels_L, soft_labels_R) 88 | else: 89 | instance_loss += self.lambda_ * inst_CL_hard(aug1, aug2) 90 | 91 | #-------------------------------------------------# 92 | # DEPTH = 3 93 | #-------------------------------------------------# 94 | aug1 = self.conv_block3(aug1) 95 | aug2 = self.conv_block3(aug2) 96 | 97 | if self.soft_temporal: 98 | d=2 99 | timelag = timelag_sigmoid(aug1.shape[2],self.tau_temp*(2**d)) 100 | timelag = torch.tensor(timelag, device=aug1.device) 101 | timelag_L, timelag_R = dup_matrix(timelag) 102 | temporal_loss += (1-self.lambda_) * temp_CL_soft(aug1, aug2, timelag_L, timelag_R) 103 | else: 104 | temporal_loss += (1-self.lambda_) * temp_CL_hard(aug1, aug2) 105 | 106 | if self.soft_instance: 107 | instance_loss += self.lambda_ * inst_CL_soft(aug1, aug2, soft_labels_L, soft_labels_R) 108 | del soft_labels_L, soft_labels_R 109 | else: 110 | instance_loss += self.lambda_ * inst_CL_hard(aug1, aug2) 111 | 112 | else: 113 | aug = self.conv_block1(aug1) 114 | aug = self.conv_block2(aug) 115 | aug = self.conv_block3(aug) 116 | 117 | ############################################################################ 118 | ############################################################################ 119 | 120 | if train: 121 | aug1_flat = aug1.reshape(aug1.shape[0], -1) 122 | aug2_flat = aug2.reshape(aug2.shape[0], -1) 123 | aug1_logits = self.logits(aug1_flat) 124 | aug2_logits = self.logits(aug2_flat) 125 | final_loss = temporal_loss + instance_loss 126 | return aug1_logits, aug2_logits, aug1, aug2, final_loss 127 | 128 | else: 129 | aug_flat = aug.reshape(aug.shape[0], -1) 130 | aug_logits = self.logits(aug_flat) 131 | return aug_logits, aug 132 | -------------------------------------------------------------------------------- /softclt_catcc/models/loss.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | 5 | class NTXentLoss(torch.nn.Module): 6 | 7 | def __init__(self, device, batch_size, temperature, use_cosine_similarity): 8 | super(NTXentLoss, self).__init__() 9 | self.batch_size = batch_size 10 | self.temperature = temperature 11 | self.device = device 12 | self.softmax = torch.nn.Softmax(dim=-1) 13 | self.mask_samples_from_same_repr = self._get_correlated_mask().type(torch.bool) 14 | self.similarity_function = self._get_similarity_function(use_cosine_similarity) 15 | self.criterion = torch.nn.CrossEntropyLoss(reduction="sum") 16 | 17 | def _get_similarity_function(self, use_cosine_similarity): 18 | if use_cosine_similarity: 19 | self._cosine_similarity = torch.nn.CosineSimilarity(dim=-1) 20 | return self._cosine_simililarity 21 | else: 22 | return self._dot_simililarity 23 | 24 | def _get_correlated_mask(self): 25 | diag = np.eye(2 * self.batch_size) 26 | l1 = np.eye((2 * self.batch_size), 2 * self.batch_size, k=-self.batch_size) 27 | l2 = np.eye((2 * self.batch_size), 2 * self.batch_size, k=self.batch_size) 28 | mask = torch.from_numpy((diag + l1 + l2)) 29 | mask = (1 - mask).type(torch.bool) 30 | return mask.to(self.device) 31 | 32 | @staticmethod 33 | def _dot_simililarity(x, y): 34 | v = torch.tensordot(x.unsqueeze(1), y.T.unsqueeze(0), dims=2) 35 | # x shape: (N, 1, C) 36 | # y shape: (1, C, 2N) 37 | # v shape: (N, 2N) 38 | return v 39 | 40 | def _cosine_simililarity(self, x, y): 41 | # x shape: (N, 1, C) 42 | # y shape: (1, 2N, C) 43 | # v shape: (N, 2N) 44 | v = self._cosine_similarity(x.unsqueeze(1), y.unsqueeze(0)) 45 | return v 46 | 47 | def forward(self, zis, zjs): 48 | representations = torch.cat([zjs, zis], dim=0) 49 | similarity_matrix = self.similarity_function(representations, representations) 50 | # filter out the scores from the positive samples 51 | l_pos = torch.diag(similarity_matrix, self.batch_size) 52 | r_pos = torch.diag(similarity_matrix, -self.batch_size) 53 | positives = torch.cat([l_pos, r_pos]).view(2 * self.batch_size, 1) 54 | 55 | negatives = similarity_matrix[self.mask_samples_from_same_repr].view(2 * self.batch_size, -1) 56 | 57 | logits = torch.cat((positives, negatives), dim=1) 58 | logits /= self.temperature 59 | 60 | labels = torch.zeros(2 * self.batch_size).to(self.device).long() 61 | loss = self.criterion(logits, labels) 62 | 63 | return loss / (2 * self.batch_size) 64 | 65 | 66 | class SupConLoss(torch.nn.Module): 67 | """Supervised Contrastive Learning: https://arxiv.org/pdf/2004.11362.pdf. 68 | It also supports the unsupervised contrastive loss in SimCLR""" 69 | 70 | def __init__(self, device, temperature=0.2, contrast_mode='all'): 71 | super(SupConLoss, self).__init__() 72 | self.temperature = temperature 73 | self.contrast_mode = contrast_mode 74 | self.device = device 75 | 76 | def forward(self, features, labels=None, mask=None): 77 | """Compute loss for model. If both `labels` and `mask` are None, 78 | it degenerates to SimCLR unsupervised loss: 79 | https://arxiv.org/pdf/2002.05709.pdf 80 | Args: 81 | features: hidden vector of shape [bsz, n_views, ...]. 82 | labels: ground truth of shape [bsz]. 83 | mask: contrastive mask of shape [bsz, bsz], mask_{i,j}=1 if sample j 84 | has the same class as sample i. Can be asymmetric. 85 | Returns: 86 | A loss scalar. 87 | """ 88 | device = self.device # 'cuda' #(torch.device('cuda') 89 | # if features.is_cuda 90 | # else torch.device('cpu')) 91 | 92 | if len(features.shape) < 3: 93 | raise ValueError('`features` needs to be [bsz, n_views, ...],' 94 | 'at least 3 dimensions are required') 95 | if len(features.shape) > 3: 96 | features = features.view(features.shape[0], features.shape[1], -1) 97 | 98 | batch_size = features.shape[0] 99 | if labels is not None and mask is not None: 100 | raise ValueError('Cannot define both `labels` and `mask`') 101 | elif labels is None and mask is None: 102 | mask = torch.eye(batch_size, dtype=torch.float32).to(device) 103 | elif labels is not None: 104 | labels = labels.contiguous().view(-1, 1) 105 | if labels.shape[0] != batch_size: 106 | raise ValueError('Num of labels does not match num of features') 107 | mask = torch.eq(labels, labels.T).float().to(device) 108 | else: 109 | mask = mask.float().to(device) 110 | 111 | contrast_count = features.shape[1] 112 | contrast_feature = torch.cat(torch.unbind(features, dim=1), dim=0) 113 | if self.contrast_mode == 'one': 114 | anchor_feature = features[:, 0] 115 | anchor_count = 1 116 | elif self.contrast_mode == 'all': 117 | anchor_feature = contrast_feature 118 | anchor_count = contrast_count 119 | else: 120 | raise ValueError('Unknown mode: {}'.format(self.contrast_mode)) 121 | 122 | # compute logits 123 | anchor_dot_contrast = torch.div( 124 | torch.matmul(anchor_feature, contrast_feature.T), 125 | self.temperature) 126 | # for numerical stability 127 | logits_max, _ = torch.max(anchor_dot_contrast, dim=1, keepdim=True) 128 | logits = anchor_dot_contrast - logits_max.detach() 129 | 130 | # tile mask 131 | mask = mask.repeat(anchor_count, contrast_count) 132 | # mask-out self-contrast cases 133 | logits_mask = torch.scatter( 134 | torch.ones_like(mask), 135 | 1, 136 | torch.arange(batch_size * anchor_count).view(-1, 1).to(device), 137 | 0 138 | ) 139 | mask = mask * logits_mask 140 | 141 | # compute log_prob 142 | exp_logits = torch.exp(logits) * logits_mask 143 | log_prob = logits - torch.log(exp_logits.sum(1, keepdim=True)) 144 | 145 | # compute mean of log-likelihood over positive 146 | mean_log_prob_pos = (mask * log_prob).sum(1) / mask.sum(1) 147 | 148 | # loss 149 | # loss = - (self.temperature / self.base_temperature) * mean_log_prob_pos 150 | loss = - self.temperature * mean_log_prob_pos 151 | loss = loss.view(anchor_count, batch_size).mean() 152 | 153 | return loss 154 | -------------------------------------------------------------------------------- /softclt_ts2vec/tasks/forecasting_separate.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | import time 4 | from . import _eval_protocols as eval_protocols 5 | 6 | def eval_forecasting(model, data, train_slice, valid_slice, test_slice, scaler, pred_lens, n_covariate_cols,univar): 7 | padding = 200 8 | 9 | t = time.time() 10 | 11 | all_repr = model.encode( 12 | data, 13 | casual=True, 14 | sliding_length=1, 15 | sliding_padding=padding, 16 | batch_size=256 17 | ) 18 | ts2vec_infer_time = time.time() - t 19 | train_repr = all_repr[:, train_slice] 20 | valid_repr = all_repr[:, valid_slice] 21 | test_repr = all_repr[:, test_slice] 22 | train_data = data[:, train_slice, n_covariate_cols:] 23 | valid_data = data[:, valid_slice, n_covariate_cols:] 24 | test_data = data[:, test_slice, n_covariate_cols:] 25 | 26 | ours_result = {} 27 | lr_train_time = {} 28 | lr_infer_time = {} 29 | out_log = {} 30 | for pred_len in pred_lens: 31 | print('train_repr',train_repr.shape) 32 | print('train_data',train_data.shape) 33 | D = train_data.shape[2] 34 | for d in range(D): 35 | print(train_data[0,:,d]) 36 | train_features, train_labels = generate_pred_samples(train_repr, train_data, pred_len, drop=padding) 37 | valid_features, valid_labels = generate_pred_samples(valid_repr, valid_data, pred_len) 38 | test_features, test_labels = generate_pred_samples(test_repr, test_data, pred_len) 39 | t = time.time() 40 | print('train_features',train_features.shape) 41 | print('train_labels',train_labels.shape) 42 | print('valid_features',valid_features.shape) 43 | print('valid_labels',valid_labels.shape) 44 | fadsfsd 45 | lr = eval_protocols.fit_ridge(train_features, train_labels, valid_features, valid_labels) 46 | lr_train_time[pred_len] = time.time() - t 47 | t = time.time() 48 | test_pred = lr.predict(test_features) 49 | lr_infer_time[pred_len] = time.time() - t 50 | ori_shape = test_data.shape[0], -1, pred_len, test_data.shape[2] 51 | test_pred = test_pred.reshape(ori_shape) 52 | test_labels = test_labels.reshape(ori_shape) 53 | 54 | if test_data.shape[0] > 1: 55 | if univar: 56 | temp_2d = test_pred.reshape(test_pred.shape[0],-1) 57 | temp_2d = scaler.inverse_transform(temp_2d.T) 58 | temp_2d = temp_2d.T 59 | test_pred_inv = temp_2d.reshape(test_pred.shape) 60 | 61 | temp_2d = test_labels.reshape(test_labels.shape[0],-1) 62 | temp_2d = scaler.inverse_transform(temp_2d.T) 63 | temp_2d = temp_2d.T 64 | test_labels_inv = temp_2d.reshape(test_labels.shape) 65 | 66 | else: 67 | a,b,c,d = test_pred.shape 68 | test_pred_reshaped = np.transpose(test_pred, (0, 2, 1, 3)) # Transpose the array to shape (a, c, b, d) 69 | test_pred_reshaped = np.reshape(test_pred_reshaped, (a, b*c, d)) # Reshape the array to shape (a, b*c, d) 70 | test_pred_reshaped = test_pred_reshaped.reshape((a*b*c, d)) 71 | test_pred_reshaped = scaler.inverse_transform(test_pred_reshaped) 72 | test_pred_reshaped = test_pred_reshaped.reshape((a, b*c, d)) 73 | test_pred_inv = test_pred_reshaped.reshape(a,b,c,d) 74 | 75 | a,b,c,d = test_labels.shape 76 | test_labels_reshaped = np.transpose(test_labels, (0, 2, 1, 3)) # Transpose the array to shape (a, c, b, d) 77 | test_labels_reshaped = np.reshape(test_labels_reshaped, (a, b*c, d)) # Reshape the array to shape (a, b*c, d) 78 | test_labels_reshaped = test_labels_reshaped.reshape((a*b*c, d)) 79 | test_labels_reshaped = scaler.inverse_transform(test_labels_reshaped) 80 | test_labels_reshaped = test_labels_reshaped.reshape((a, b*c, d)) 81 | test_labels_inv = test_labels_reshaped.reshape(a,b,c,d) 82 | #test_pred_inv = scaler.inverse_transform(test_pred.swapaxes(0, 3)).swapaxes(0, 3) 83 | #test_labels_inv = scaler.inverse_transform(test_labels.swapaxes(0, 3)).swapaxes(0, 3) 84 | else: 85 | if univar: 86 | temp_2d = test_pred.reshape(test_pred.shape[0],-1) 87 | temp_2d = scaler.inverse_transform(temp_2d) 88 | test_pred_inv = temp_2d.reshape(test_pred.shape) 89 | temp_2d = test_labels.reshape(test_labels.shape[0],-1) 90 | temp_2d = scaler.inverse_transform(temp_2d) 91 | test_labels_inv = temp_2d.reshape(test_labels.shape) 92 | else: 93 | a,b,c,d = test_pred.shape 94 | test_pred_reshaped = np.transpose(test_pred, (0, 2, 1, 3)) # Transpose the array to shape (a, c, b, d) 95 | test_pred_reshaped = np.reshape(test_pred_reshaped, (a, b*c, d)) # Reshape the array to shape (a, b*c, d) 96 | test_pred_reshaped = test_pred_reshaped.reshape((a*b*c, d)) 97 | test_pred_reshaped = scaler.inverse_transform(test_pred_reshaped) 98 | test_pred_reshaped = test_pred_reshaped.reshape((a, b*c, d)) 99 | test_pred_inv = test_pred_reshaped.reshape(a,b,c,d) 100 | 101 | a,b,c,d = test_labels.shape 102 | test_labels_reshaped = np.transpose(test_labels, (0, 2, 1, 3)) # Transpose the array to shape (a, c, b, d) 103 | test_labels_reshaped = np.reshape(test_labels_reshaped, (a, b*c, d)) # Reshape the array to shape (a, b*c, d) 104 | test_labels_reshaped = test_labels_reshaped.reshape((a*b*c, d)) 105 | test_labels_reshaped = scaler.inverse_transform(test_labels_reshaped) 106 | test_labels_reshaped = test_labels_reshaped.reshape((a, b*c, d)) 107 | test_labels_inv = test_labels_reshaped.reshape(a,b,c,d) 108 | 109 | #test_pred_inv = scaler.inverse_transform(test_pred) 110 | #test_labels_inv = scaler.inverse_transform(test_labels) 111 | 112 | out_log[pred_len] = { 113 | 'norm': test_pred, 114 | 'raw': test_pred_inv, 115 | 'norm_gt': test_labels, 116 | 'raw_gt': test_labels_inv 117 | } 118 | ours_result[pred_len] = { 119 | 'norm': cal_metrics(test_pred, test_labels), 120 | 'raw': cal_metrics(test_pred_inv, test_labels_inv) 121 | } 122 | 123 | eval_res = { 124 | 'ours': ours_result, 125 | 'ts2vec_infer_time': ts2vec_infer_time, 126 | 'lr_train_time': lr_train_time, 127 | 'lr_infer_time': lr_infer_time 128 | } 129 | return out_log, eval_res 130 | -------------------------------------------------------------------------------- /softclt_catcc/main_finetune_TL.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | from datetime import datetime 4 | 5 | import numpy as np 6 | import torch 7 | 8 | from dataloader.dataloader import * 9 | from models.TC import TC 10 | from models.model import base_Model 11 | from trainer.trainer import * 12 | from utils import _calc_metrics 13 | from utils import _logger, set_requires_grad 14 | 15 | start_time = datetime.now() 16 | 17 | 18 | ######################## Model parameters ######################## 19 | parser = argparse.ArgumentParser() 20 | home_dir = os.getcwd() 21 | parser.add_argument('--seed', default=1, type=int, help='seed value') 22 | parser.add_argument('--training_mode', default='self_supervised', type=str, 23 | help='Modes of choice: random_init, supervised, self_supervised, SupCon, ft_linear, gen_pseudo_labels') 24 | 25 | parser.add_argument('--source_dataset', default='SleepEEG', type=str, help='Datadset of choice: EEG, HAR, Epilepsy, pFD') 26 | parser.add_argument('--target_dataset', default='HAR', type=str, help='Dataset of choice: EEG, HAR, Epilepsy, pFD') 27 | parser.add_argument('--data_path', default=r'data/', type=str, help='Path containing dataset') 28 | parser.add_argument('--data_perc', default=1, type=int, help='data percentage') 29 | parser.add_argument('--logs_save_dir', default='experiments_logs', type=str, help='saving directory') 30 | parser.add_argument('--device', default='7', type=str, help='cpu or cuda') 31 | parser.add_argument('--home_path', default=home_dir, type=str, help='Project home directory') 32 | 33 | ############################################################################################################################################ 34 | parser.add_argument('--lambda_', default=0.5, type=float) 35 | parser.add_argument('--lambda_aux', type=float, default=0.5) 36 | parser.add_argument('--alpha', type=float, default=0.5) 37 | 38 | parser.add_argument('--tau_temp', type=float, default=1) 39 | parser.add_argument('--tau_inst', type=float, default=1) 40 | 41 | parser.add_argument('--load_epoch', type=int, default=40) 42 | parser.add_argument('--save_epoch', type=int, default=20) 43 | parser.add_argument('--num_epochs_finetune', type=int, default=20) 44 | 45 | parser.add_argument('--batch_size', type=int, default=999) 46 | parser.add_argument('--dist', type=str, default='cos') 47 | ############################################################################################################################################ 48 | 49 | args = parser.parse_args() 50 | args.soft_instance = (args.tau_inst > 0) 51 | args.soft_temporal = (args.tau_temp > 0) 52 | source = args.source_dataset 53 | target = args.target_dataset 54 | 55 | device = torch.device(f'cuda:{args.device}') 56 | assert args.training_mode in ['linear_probing','fine_tune'] 57 | 58 | data_path = os.path.join(args.data_path, target) 59 | training_mode = args.training_mode 60 | 61 | logs_save_dir = args.logs_save_dir 62 | os.makedirs(logs_save_dir, exist_ok=True) 63 | 64 | exec(f'from config_files.{source}_Configs import Config as Configs') 65 | exec(f'from config_files.{target}_Configs import Config as Configs_target') 66 | configs = Configs() 67 | configs_target = Configs_target() 68 | 69 | # ##### fix random seeds for reproducibility ######## 70 | SEED = args.seed 71 | torch.manual_seed(SEED) 72 | torch.backends.cudnn.deterministic = False 73 | torch.backends.cudnn.benchmark = False 74 | np.random.seed(SEED) 75 | ##################################################### 76 | 77 | settings = f"INST{int(args.soft_instance)}_TEMP{int(args.soft_temporal)}_" 78 | settings += f"inst{args.tau_inst}_temp{args.tau_temp}_" 79 | settings += f"lambda{args.lambda_}_lambda_aux{args.lambda_aux}_{args.dist}" 80 | 81 | base = os.path.join(logs_save_dir, f'{source}2{target}', settings) 82 | log_dir = os.path.join(base, training_mode + f"_seed_{SEED}") 83 | 84 | os.makedirs(os.path.join(log_dir, 'saved_models'), exist_ok=True) 85 | print('='*50) 86 | print(log_dir) 87 | print('='*50) 88 | 89 | counter = 0 90 | src_counter = 0 91 | 92 | log_file_name = os.path.join(log_dir, f"logs_{datetime.now().strftime('%d_%m_%Y_%H_%M_%S')}.log") 93 | logger = _logger(log_file_name) 94 | logger.debug("=" * 45) 95 | logger.debug(f'PRETRAIN Dataset: {source}') 96 | logger.debug(f'FINETUNE Dataset: {target}') 97 | logger.debug(f'Mode: {training_mode}') 98 | logger.debug("=" * 45) 99 | 100 | 101 | # Load datasets 102 | train_dl, test_dl = data_generator_wo_val(data_path, configs_target, 103 | training_mode, int(args.data_perc), args.batch_size) 104 | logger.debug("Data loaded ...") 105 | 106 | # Load Model 107 | model = base_Model(configs_target, args).to(device) 108 | temporal_contr_model = TC(configs_target, device).to(device) 109 | 110 | pretrained_weight_path = os.path.join(args.logs_save_dir, args.source_dataset, settings, 111 | f"self_supervised_seed_{SEED}","saved_models", f"ckp_{args.load_epoch}.pt") 112 | pre_W = torch.load(pretrained_weight_path, map_location=device)["model_state_dict"] 113 | model_W = model.state_dict() 114 | 115 | for k in model_W.keys(): 116 | if (model_W[k].shape != pre_W[k].shape) & ('logit' not in k): 117 | _, C_target, _ = model_W[k].shape # 3 118 | _, C_source, _ = pre_W[k].shape # 1 119 | dup = C_target // C_source 120 | pre_W[k] = pre_W[k].repeat(1,dup,1) 121 | 122 | pre_W_copy = pre_W.copy() 123 | for i in pre_W_copy.keys(): 124 | if 'logits' in i: 125 | del pre_W[i] 126 | 127 | model_W.update(pre_W) 128 | model.load_state_dict(model_W) 129 | 130 | if training_mode == "linear_probing" : 131 | set_requires_grad(model, pre_W, requires_grad=False) # Freeze everything except last layer. 132 | 133 | model_optimizer = torch.optim.Adam(model.parameters(), lr=configs.lr, 134 | betas=(configs.beta1, configs.beta2), 135 | weight_decay=3e-4) 136 | 137 | temporal_contr_optimizer = torch.optim.Adam(temporal_contr_model.parameters(), 138 | lr=configs.lr, 139 | betas=(configs.beta1, configs.beta2), 140 | weight_decay=3e-4) 141 | 142 | Trainer_wo_DTW_wo_val(args, model, temporal_contr_model, model_optimizer, temporal_contr_optimizer, train_dl, test_dl, device, 143 | logger, configs, log_dir, training_mode) 144 | 145 | outs = model_evaluate(model, temporal_contr_model, test_dl, device, training_mode) 146 | total_loss, total_acc, pred_labels, true_labels = outs 147 | save_setting = f'load_{args.load_epoch}_ft_{args.num_epochs_finetune}_{args.training_mode}' 148 | _calc_metrics(pred_labels, true_labels, 149 | os.path.join(log_dir,save_setting), args.home_path) 150 | 151 | logger.debug(f"Training time is : {datetime.now() - start_time}") 152 | 153 | 154 | 155 | -------------------------------------------------------------------------------- /softclt_ts2vec/tasks/anomaly_detection.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import time 3 | from sklearn.metrics import f1_score, precision_score, recall_score 4 | import bottleneck as bn 5 | 6 | # consider delay threshold and missing segments 7 | def get_range_proba(predict, label, delay=7): 8 | splits = np.where(label[1:] != label[:-1])[0] + 1 9 | is_anomaly = label[0] == 1 10 | new_predict = np.array(predict) 11 | pos = 0 12 | 13 | for sp in splits: 14 | if is_anomaly: 15 | if 1 in predict[pos:min(pos + delay + 1, sp)]: 16 | new_predict[pos: sp] = 1 17 | else: 18 | new_predict[pos: sp] = 0 19 | is_anomaly = not is_anomaly 20 | pos = sp 21 | sp = len(label) 22 | 23 | if is_anomaly: # anomaly in the end 24 | if 1 in predict[pos: min(pos + delay + 1, sp)]: 25 | new_predict[pos: sp] = 1 26 | else: 27 | new_predict[pos: sp] = 0 28 | 29 | return new_predict 30 | 31 | 32 | # set missing = 0 33 | def reconstruct_label(timestamp, label): 34 | timestamp = np.asarray(timestamp, np.int64) 35 | index = np.argsort(timestamp) 36 | 37 | timestamp_sorted = np.asarray(timestamp[index]) 38 | interval = np.min(np.diff(timestamp_sorted)) 39 | 40 | label = np.asarray(label, np.int64) 41 | label = np.asarray(label[index]) 42 | 43 | idx = (timestamp_sorted - timestamp_sorted[0]) // interval 44 | 45 | new_label = np.zeros(shape=((timestamp_sorted[-1] - timestamp_sorted[0]) // interval + 1,), dtype=np.int) 46 | new_label[idx] = label 47 | 48 | return new_label 49 | 50 | 51 | def eval_ad_result(test_pred_list, test_labels_list, test_timestamps_list, delay): 52 | labels = [] 53 | pred = [] 54 | for test_pred, test_labels, test_timestamps in zip(test_pred_list, test_labels_list, test_timestamps_list): 55 | assert test_pred.shape == test_labels.shape == test_timestamps.shape 56 | test_labels = reconstruct_label(test_timestamps, test_labels) 57 | test_pred = reconstruct_label(test_timestamps, test_pred) 58 | test_pred = get_range_proba(test_pred, test_labels, delay) 59 | labels.append(test_labels) 60 | pred.append(test_pred) 61 | labels = np.concatenate(labels) 62 | pred = np.concatenate(pred) 63 | return { 64 | 'f1': f1_score(labels, pred), 65 | 'precision': precision_score(labels, pred), 66 | 'recall': recall_score(labels, pred) 67 | } 68 | 69 | 70 | def np_shift(arr, num, fill_value=np.nan): 71 | result = np.empty_like(arr) 72 | if num > 0: 73 | result[:num] = fill_value 74 | result[num:] = arr[:-num] 75 | elif num < 0: 76 | result[num:] = fill_value 77 | result[:num] = arr[-num:] 78 | else: 79 | result[:] = arr 80 | return result 81 | 82 | 83 | def eval_anomaly_detection(model, all_train_data, all_train_labels, all_train_timestamps, all_test_data, all_test_labels, all_test_timestamps, delay): 84 | t = time.time() 85 | 86 | all_train_repr = {} 87 | all_test_repr = {} 88 | all_train_repr_wom = {} 89 | all_test_repr_wom = {} 90 | for k in all_train_data: 91 | train_data = all_train_data[k] 92 | test_data = all_test_data[k] 93 | 94 | full_repr = model.encode( 95 | np.concatenate([train_data, test_data]).reshape(1, -1, 1), 96 | mask='mask_last', 97 | casual=True, 98 | sliding_length=1, 99 | sliding_padding=200, 100 | batch_size=256 101 | ).squeeze() 102 | all_train_repr[k] = full_repr[:len(train_data)] 103 | all_test_repr[k] = full_repr[len(train_data):] 104 | 105 | full_repr_wom = model.encode( 106 | np.concatenate([train_data, test_data]).reshape(1, -1, 1), 107 | casual=True, 108 | sliding_length=1, 109 | sliding_padding=200, 110 | batch_size=256 111 | ).squeeze() 112 | all_train_repr_wom[k] = full_repr_wom[:len(train_data)] 113 | all_test_repr_wom[k] = full_repr_wom[len(train_data):] 114 | 115 | res_log = [] 116 | labels_log = [] 117 | timestamps_log = [] 118 | for k in all_train_data: 119 | train_data = all_train_data[k] 120 | train_labels = all_train_labels[k] 121 | train_timestamps = all_train_timestamps[k] 122 | 123 | test_data = all_test_data[k] 124 | test_labels = all_test_labels[k] 125 | test_timestamps = all_test_timestamps[k] 126 | 127 | train_err = np.abs(all_train_repr_wom[k] - all_train_repr[k]).sum(axis=1) 128 | test_err = np.abs(all_test_repr_wom[k] - all_test_repr[k]).sum(axis=1) 129 | 130 | ma = np_shift(bn.move_mean(np.concatenate([train_err, test_err]), 21), 1) 131 | train_err_adj = (train_err - ma[:len(train_err)]) / ma[:len(train_err)] 132 | test_err_adj = (test_err - ma[len(train_err):]) / ma[len(train_err):] 133 | train_err_adj = train_err_adj[22:] 134 | 135 | thr = np.mean(train_err_adj) + 4 * np.std(train_err_adj) 136 | test_res = (test_err_adj > thr) * 1 137 | 138 | for i in range(len(test_res)): 139 | if i >= delay and test_res[i-delay:i].sum() >= 1: 140 | test_res[i] = 0 141 | 142 | res_log.append(test_res) 143 | labels_log.append(test_labels) 144 | timestamps_log.append(test_timestamps) 145 | t = time.time() - t 146 | 147 | eval_res = eval_ad_result(res_log, labels_log, timestamps_log, delay) 148 | eval_res['infer_time'] = t 149 | return res_log, eval_res 150 | 151 | 152 | def eval_anomaly_detection_coldstart(model, all_train_data, all_train_labels, all_train_timestamps, all_test_data, all_test_labels, all_test_timestamps, delay): 153 | t = time.time() 154 | 155 | all_data = {} 156 | all_repr = {} 157 | all_repr_wom = {} 158 | for k in all_train_data: 159 | all_data[k] = np.concatenate([all_train_data[k], all_test_data[k]]) 160 | all_repr[k] = model.encode( 161 | all_data[k].reshape(1, -1, 1), 162 | mask='mask_last', 163 | casual=True, 164 | sliding_length=1, 165 | sliding_padding=200, 166 | batch_size=256 167 | ).squeeze() 168 | all_repr_wom[k] = model.encode( 169 | all_data[k].reshape(1, -1, 1), 170 | casual=True, 171 | sliding_length=1, 172 | sliding_padding=200, 173 | batch_size=256 174 | ).squeeze() 175 | 176 | res_log = [] 177 | labels_log = [] 178 | timestamps_log = [] 179 | for k in all_data: 180 | data = all_data[k] 181 | labels = np.concatenate([all_train_labels[k], all_test_labels[k]]) 182 | timestamps = np.concatenate([all_train_timestamps[k], all_test_timestamps[k]]) 183 | 184 | err = np.abs(all_repr_wom[k] - all_repr[k]).sum(axis=1) 185 | ma = np_shift(bn.move_mean(err, 21), 1) 186 | err_adj = (err - ma) / ma 187 | 188 | MIN_WINDOW = len(data) // 10 189 | thr = bn.move_mean(err_adj, len(err_adj), MIN_WINDOW) + 4 * bn.move_std(err_adj, len(err_adj), MIN_WINDOW) 190 | res = (err_adj > thr) * 1 191 | 192 | for i in range(len(res)): 193 | if i >= delay and res[i-delay:i].sum() >= 1: 194 | res[i] = 0 195 | 196 | res_log.append(res[MIN_WINDOW:]) 197 | labels_log.append(labels[MIN_WINDOW:]) 198 | timestamps_log.append(timestamps[MIN_WINDOW:]) 199 | t = time.time() - t 200 | 201 | eval_res = eval_ad_result(res_log, labels_log, timestamps_log, delay) 202 | eval_res['infer_time'] = t 203 | return res_log, eval_res 204 | 205 | -------------------------------------------------------------------------------- /softclt_catcc/trainer/train_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | sys.path.append("..") 5 | import numpy as np 6 | 7 | import torch 8 | import torch.nn as nn 9 | import torch.nn.functional as F 10 | 11 | from models.loss import NTXentLoss, SupConLoss 12 | 13 | # (densify) 14 | # 1) model_train 15 | # 2) model_train_wo_DTW 16 | # 3) model_evaluate 17 | # 4) gen_pseudo_labels 18 | 19 | def densify(x, tau, alpha=0.5): 20 | return ((2*alpha) / (1 + np.exp(-tau*x))) + (1-alpha)*np.eye(x.shape[0]) 21 | 22 | def model_train(soft_labels, model, temporal_contr_model, model_optimizer, temp_cont_optimizer, criterion, train_loader, config, 23 | device, training_mode, lambda_aux): 24 | total_loss = [] 25 | total_acc = [] 26 | model.train() 27 | temporal_contr_model.train() 28 | soft_labels = torch.tensor(soft_labels, device=device) 29 | 30 | for _, (idx, data, labels, aug1, aug2) in enumerate(train_loader): 31 | data, labels = data.float().to(device), labels.long().to(device) 32 | aug1, aug2 = aug1.float().to(device), aug2.float().to(device) 33 | aug1 = aug1*100 34 | aug2 = aug2*100 35 | 36 | model_optimizer.zero_grad() 37 | temp_cont_optimizer.zero_grad() 38 | 39 | if training_mode == "self_supervised" or training_mode == "SupCon": 40 | 41 | soft_labels_batch = soft_labels[idx][:,idx] 42 | 43 | _, _, features1, features2, final_loss = model(aug1, aug2, soft_labels_batch) 44 | del soft_labels_batch 45 | 46 | features1 = F.normalize(features1, dim=1) 47 | features2 = F.normalize(features2, dim=1) 48 | 49 | temp_cont_loss1, temp_cont_feat1 = temporal_contr_model(features1, features2) 50 | temp_cont_loss2, temp_cont_feat2 = temporal_contr_model(features2, features1) 51 | 52 | 53 | if training_mode == "self_supervised": 54 | lambda1 = 1 55 | lambda2 = 0.7 56 | nt_xent_criterion = NTXentLoss(device, config.batch_size, config.Context_Cont.temperature, 57 | config.Context_Cont.use_cosine_similarity) 58 | loss = (temp_cont_loss1 + temp_cont_loss2) * lambda1 + \ 59 | nt_xent_criterion(temp_cont_feat1, temp_cont_feat2) * lambda2 60 | 61 | 62 | elif training_mode == "SupCon": 63 | lambda1 = 0.01 64 | lambda2 = 0.1 65 | Sup_contrastive_criterion = SupConLoss(device) 66 | 67 | supCon_features = torch.cat([temp_cont_feat1.unsqueeze(1), temp_cont_feat2.unsqueeze(1)], dim=1) 68 | loss = (temp_cont_loss1 + temp_cont_loss2) * lambda1 + Sup_contrastive_criterion(supCon_features, 69 | labels) * lambda2 70 | 71 | else: 72 | output = model(data, 0, 0, train=False) 73 | predictions, _ = output 74 | loss = criterion(predictions, labels) 75 | total_acc.append(labels.eq(predictions.detach().argmax(dim=1)).float().mean()) 76 | 77 | if (training_mode == "self_supervised") : 78 | loss += lambda_aux*final_loss 79 | 80 | if training_mode == 'SupCon': 81 | loss += 0.1*lambda_aux*final_loss 82 | 83 | 84 | total_loss.append(loss.item()) 85 | loss.backward() 86 | model_optimizer.step() 87 | temp_cont_optimizer.step() 88 | 89 | total_loss = torch.tensor(total_loss).mean() 90 | 91 | if (training_mode == "self_supervised") or (training_mode == "SupCon"): 92 | total_acc = 0 93 | else: 94 | total_acc = torch.tensor(total_acc).mean() 95 | return total_loss, total_acc 96 | 97 | 98 | def model_train_wo_DTW(dist_func, dist_type, tau_inst, model, temporal_contr_model, 99 | model_optimizer, temp_cont_optimizer, criterion, train_loader, 100 | config, device, training_mode, lambda_aux): 101 | total_loss = [] 102 | total_acc = [] 103 | model.train() 104 | temporal_contr_model.train() 105 | 106 | for _, (_, data, labels, aug1, aug2) in enumerate(train_loader): 107 | data, labels = data.float().to(device), labels.long().to(device) 108 | aug1, aug2 = aug1.float().to(device), aug2.float().to(device) 109 | aug1 = aug1*100 110 | aug2 = aug2*100 111 | 112 | model_optimizer.zero_grad() 113 | temp_cont_optimizer.zero_grad() 114 | 115 | if training_mode == "self_supervised" or training_mode == "SupCon": 116 | temp = data.view(data.shape[0], -1).detach().cpu().numpy() 117 | dist_mat_batch = dist_func(temp) 118 | if dist_type=='euc': 119 | dist_mat_batch = (dist_mat_batch - np.min(dist_mat_batch)) / (np.max(dist_mat_batch) - np.min(dist_mat_batch)) 120 | dist_mat_batch = - dist_mat_batch 121 | dist_mat_batch = densify(dist_mat_batch, tau_inst, alpha=0.5) 122 | dist_mat_batch = torch.tensor(dist_mat_batch, device=device) 123 | _, _, features1, features2, final_loss = model(aug1, aug2, dist_mat_batch) 124 | del dist_mat_batch 125 | 126 | features1 = F.normalize(features1, dim=1) 127 | features2 = F.normalize(features2, dim=1) 128 | 129 | temp_cont_loss1, temp_cont_feat1 = temporal_contr_model(features1, features2) 130 | temp_cont_loss2, temp_cont_feat2 = temporal_contr_model(features2, features1) 131 | 132 | 133 | if training_mode == "self_supervised": 134 | lambda1 = 1 135 | lambda2 = 0.7 136 | nt_xent_criterion = NTXentLoss(device, config.batch_size, config.Context_Cont.temperature, 137 | config.Context_Cont.use_cosine_similarity) 138 | loss = (temp_cont_loss1 + temp_cont_loss2) * lambda1 + \ 139 | nt_xent_criterion(temp_cont_feat1, temp_cont_feat2) * lambda2 140 | 141 | 142 | elif training_mode == "SupCon": 143 | lambda1 = 0.01 144 | lambda2 = 0.1 145 | Sup_contrastive_criterion = SupConLoss(device) 146 | 147 | supCon_features = torch.cat([temp_cont_feat1.unsqueeze(1), temp_cont_feat2.unsqueeze(1)], dim=1) 148 | loss = (temp_cont_loss1 + temp_cont_loss2) * lambda1 + Sup_contrastive_criterion(supCon_features, 149 | labels) * lambda2 150 | 151 | else: 152 | output = model(data, 0, 0, train=False) 153 | predictions, _ = output 154 | loss = criterion(predictions, labels) 155 | total_acc.append(labels.eq(predictions.detach().argmax(dim=1)).float().mean()) 156 | 157 | if (training_mode == "self_supervised") : 158 | loss += lambda_aux*final_loss 159 | 160 | if training_mode == 'SupCon': 161 | loss += 0.1*lambda_aux*final_loss 162 | 163 | 164 | total_loss.append(loss.item()) 165 | loss.backward() 166 | model_optimizer.step() 167 | temp_cont_optimizer.step() 168 | 169 | total_loss = torch.tensor(total_loss).mean() 170 | 171 | if (training_mode == "self_supervised") or (training_mode == "SupCon"): 172 | total_acc = 0 173 | else: 174 | total_acc = torch.tensor(total_acc).mean() 175 | return total_loss, total_acc 176 | 177 | 178 | def model_evaluate(model, temporal_contr_model, test_dl, device, training_mode): 179 | model.eval() 180 | temporal_contr_model.eval() 181 | 182 | total_loss = [] 183 | total_acc = [] 184 | 185 | criterion = nn.CrossEntropyLoss() 186 | outs = np.array([]) 187 | trgs = np.array([]) 188 | 189 | with torch.no_grad(): 190 | for _, data, labels, _, _ in test_dl: 191 | data, labels = data.float().to(device), labels.long().to(device) 192 | 193 | if (training_mode == "self_supervised") or (training_mode == "SupCon"): 194 | pass 195 | else: 196 | output = model(data, 0, 0, train=False) 197 | 198 | if (training_mode != "self_supervised") and (training_mode != "SupCon"): 199 | predictions, features = output 200 | loss = criterion(predictions, labels) 201 | total_acc.append(labels.eq(predictions.detach().argmax(dim=1)).float().mean()) 202 | total_loss.append(loss.item()) 203 | 204 | pred = predictions.max(1, keepdim=True)[1] # get the index of the max log-probability 205 | outs = np.append(outs, pred.cpu().numpy()) 206 | trgs = np.append(trgs, labels.data.cpu().numpy()) 207 | 208 | if (training_mode == "self_supervised") or (training_mode == "SupCon"): 209 | total_loss = 0 210 | total_acc = 0 211 | return total_loss, total_acc, [], [] 212 | else: 213 | total_loss = torch.tensor(total_loss).mean() # average loss 214 | total_acc = torch.tensor(total_acc).mean() # average acc 215 | return total_loss, total_acc, outs, trgs 216 | 217 | 218 | def gen_pseudo_labels(model, dataloader, device, experiment_log_dir, pc): 219 | model.eval() 220 | softmax = nn.Softmax(dim=1) 221 | 222 | all_pseudo_labels = np.array([]) 223 | all_labels = np.array([]) 224 | all_data = [] 225 | 226 | with torch.no_grad(): 227 | for _, data, labels, _, _ in dataloader: 228 | data = data.float().to(device) 229 | labels = labels.view((-1)).long().to(device) 230 | 231 | output = model(data, 0, 0, train=False) 232 | predictions, features = output 233 | 234 | normalized_preds = softmax(predictions) 235 | pseudo_labels = normalized_preds.max(1, keepdim=True)[1].squeeze() 236 | all_pseudo_labels = np.append(all_pseudo_labels, pseudo_labels.cpu().numpy()) 237 | 238 | all_labels = np.append(all_labels, labels.cpu().numpy()) 239 | all_data.append(data) 240 | 241 | all_data = torch.cat(all_data, dim=0) 242 | 243 | data_save = dict() 244 | data_save["samples"] = all_data 245 | data_save["labels"] = torch.LongTensor(torch.from_numpy(all_pseudo_labels).long()) 246 | file_name = f"pseudo_train_data_{str(pc)}perc.pt" 247 | torch.save(data_save, os.path.join(experiment_log_dir, file_name)) 248 | print("Pseudo labels generated ...") 249 | -------------------------------------------------------------------------------- /softclt_catcc/main_semi_classification.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import sys 4 | from datetime import datetime 5 | 6 | import numpy as np 7 | import torch 8 | 9 | from dataloader.dataloader import * 10 | from models.TC import TC 11 | from models.model import base_Model 12 | from trainer.trainer import * 13 | from utils import _calc_metrics, copy_Files 14 | from utils import _logger, set_requires_grad 15 | 16 | start_time = datetime.now() 17 | 18 | def densify(x, tau, alpha): 19 | return ((2*alpha) / (1 + np.exp(-tau*x))) + (1-alpha)*np.eye(x.shape[0]) 20 | 21 | parser = argparse.ArgumentParser() 22 | 23 | ######################## Model parameters ######################## 24 | home_dir = os.getcwd() 25 | parser.add_argument('--seed', default=1, type=int, help='seed value') 26 | parser.add_argument('--training_mode', default='self_supervised', type=str, 27 | help='Modes of choice: random_init, supervised, self_supervised, SupCon, ft_linear, gen_pseudo_labels') 28 | 29 | parser.add_argument('--selected_dataset', default='HAR', type=str, help='Dataset of choice: EEG, HAR, Epilepsy, pFD') 30 | parser.add_argument('--data_path', default=r'data/', type=str, help='Path containing dataset') 31 | parser.add_argument('--data_perc', default=1, type=int, help='data percentage') 32 | parser.add_argument('--logs_save_dir', default='experiments_logs', type=str, help='saving directory') 33 | parser.add_argument('--device', default='7', type=str, help='cpu or cuda') 34 | parser.add_argument('--home_path', default=home_dir, type=str, help='Project home directory') 35 | 36 | ############################################################################################################################################ 37 | parser.add_argument('--lambda_', default=0.5, type=float) 38 | parser.add_argument('--lambda_aux', type=float, default=0.5) 39 | 40 | parser.add_argument('--tau_temp', type=float, default=1) 41 | parser.add_argument('--tau_inst', type=float, default=6) 42 | parser.add_argument('--alpha', type=float, default=0.5) 43 | 44 | parser.add_argument('--num_epochs', type=int, default=100) 45 | parser.add_argument('--save_epoch', type=int, default=20) 46 | parser.add_argument('--load_epoch', type=int, default=40) 47 | parser.add_argument('--batch_size', type=int, default=999) 48 | 49 | parser.add_argument('--dist_metric', type=str, default='DTW') 50 | 51 | ############################################################################################################################################ 52 | args = parser.parse_args() 53 | 54 | device = torch.device(f'cuda:{args.device}') 55 | #experiment_description = f"{args.selected_dataset}_experiment" 56 | data_type = args.selected_dataset 57 | training_mode = args.training_mode 58 | run_description = args.selected_dataset 59 | 60 | args.soft_temporal = (args.tau_temp > 0) 61 | args.soft_instance = (args.tau_inst > 0) 62 | logs_save_dir = args.logs_save_dir 63 | os.makedirs(logs_save_dir, exist_ok=True) 64 | 65 | exec(f'from config_files.{data_type}_Configs import Config as Configs') 66 | configs = Configs() 67 | 68 | # ##### fix random seeds for reproducibility ######## 69 | SEED = args.seed 70 | torch.manual_seed(SEED) 71 | torch.backends.cudnn.deterministic = False 72 | torch.backends.cudnn.benchmark = False 73 | np.random.seed(SEED) 74 | ##################################################### 75 | 76 | settings = f"TEMP{int(args.soft_temporal)}_INST{int(args.soft_instance)}_" 77 | settings += f'tau_temp{float(args.tau_temp)}_tau_inst{float(args.tau_inst)}_' 78 | settings += f'lambda{args.lambda_}_lambda_aux{args.lambda_aux}' 79 | 80 | #base = os.path.join(logs_save_dir, experiment_description, run_description, settings) 81 | base = os.path.join(logs_save_dir, run_description, settings) 82 | if args.training_mode in ['self_supervised','train_linear']: 83 | experiment_log_dir = os.path.join(base, training_mode + f"_seed_{SEED}") 84 | else: 85 | experiment_log_dir = os.path.join(base, f'{args.data_perc}p', training_mode + f"_seed_{SEED}") 86 | 87 | print('='*50) 88 | print(experiment_log_dir) 89 | print('='*50) 90 | os.makedirs(os.path.join(experiment_log_dir,'saved_models'), exist_ok=True) 91 | 92 | # loop through domains 93 | counter = 0 94 | src_counter = 0 95 | 96 | # Logging 97 | log_file_name = os.path.join(experiment_log_dir, f"logs_{datetime.now().strftime('%d_%m_%Y_%H_%M_%S')}.log") 98 | logger = _logger(log_file_name) 99 | logger.debug("=" * 45) 100 | logger.debug(f'Dataset: {data_type}') 101 | logger.debug(f'Mode: {training_mode}') 102 | logger.debug("=" * 45) 103 | 104 | # Load datasets 105 | data_path = os.path.join(args.data_path, data_type) 106 | if data_type not in ['HAR','sleepEDF','Epilepsy']: 107 | train_dl, test_dl = data_generator_wo_val(data_path, configs, training_mode, int(args.data_perc), args.batch_size) 108 | else: 109 | train_dl, valid_dl, test_dl = data_generator(data_path, configs, training_mode, int(args.data_perc), args.batch_size) 110 | 111 | logger.debug("Data loaded ...") 112 | 113 | # Load Model 114 | model = base_Model(configs, args).to(device) 115 | temporal_contr_model = TC(configs, device).to(device) 116 | 117 | if training_mode == 'ft' : 118 | load_from = os.path.join(base, f'{args.data_perc}p',f"ft_linear_seed_{SEED}") 119 | chkpoint = torch.load(os.path.join(load_from,"saved_models",f"ckp_{args.load_epoch}.pt"), map_location=device) 120 | pretrained_dict = chkpoint["model_state_dict"] 121 | 122 | model_dict = model.state_dict() 123 | del_list = ['logits'] 124 | pretrained_dict_copy = pretrained_dict.copy() 125 | for i in pretrained_dict_copy.keys(): 126 | for j in del_list: 127 | if j in i: 128 | del pretrained_dict[i] 129 | model_dict.update(pretrained_dict) 130 | model.load_state_dict(model_dict) 131 | 132 | if training_mode == 'ft_linear' : 133 | load_from = os.path.join(base, f"self_supervised_seed_{SEED}") 134 | chkpoint = torch.load(os.path.join(load_from, "saved_models", f"ckp_{args.load_epoch}.pt"), map_location=device) 135 | pretrained_dict = chkpoint["model_state_dict"] 136 | model_dict = model.state_dict() 137 | del_list = ['logits'] 138 | pretrained_dict_copy = pretrained_dict.copy() 139 | for i in pretrained_dict_copy.keys(): 140 | for j in del_list: 141 | if j in i: 142 | del pretrained_dict[i] 143 | model_dict.update(pretrained_dict) 144 | model.load_state_dict(model_dict) 145 | set_requires_grad(model, pretrained_dict, requires_grad=False) # Freeze everything except last layer. 146 | 147 | if training_mode == "gen_pseudo_labels": 148 | load_from = os.path.join(base, f"{args.data_perc}p", f'ft_seed_{SEED}') 149 | chkpoint = torch.load(os.path.join(load_from,"saved_models",f"ckp_{args.load_epoch}.pt"), map_location=device) 150 | #chkpoint = torch.load(os.path.join(load_from, "ckp_last.pt"), map_location=device) 151 | pretrained_dict = chkpoint["model_state_dict"] 152 | model.load_state_dict(pretrained_dict) 153 | gen_pseudo_labels(model, train_dl, device, data_path, args.data_perc) 154 | sys.exit(0) 155 | 156 | if "train_linear" in training_mode: 157 | if 'SupCon' in training_mode: 158 | #load_from = os.path.join(base, f"{args.data_perc}p", f"SupCon_seed_{SEED}") 159 | load_from = os.path.join(base, f"{args.data_perc}p", f"SupCon_seed_{SEED}") 160 | else: 161 | #load_from = os.path.join(base, f"self_supervised_seed_{SEED}") 162 | load_from = os.path.join(base, f"self_supervised_seed_{SEED}") 163 | chkpoint = torch.load(os.path.join(load_from,"saved_models",f"ckp_{args.load_epoch}.pt"), map_location=device) 164 | 165 | 166 | 167 | pretrained_dict = chkpoint["model_state_dict"] 168 | model_dict = model.state_dict() 169 | pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict} 170 | del_list = ['logits'] 171 | pretrained_dict_copy = pretrained_dict.copy() 172 | for i in pretrained_dict_copy.keys(): 173 | for j in del_list: 174 | if j in i: 175 | del pretrained_dict[i] 176 | 177 | model_dict.update(pretrained_dict) 178 | model.load_state_dict(model_dict) 179 | set_requires_grad(model, pretrained_dict, requires_grad=False) # Freeze everything except last layer. 180 | 181 | if training_mode == "SupCon": 182 | load_from = os.path.join(base, f"{args.data_perc}p", f'ft_seed_{SEED}') 183 | chkpoint = torch.load(os.path.join(load_from,"saved_models",f"ckp_{args.load_epoch}.pt"), map_location=device) 184 | pretrained_dict = chkpoint["model_state_dict"] 185 | model.load_state_dict(pretrained_dict) 186 | 187 | model_optimizer = torch.optim.Adam(model.parameters(), lr=configs.lr, betas=(configs.beta1, configs.beta2), 188 | weight_decay=3e-4) 189 | 190 | temporal_contr_optimizer = torch.optim.Adam(temporal_contr_model.parameters(), lr=configs.lr, 191 | betas=(configs.beta1, configs.beta2), weight_decay=3e-4) 192 | 193 | if training_mode == "self_supervised" or training_mode == "SupCon": # to do it only once 194 | #copy_Files(os.path.join(logs_save_dir, experiment_description, run_description, settings), data_type) 195 | copy_Files(os.path.join(logs_save_dir, run_description, settings), data_type) 196 | 197 | if args.soft_instance: 198 | MAT_PATH = f'data/{data_type}/{args.dist_metric}.npy' 199 | if os.path.exists(MAT_PATH): 200 | sim_mat = np.load(MAT_PATH) 201 | else: 202 | print(f"Saving {{args.dist_metric}} ...") 203 | sim_mat = load_sim_matrix(args.selected_dataset) 204 | dist_mat = 1-sim_mat 205 | soft_labels = densify(-dist_mat, args.tau_inst, args.alpha) 206 | del sim_mat, dist_mat 207 | else: 208 | soft_labels = 0 209 | 210 | 211 | if data_type not in ['HAR','sleepEDF','Epilepsy']: 212 | Trainer_wo_val(args, soft_labels, model, temporal_contr_model, model_optimizer, 213 | temporal_contr_optimizer, train_dl, test_dl, device, 214 | logger, configs, experiment_log_dir, training_mode) 215 | else: 216 | Trainer(args, soft_labels, model, temporal_contr_model, model_optimizer, 217 | temporal_contr_optimizer, train_dl, valid_dl, test_dl, device, 218 | logger, configs, experiment_log_dir, training_mode) 219 | 220 | if ('linear' in training_mode) | (training_mode == 'ft'): 221 | outs = model_evaluate(model, temporal_contr_model, test_dl, device, training_mode) 222 | total_loss, total_acc, pred_labels, true_labels = outs 223 | _calc_metrics(pred_labels, true_labels, os.path.join(experiment_log_dir), args.home_path, args.num_epochs) 224 | 225 | logger.debug(f"Training time is : {datetime.now() - start_time}") 226 | -------------------------------------------------------------------------------- /softclt_ts2vec/train.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | import argparse 4 | import os 5 | import sys 6 | import time 7 | import random 8 | import datetime 9 | from soft_ts2vec import TS2Vec 10 | import tasks 11 | import datautils 12 | from utils import init_dl_program, pkl_save, data_dropout 13 | 14 | from utils_distance_matrix import * 15 | import sys 16 | 17 | def save_checkpoint_callback( 18 | save_every=1, 19 | unit='epoch'): 20 | assert unit in ('epoch', 'iter') 21 | def callback(model, loss): 22 | n = model.n_epochs if unit == 'epoch' else model.n_iters 23 | if n % save_every == 0: 24 | model.save(f'{run_dir}/model_{n}.pkl') 25 | return callback 26 | 27 | def fix_seed(expid): 28 | SEED = 2000 + expid 29 | random.seed(SEED) 30 | np.random.seed(SEED) 31 | np.random.seed(SEED) 32 | torch.manual_seed(SEED) 33 | torch.cuda.manual_seed(SEED) 34 | torch.cuda.manual_seed_all(SEED) 35 | torch.backends.cudnn.deterministic = True 36 | torch.backends.cudnn.benchmark = False 37 | 38 | if __name__ == '__main__': 39 | parser = argparse.ArgumentParser() 40 | parser.add_argument('dataset', help='The dataset name') 41 | parser.add_argument('--loader', type=str, required=True, help='The data loader used to load the experimental data. This can be set to UCR, UEA, forecast_csv, forecast_csv_univar, anomaly, or anomaly_coldstart') 42 | parser.add_argument('--dist_type', type=str, default='DTW') 43 | parser.add_argument('--gpu', type=int, default=0, help='The gpu no. used for training and inference (defaults to 0)') 44 | parser.add_argument('--batch-size', type=int, default=8, help='The batch size (defaults to 8)') 45 | parser.add_argument('--lr', type=float, default=0.001, help='The learning rate (defaults to 0.001)') 46 | parser.add_argument('--repr-dims', type=int, default=320, help='The representation dimension (defaults to 320)') 47 | parser.add_argument('--max-train-length', type=int, default=3000, help='For sequence with a length greater than , it would be cropped into some sequences, each of which has a length less than (defaults to 3000)') 48 | parser.add_argument('--iters', type=int, default=None, help='The number of iterations') 49 | parser.add_argument('--epochs', type=int, default=None, help='The number of epochs') 50 | parser.add_argument('--save-every', type=int, default=None, help='Save the checkpoint every iterations/epochs') 51 | parser.add_argument('--seed', type=int, default=None, help='The random seed') 52 | parser.add_argument('--eval', action="store_true", help='Whether to perform evaluation after training') 53 | parser.add_argument('--irregular', type=float, default=0, help='The ratio of missing observations (defaults to 0)') 54 | parser.add_argument('--tau_inst', type=float, default=0) 55 | parser.add_argument('--tau_temp', type=float, default=0) 56 | parser.add_argument('--alpha', type=float, default=0.5) 57 | parser.add_argument('--lambda_', type=float, default=0.5) 58 | parser.add_argument('--expid', type=int, default=2) 59 | parser.add_argument('--separate_reg', action="store_true", help='Whether to perform weighting in temporal loss') 60 | args = parser.parse_args() 61 | 62 | print("Dataset:", args.dataset) 63 | print("Arguments:", str(args)) 64 | 65 | args.soft_instance = (args.tau_inst > 0) 66 | args.soft_temporal = (args.tau_temp > 0) 67 | args.soft_cl = args.soft_instance + args.soft_temporal 68 | if 'forecast' in args.loader: 69 | args.soft_instance == False 70 | 71 | if args.loader in ['UCR','UEA']: 72 | result_name = f'results_classification_{args.loader}/' 73 | else: 74 | result_name = f'results_{args.loader}/' 75 | 76 | run_dir = result_name + f'TEMP{int(args.soft_temporal)}_INST{int(args.soft_instance)}' 77 | if args.soft_cl: 78 | run_dir += f'_tau_temp{float(args.tau_temp)}_tau_inst{float(args.tau_inst)}' 79 | if args.lambda_==0: 80 | run_dir += '_no_instCL' 81 | 82 | run_dir = os.path.join(run_dir, args.dataset, f'bs{args.batch_size}', f'run{args.expid}') 83 | os.makedirs(run_dir, exist_ok=True) 84 | 85 | file_exists = os.path.join(run_dir,f'eval_res_{args.dist_type}.pkl') 86 | 87 | if os.path.isfile(file_exists): 88 | print('You alreay have the results. Bye Bye~') 89 | sys.exit(0) 90 | 91 | fix_seed(args.expid) 92 | device = init_dl_program(args.gpu, seed=args.seed)#, max_threads=args.max_threads) 93 | 94 | print('Loading data... ', end='') 95 | if args.loader == 'UCR': 96 | task_type = 'classification' 97 | train_data, train_labels, test_data, test_labels, sim_mat = datautils.load_UCR(args.dataset, args.dist_type) 98 | 99 | elif args.loader == 'UEA': 100 | task_type = 'classification' 101 | train_data, train_labels, test_data, test_labels, sim_mat = datautils.load_UEA(args.dataset, args.max_train_length) 102 | 103 | elif args.loader == 'semi': 104 | task_type = 'semi-classification' 105 | train_data, train_labels, train1_data, train1_labels, train5_data, train5_labels, test_data, test_labels, sim_mat = datautils.load_semi_SSL(args.dataset) 106 | 107 | elif args.loader == 'forecast_csv': 108 | task_type = 'forecasting' 109 | UNI = False 110 | data, train_slice, valid_slice, test_slice, scaler, pred_lens, n_covariate_cols = datautils.load_forecast_csv(args.dataset, args.max_train_length, univar=UNI) 111 | train_data = data[:, train_slice] 112 | 113 | elif args.loader == 'forecast_csv_univar': 114 | task_type = 'forecasting' 115 | UNI = True 116 | data, train_slice, valid_slice, test_slice, scaler, pred_lens, n_covariate_cols = datautils.load_forecast_csv(args.dataset, args.max_train_length, univar=UNI) 117 | train_data = data[:, train_slice] 118 | 119 | elif args.loader == 'anomaly': 120 | task_type = 'anomaly_detection' 121 | train_data, all_train_data, all_train_labels, all_train_timestamps, all_test_data, all_test_labels, all_test_timestamps, delay = datautils.load_anomaly(args.dataset, args.max_train_length, cold=False) 122 | 123 | elif args.loader == 'anomaly_coldstart': 124 | task_type = 'anomaly_detection_coldstart' 125 | _, all_train_data, all_train_labels, all_train_timestamps, all_test_data, all_test_labels, all_test_timestamps, delay = datautils.load_anomaly(args.dataset, args.max_train_length, cold=True) 126 | train_data, _, _, _, _ = datautils.load_UCR('FordA') 127 | 128 | else: 129 | raise ValueError(f"Unknown loader {args.loader}.") 130 | 131 | if args.irregular > 0: 132 | if task_type == 'classification': 133 | train_data = data_dropout(train_data, args.irregular) 134 | test_data = data_dropout(test_data, args.irregular) 135 | else: 136 | raise ValueError(f"Task type {task_type} is not supported when irregular>0.") 137 | 138 | config = dict( 139 | batch_size=args.batch_size, 140 | lr=args.lr, 141 | tau_temp=args.tau_temp, 142 | lambda_ = args.lambda_, 143 | output_dims=args.repr_dims, 144 | max_train_length=args.max_train_length, 145 | soft_instance = args.soft_instance, 146 | soft_temporal = args.soft_temporal 147 | ) 148 | 149 | if args.save_every is not None: 150 | unit = 'epoch' if args.epochs is not None else 'iter' 151 | config[f'after_{unit}_callback'] = save_checkpoint_callback(args.save_every, unit) 152 | 153 | t = time.time() 154 | 155 | model = TS2Vec( 156 | input_dims=train_data.shape[-1], 157 | device=device, 158 | **config 159 | ) 160 | 161 | if args.soft_instance: 162 | dist_mat = 1 - sim_mat 163 | del sim_mat 164 | sim_mat = densify(-dist_mat, args.tau_inst, args.alpha) 165 | print('Soft Assignment Matrix', sim_mat.shape) 166 | 167 | if task_type in ['forecasting','anomaly_detection','anomaly_detection_coldstart']: 168 | sim_mat = None 169 | 170 | if task_type == 'classification': 171 | loss_log = model.fit( 172 | train_data, train_labels, test_data, test_labels, 173 | sim_mat, run_dir, n_epochs=args.epochs, n_iters=args.iters, 174 | verbose=True) 175 | else: 176 | loss_log = model.fit( 177 | train_data, 0, 0, 0, 178 | sim_mat,run_dir, n_epochs=args.epochs, n_iters=args.iters, 179 | verbose=True) 180 | 181 | #model.save(f'{run_dir}/model.pkl') 182 | t = time.time() - t 183 | print(f"\nTraining time: {datetime.timedelta(seconds=t)}\n") 184 | 185 | if args.eval: 186 | if task_type == 'classification': 187 | out, eval_res = tasks.eval_classification(model, train_data, train_labels, test_data, test_labels, eval_protocol='svm') 188 | elif task_type == 'forecasting': 189 | if args.dataset == 'electricity': 190 | if args.separate_reg: 191 | out, eval_res = tasks.eval_forecasting_separate(model, data, train_slice, valid_slice, test_slice, scaler, pred_lens, n_covariate_cols,univar=True) 192 | else: 193 | out, eval_res = tasks.eval_forecasting(model, data, train_slice, valid_slice, test_slice, scaler, pred_lens, n_covariate_cols,univar=True) 194 | else: 195 | if args.separate_reg: 196 | out, eval_res = tasks.eval_forecasting_separate(model, data, train_slice, valid_slice, test_slice, scaler, pred_lens, n_covariate_cols,univar=True) 197 | else: 198 | out, eval_res = tasks.eval_forecasting(model, data, train_slice, valid_slice, test_slice, scaler, pred_lens, n_covariate_cols,univar=UNI) 199 | elif task_type == 'anomaly_detection': 200 | out, eval_res = tasks.eval_anomaly_detection(model, all_train_data, all_train_labels, all_train_timestamps, all_test_data, all_test_labels, all_test_timestamps, delay) 201 | elif task_type == 'anomaly_detection_coldstart': 202 | out, eval_res = tasks.eval_anomaly_detection_coldstart(model, all_train_data, all_train_labels, all_train_timestamps, all_test_data, all_test_labels, all_test_timestamps, delay) 203 | elif task_type == 'semi-classification': 204 | out, eval_res = tasks.eval_semi_classification(model, 205 | train1_data, train1_labels, 206 | train5_data, train5_labels, 207 | train_labels, 208 | test_data, test_labels, eval_protocol='svm') 209 | else: 210 | assert False 211 | 212 | #pkl_save(f'{run_dir}/out.pkl', out) 213 | pkl_save(f'{run_dir}/eval_res_{args.dist_type}.pkl', eval_res) 214 | print('Evaluation result:', eval_res) 215 | 216 | print("Finished.") 217 | -------------------------------------------------------------------------------- /softclt_catcc/dataloader/dataloader.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tqdm 3 | 4 | import numpy as np 5 | import pandas as pd 6 | from fastdtw import fastdtw 7 | from tslearn.metrics import dtw, dtw_path,gak 8 | 9 | import torch 10 | from torch.utils.data import Dataset 11 | 12 | from .augmentations import DataTransform 13 | from sklearn.preprocessing import MinMaxScaler 14 | 15 | def get_DTW(UTS_tr): 16 | N = len(UTS_tr) 17 | dist_mat = np.zeros((N,N)) 18 | for i in tqdm.tqdm(range(N)): 19 | for j in range(N): 20 | if i>j: 21 | dist = dtw(UTS_tr[i].reshape(-1,1), UTS_tr[j].reshape(-1,1)) 22 | dist_mat[i,j] = dist 23 | dist_mat[j,i] = dist 24 | elif i==j: 25 | dist_mat[i,j] = 0 26 | else : 27 | pass 28 | return dist_mat 29 | 30 | def get_TAM(UTS_tr): 31 | N = len(UTS_tr) 32 | dist_mat = np.zeros((N,N)) 33 | for i in tqdm.tqdm(range(N)): 34 | for j in range(N): 35 | if i>j: 36 | k = dtw_path(UTS_tr[i].reshape(-1,1), 37 | UTS_tr[j].reshape(-1,1))[0] 38 | p = [np.array([i[0] for i in k]), 39 | np.array([i[1] for i in k])] 40 | dist = tam(p) 41 | dist_mat[i,j] = dist 42 | dist_mat[j,i] = dist 43 | elif i==j: 44 | dist_mat[i,j] = 0 45 | else : 46 | pass 47 | return dist_mat 48 | 49 | def get_GAK(UTS_tr): 50 | N = len(UTS_tr) 51 | dist_mat = np.zeros((N,N)) 52 | for i in tqdm.tqdm(range(N)): 53 | for j in range(N): 54 | if i>j: 55 | dist = gak(UTS_tr[i].reshape(-1,1), 56 | UTS_tr[j].reshape(-1,1)) 57 | dist_mat[i,j] = dist 58 | dist_mat[j,i] = dist 59 | elif i==j: 60 | dist_mat[i,j] = 0 61 | else : 62 | pass 63 | return dist_mat 64 | 65 | 66 | def get_MDTW(MTS_tr): 67 | N = MTS_tr.shape[0] 68 | dist_mat = np.zeros((N,N)) 69 | for i in tqdm.tqdm(range(N)): 70 | for j in range(N): 71 | if i>j: 72 | mdtw_dist = dtw(MTS_tr[i], MTS_tr[j]) 73 | dist_mat[i,j] = mdtw_dist 74 | dist_mat[j,i] = mdtw_dist 75 | elif i==j: 76 | dist_mat[i,j] = 0 77 | else : 78 | pass 79 | return dist_mat 80 | 81 | def get_COS(MTS_tr): 82 | cos_sim_matrix = -cosine_similarity(MTS_tr) 83 | return cos_sim_matrix 84 | 85 | def get_EUC(MTS_tr): 86 | return euclidean_distances(MTS_tr) 87 | 88 | def save_sim_mat(X_tr, min_ = 0, max_ = 1, multivariate=False, type_='DTW'): 89 | N = dist_mat.shape[0] 90 | if multivariate: 91 | assert type=='DTW' 92 | dist_mat = get_MDTW(X_tr) 93 | else: 94 | if type_=='DTW': 95 | dist_mat = get_DTW(X_tr) 96 | elif type_=='TAM': 97 | dist_mat = get_TAM(X_tr) 98 | elif type_=='COS': 99 | dist_mat = get_COS(X_tr) 100 | elif type_=='EUC': 101 | dist_mat = get_EUC(X_tr) 102 | elif type_=='GAK': 103 | dist_mat = get_GAK(X_tr) 104 | 105 | # (1) distance matrix 106 | diag_indices = np.diag_indices(N) 107 | mask = np.ones(dist_mat.shape, dtype=bool) 108 | mask[diag_indices] = False 109 | temp = dist_mat[mask].reshape(N, N-1) 110 | dist_mat[diag_indices] = temp.min() 111 | 112 | # (2) normalized distance matrix 113 | scaler = MinMaxScaler(feature_range=(min_, max_)) 114 | dist_mat = scaler.fit_transform(dist_mat) 115 | 116 | # (3) normalized similarity matrix 117 | return 1 - dist_mat 118 | 119 | 120 | class Load_Dataset(Dataset): 121 | # Initialize your data, download, etc. 122 | def __init__(self, dataset, config, training_mode): 123 | super(Load_Dataset, self).__init__() 124 | self.training_mode = training_mode 125 | 126 | X_train = dataset["samples"] 127 | y_train = dataset["labels"] 128 | 129 | if len(X_train.shape) < 3: 130 | X_train = X_train.unsqueeze(2) 131 | 132 | if X_train.shape.index(min(X_train.shape)) != 1: # make sure the Channels in second dim 133 | X_train = X_train.permute(0, 2, 1) 134 | 135 | if isinstance(X_train, np.ndarray): 136 | self.x_data = torch.from_numpy(X_train) 137 | self.y_data = torch.from_numpy(y_train).long() 138 | else: 139 | self.x_data = X_train 140 | self.y_data = y_train 141 | 142 | self.len = X_train.shape[0] 143 | if training_mode == "self_supervised" or training_mode == "SupCon": # no need to apply Augmentations in other modes 144 | self.aug1, self.aug2 = DataTransform(self.x_data, config) 145 | 146 | def __getitem__(self, index): 147 | if self.training_mode == "self_supervised" or self.training_mode == "SupCon": 148 | return index, self.x_data[index], self.y_data[index], self.aug1[index], self.aug2[index] 149 | else: 150 | return index, self.x_data[index], self.y_data[index], self.x_data[index], self.x_data[index] 151 | 152 | def __len__(self): 153 | return self.len 154 | 155 | 156 | def data_generator(data_path, configs, training_mode, pc, batch_size): 157 | #batch_size = configs.batch_size 158 | 159 | if training_mode != "SupCon": 160 | if ('ft' in training_mode) & (pc == 1): 161 | print('1%') 162 | train_dataset = torch.load(os.path.join(data_path, "train_1perc.pt")) 163 | elif ('ft' in training_mode) & (pc == 5): 164 | print('5%') 165 | train_dataset = torch.load(os.path.join(data_path, "train_5perc.pt")) 166 | elif ('ft' in training_mode) & (pc == 10): 167 | print('10%') 168 | train_dataset = torch.load(os.path.join(data_path, "train_10perc.pt")) 169 | elif ('ft' in training_mode) & (pc == 50): 170 | print('50%') 171 | train_dataset = torch.load(os.path.join(data_path, "train_50perc.pt")) 172 | elif ('ft' in training_mode) & (pc == 75): 173 | print('75%') 174 | train_dataset = torch.load(os.path.join(data_path, "train_75perc.pt")) 175 | else: 176 | train_dataset = torch.load(os.path.join(data_path, "train.pt")) 177 | else : 178 | train_dataset = torch.load(os.path.join(data_path, f"pseudo_train_data_{str(pc)}perc.pt")) 179 | 180 | valid_dataset = torch.load(os.path.join(data_path, "val.pt")) 181 | test_dataset = torch.load(os.path.join(data_path, "test.pt")) 182 | print('Train size: ',train_dataset['samples'].shape) 183 | print('Valid size: ',valid_dataset['samples'].shape) 184 | print('Test size: ',test_dataset['samples'].shape) 185 | train_dataset = Load_Dataset(train_dataset, configs, training_mode) 186 | valid_dataset = Load_Dataset(valid_dataset, configs, training_mode) 187 | test_dataset = Load_Dataset(test_dataset, configs, training_mode) 188 | 189 | if batch_size == 999: 190 | if train_dataset.__len__() < batch_size: 191 | if train_dataset.__len__() > 16: 192 | batch_size = 16 193 | else: 194 | batch_size = 4 195 | else: 196 | batch_size = configs.batch_size 197 | 198 | print('batch_size',batch_size) 199 | train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, 200 | shuffle=True, drop_last=configs.drop_last, num_workers=0) 201 | valid_loader = torch.utils.data.DataLoader(dataset=valid_dataset, batch_size=batch_size, 202 | shuffle=False, drop_last=configs.drop_last, num_workers=0) 203 | test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, 204 | shuffle=False, drop_last=False, num_workers=0) 205 | return train_loader, valid_loader, test_loader 206 | 207 | 208 | def data_generator_wo_val(data_path, configs, training_mode, pc, batch_size): 209 | #batch_size = configs.batch_size 210 | 211 | if training_mode != "SupCon": 212 | if ('ft' in training_mode) & (pc == 1): 213 | print('1%') 214 | train_dataset = torch.load(os.path.join(data_path, "train_1perc.pt")) 215 | elif ('ft' in training_mode) & (pc == 5): 216 | print('5%') 217 | train_dataset = torch.load(os.path.join(data_path, "train_5perc.pt")) 218 | elif ('ft' in training_mode) & (pc == 10): 219 | print('10%') 220 | train_dataset = torch.load(os.path.join(data_path, "train_10perc.pt")) 221 | elif ('ft' in training_mode) & (pc == 50): 222 | print('50%') 223 | train_dataset = torch.load(os.path.join(data_path, "train_50perc.pt")) 224 | elif ('ft' in training_mode) & (pc == 75): 225 | print('75%') 226 | train_dataset = torch.load(os.path.join(data_path, "train_75perc.pt")) 227 | else: 228 | train_dataset = torch.load(os.path.join(data_path, "train.pt")) 229 | else : 230 | train_dataset = torch.load(os.path.join(data_path, f"pseudo_train_data_{str(pc)}perc.pt")) 231 | 232 | # valid_dataset = torch.load(os.path.join(data_path, "val.pt")) 233 | test_dataset = torch.load(os.path.join(data_path, "test.pt")) 234 | train_dataset['samples'] = train_dataset['samples']#[:1000,:,:] 235 | test_dataset['samples'] = test_dataset['samples']#[:1000,:,:] 236 | train_dataset['labels'] = train_dataset['labels']#[:1000] 237 | test_dataset['labels'] = test_dataset['labels']#[:1000] 238 | print('Train size: ',train_dataset['samples'].shape) 239 | print('Test size: ',test_dataset['samples'].shape) 240 | 241 | train_dataset = Load_Dataset(train_dataset, configs, training_mode) 242 | test_dataset = Load_Dataset(test_dataset, configs, training_mode) 243 | 244 | if train_dataset.__len__() < configs.batch_size: 245 | if train_dataset.__len__() > 16: 246 | batch_size = 16 247 | else: 248 | batch_size = 4 249 | else: 250 | batch_size = configs.batch_size 251 | 252 | train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, 253 | shuffle=True, drop_last=configs.drop_last, num_workers=0) 254 | test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, 255 | shuffle=False, drop_last=False, num_workers=0) 256 | return train_loader,test_loader 257 | 258 | def set_nan_to_zero(a): 259 | where_are_NaNs = np.isnan(a) 260 | a[where_are_NaNs] = 0 261 | return a 262 | 263 | def normalize_TS(TS): 264 | TS = set_nan_to_zero(TS) 265 | if TS.ndim == 2: 266 | print('Preprocessing UTS ...') 267 | TS_max = TS.max(axis = 1).reshape(-1,1) 268 | TS_min = TS.min(axis = 1).reshape(-1,1) 269 | TS = (TS - TS_min)/(TS_max - TS_min + (1e-6)) 270 | elif TS.ndim == 3: 271 | print('Preprocessing MTS ...') 272 | N, D, L = TS.shape 273 | TS_max = TS.max(axis=2).reshape(N,D,1) 274 | TS_min = TS.min(axis=2).reshape(N,D,1) 275 | TS = (TS - TS_min) / (TS_max - TS_min + (1e-6)) 276 | return TS 277 | 278 | 279 | def load_sim_matrix(dataset, type_='DTW'): 280 | tr = torch.load(f'data/{dataset}/train.pt') 281 | train = tr['samples'].detach().cpu().numpy().astype(np.float64) 282 | 283 | if (train.ndim==3) & (train.shape[1]==1): 284 | train=train.squeeze(1) 285 | elif (train.ndim==3) & (train.shape[1]>1): 286 | train=train.transpose(0,2,1) 287 | 288 | os.makedirs(f'data/{dataset}', exist_ok=True) 289 | MAT_PATH = os.path.join(f'data/{dataset}',f'{type_}.npy') 290 | 291 | if os.path.exists(MAT_PATH): 292 | print(f"{type_} already exists") 293 | sim_mat = np.load(MAT_PATH) 294 | else: 295 | print(f"Saving {type_} ...") 296 | sim_mat = save_sim_mat(normalize_TS(train), min_ = 0, max_ = 1) 297 | np.save(MAT_PATH, sim_mat) 298 | 299 | return sim_mat -------------------------------------------------------------------------------- /softclt_catcc/models/soft_losses.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from models.timelags import * 4 | 5 | ######################################################################################################## 6 | ## 1. Soft Contrastive Losses 7 | ######################################################################################################## 8 | #------------------------------------------------------------------------------------------# 9 | # (1) Instance-wise CL 10 | #------------------------------------------------------------------------------------------# 11 | def inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R): 12 | B, T = z1.size(0), z1.size(1) 13 | if B == 1: 14 | return z1.new_tensor(0.) 15 | z = torch.cat([z1, z2], dim=0) # 2B x T x C 16 | z = z.transpose(0, 1) # T x 2B x C 17 | sim = torch.matmul(z, z.transpose(1, 2)) # T x 2B x 2B 18 | logits = torch.tril(sim, diagonal=-1)[:, :, :-1] # T x 2B x (2B-1) 19 | logits += torch.triu(sim, diagonal=1)[:, :, 1:] 20 | logits = -F.log_softmax(logits, dim=-1) 21 | i = torch.arange(B, device=z1.device) 22 | loss = torch.sum(logits[:,i]*soft_labels_L) 23 | loss += torch.sum(logits[:,B + i]*soft_labels_R) 24 | loss /= (2*B*T) 25 | return loss 26 | 27 | #------------------------------------------------------------------------------------------# 28 | # (2) Temporal CL 29 | #------------------------------------------------------------------------------------------# 30 | def temp_CL_soft(z1, z2, timelag_L, timelag_R): 31 | B, T = z1.size(0), z1.size(1) 32 | if T == 1: 33 | return z1.new_tensor(0.) 34 | z = torch.cat([z1, z2], dim=1) # B x 2T x C 35 | sim = torch.matmul(z, z.transpose(1, 2)) # B x 2T x 2T 36 | logits = torch.tril(sim, diagonal=-1)[:, :, :-1] # B x 2T x (2T-1) 37 | logits += torch.triu(sim, diagonal=1)[:, :, 1:] 38 | logits = -F.log_softmax(logits, dim=-1) 39 | t = torch.arange(T, device=z1.device) 40 | loss = torch.sum(logits[:,t]*timelag_L) 41 | loss += torch.sum(logits[:,T + t]*timelag_R) 42 | loss /= (2*B*T) 43 | return loss 44 | 45 | #------------------------------------------------------------------------------------------# 46 | # (3) Hierarchical CL = Instance CL + Temporal CL 47 | # (The below differs by the way it generates timelag for temporal CL ) 48 | ## 3-1) hier_CL_soft : sigmoid 49 | ## 3-2) hier_CL_soft_window : window 50 | ## 3-3) hier_CL_soft_thres : threshold 51 | ## 3-4) hier_CL_soft_gaussian : gaussian 52 | ## 3-5) hier_CL_soft_interval : same interval 53 | ## 3-6) hier_CL_soft_wo_inst : 3-1) w/o instance CL 54 | #------------------------------------------------------------------------------------------# 55 | 56 | def hier_CL_soft(z1, z2, soft_labels, tau_temp=2, lambda_=0.5, temporal_unit=0, 57 | soft_temporal=False, soft_instance=False, temporal_hierarchy=True): 58 | 59 | if soft_labels is not None: 60 | soft_labels = torch.tensor(soft_labels, device=z1.device) 61 | soft_labels_L, soft_labels_R = dup_matrix(soft_labels) 62 | loss = torch.tensor(0., device=z1.device) 63 | d = 0 64 | while z1.size(1) > 1: 65 | if lambda_ != 0: 66 | if soft_instance: 67 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 68 | else: 69 | loss += lambda_ * inst_CL_hard(z1, z2) 70 | if d >= temporal_unit: 71 | if 1 - lambda_ != 0: 72 | if soft_temporal: 73 | if temporal_hierarchy: 74 | timelag = timelag_sigmoid(z1.shape[1],tau_temp*(2**d)) 75 | else: 76 | timelag = timelag_sigmoid(z1.shape[1],tau_temp) 77 | timelag = torch.tensor(timelag, device=z1.device) 78 | timelag_L, timelag_R = dup_matrix(timelag) 79 | loss += (1 - lambda_) * temp_CL_soft(z1, z2, timelag_L, timelag_R) 80 | else: 81 | loss += (1 - lambda_) * temp_CL_hard(z1, z2) 82 | d += 1 83 | z1 = F.max_pool1d(z1.transpose(1, 2), kernel_size=2).transpose(1, 2) 84 | z2 = F.max_pool1d(z2.transpose(1, 2), kernel_size=2).transpose(1, 2) 85 | 86 | if z1.size(1) == 1: 87 | if lambda_ != 0: 88 | if soft_instance: 89 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 90 | else: 91 | loss += lambda_ * inst_CL_hard(z1, z2) 92 | d += 1 93 | 94 | return loss / d 95 | 96 | 97 | def hier_CL_soft_window(z1, z2, soft_labels, window_ratio, tau_temp=2, lambda_=0.5, 98 | temporal_unit=0, soft_temporal=False, soft_instance=False): 99 | soft_labels = torch.tensor(soft_labels, device=z1.device) 100 | soft_labels_L, soft_labels_R = dup_matrix(soft_labels) 101 | loss = torch.tensor(0., device=z1.device) 102 | d = 0 103 | while z1.size(1) > 1: 104 | if lambda_ != 0: 105 | if soft_instance: 106 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 107 | else: 108 | loss += lambda_ * inst_CL_hard(z1, z2) 109 | if d >= temporal_unit: 110 | if 1 - lambda_ != 0: 111 | if soft_temporal: 112 | timelag = timelag_sigmoid_window(z1.shape[1],tau_temp*(2**d),window_ratio) 113 | timelag = torch.tensor(timelag, device=z1.device) 114 | timelag_L, timelag_R = dup_matrix(timelag) 115 | loss += (1 - lambda_) * temp_CL_soft(z1, z2, timelag_L, timelag_R) 116 | else: 117 | loss += (1 - lambda_) * temp_CL_hard(z1, z2) 118 | d += 1 119 | z1 = F.max_pool1d(z1.transpose(1, 2), kernel_size=2).transpose(1, 2) 120 | z2 = F.max_pool1d(z2.transpose(1, 2), kernel_size=2).transpose(1, 2) 121 | 122 | if z1.size(1) == 1: 123 | if lambda_ != 0: 124 | if soft_instance: 125 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 126 | else: 127 | loss += lambda_ * inst_CL_hard(z1, z2) 128 | d += 1 129 | 130 | return loss / d 131 | 132 | def hier_CL_soft_thres(z1, z2, soft_labels, threshold, lambda_=0.5, temporal_unit=0, soft_temporal=False, soft_instance=False): 133 | soft_labels = torch.tensor(soft_labels, device=z1.device) 134 | soft_labels_L, soft_labels_R = dup_matrix(soft_labels) 135 | loss = torch.tensor(0., device=z1.device) 136 | d = 0 137 | while z1.size(1) > 1: 138 | if lambda_ != 0: 139 | if soft_instance: 140 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 141 | else: 142 | loss += lambda_ * inst_CL_hard(z1, z2) 143 | if d >= temporal_unit: 144 | if 1 - lambda_ != 0: 145 | if soft_temporal: 146 | timelag = timelag_sigmoid_threshold(z1.shape[1], threshold) 147 | timelag = torch.tensor(timelag, device=z1.device) 148 | timelag_L, timelag_R = dup_matrix(timelag) 149 | loss += (1 - lambda_) * temp_CL_soft(z1, z2, timelag_L, timelag_R) 150 | else: 151 | loss += (1 - lambda_) * temp_CL_hard(z1, z2) 152 | d += 1 153 | z1 = F.max_pool1d(z1.transpose(1, 2), kernel_size=2).transpose(1, 2) 154 | z2 = F.max_pool1d(z2.transpose(1, 2), kernel_size=2).transpose(1, 2) 155 | 156 | if z1.size(1) == 1: 157 | if lambda_ != 0: 158 | if soft_instance: 159 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 160 | else: 161 | loss += lambda_ * inst_CL_hard(z1, z2) 162 | d += 1 163 | 164 | return loss / d 165 | 166 | 167 | def hier_CL_soft_gaussian(z1, z2, soft_labels, tau_temp=2, lambda_=0.5, temporal_unit=0, soft_temporal=False, soft_instance=False, temporal_hierarchy=True): 168 | soft_labels = torch.tensor(soft_labels, device=z1.device) 169 | soft_labels_L, soft_labels_R = dup_matrix(soft_labels) 170 | loss = torch.tensor(0., device=z1.device) 171 | d = 0 172 | while z1.size(1) > 1: 173 | if lambda_ != 0: 174 | if soft_instance: 175 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 176 | else: 177 | loss += lambda_ * inst_CL_hard(z1, z2) 178 | if d >= temporal_unit: 179 | if 1 - lambda_ != 0: 180 | if soft_temporal: 181 | if temporal_hierarchy: 182 | timelag = timelag_gaussian(z1.shape[1],tau_temp/(2**d)) 183 | else: 184 | timelag = timelag_gaussian(z1.shape[1],tau_temp) 185 | timelag = torch.tensor(timelag, device=z1.device) 186 | timelag_L, timelag_R = dup_matrix(timelag) 187 | loss += (1 - lambda_) * temp_CL_soft(z1, z2, timelag_L, timelag_R) 188 | else: 189 | loss += (1 - lambda_) * temp_CL_hard(z1, z2) 190 | d += 1 191 | z1 = F.max_pool1d(z1.transpose(1, 2), kernel_size=2).transpose(1, 2) 192 | z2 = F.max_pool1d(z2.transpose(1, 2), kernel_size=2).transpose(1, 2) 193 | 194 | if z1.size(1) == 1: 195 | if lambda_ != 0: 196 | if soft_instance: 197 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 198 | else: 199 | loss += lambda_ * inst_CL_hard(z1, z2) 200 | d += 1 201 | 202 | return loss / d 203 | 204 | 205 | def hier_CL_soft_interval(z1, z2, soft_labels, tau_temp=2, lambda_=0.5, temporal_unit=0, soft_temporal=False, soft_instance=False): 206 | soft_labels = torch.tensor(soft_labels, device=z1.device) 207 | soft_labels_L, soft_labels_R = dup_matrix(soft_labels) 208 | loss = torch.tensor(0., device=z1.device) 209 | d = 0 210 | while z1.size(1) > 1: 211 | if lambda_ != 0: 212 | if soft_instance: 213 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 214 | else: 215 | loss += lambda_ * inst_CL_hard(z1, z2) 216 | if d >= temporal_unit: 217 | if 1 - lambda_ != 0: 218 | if soft_temporal: 219 | timelag = timelag_same_interval(z1.shape[1],tau_temp/(2**d)) 220 | timelag = torch.tensor(timelag, device=z1.device) 221 | timelag_L, timelag_R = dup_matrix(timelag) 222 | loss += (1 - lambda_) * temp_CL_soft(z1, z2, timelag_L, timelag_R) 223 | else: 224 | loss += (1 - lambda_) * temp_CL_hard(z1, z2) 225 | d += 1 226 | z1 = F.max_pool1d(z1.transpose(1, 2), kernel_size=2).transpose(1, 2) 227 | z2 = F.max_pool1d(z2.transpose(1, 2), kernel_size=2).transpose(1, 2) 228 | 229 | if z1.size(1) == 1: 230 | if lambda_ != 0: 231 | if soft_instance: 232 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 233 | else: 234 | loss += lambda_ * inst_CL_hard(z1, z2) 235 | d += 1 236 | 237 | return loss / d 238 | 239 | def hier_CL_soft_wo_inst(z1, z2, soft_labels, tau_temp=2, lambda_=0.5, temporal_unit=0, soft_temporal=False, soft_instance=False): 240 | soft_labels = torch.tensor(soft_labels, device=z1.device) 241 | soft_labels_L, soft_labels_R = dup_matrix(soft_labels) 242 | loss = torch.tensor(0., device=z1.device) 243 | d = 0 244 | while z1.size(1) > 1: 245 | if d >= temporal_unit: 246 | if 1 - lambda_ != 0: 247 | if soft_temporal: 248 | timelag = timelag_sigmoid(z1.shape[1],tau_temp*(2**d)) 249 | timelag = torch.tensor(timelag, device=z1.device) 250 | timelag_L, timelag_R = dup_matrix(timelag) 251 | loss += (1 - lambda_) * temp_CL_soft(z1, z2, timelag_L, timelag_R) 252 | else: 253 | loss += (1 - lambda_) * temp_CL_hard(z1, z2) 254 | d += 1 255 | z1 = F.max_pool1d(z1.transpose(1, 2), kernel_size=2).transpose(1, 2) 256 | z2 = F.max_pool1d(z2.transpose(1, 2), kernel_size=2).transpose(1, 2) 257 | 258 | if z1.size(1) == 1: 259 | d += 1 260 | 261 | return loss / d 262 | 263 | -------------------------------------------------------------------------------- /softclt_ts2vec/models/soft_losses.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from models.timelags import * 4 | from models.hard_losses import * 5 | 6 | ######################################################################################################## 7 | ## 1. Soft Contrastive Losses 8 | ######################################################################################################## 9 | #------------------------------------------------------------------------------------------# 10 | # (1) Instance-wise CL 11 | #------------------------------------------------------------------------------------------# 12 | def inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R): 13 | B, T = z1.size(0), z1.size(1) 14 | if B == 1: 15 | return z1.new_tensor(0.) 16 | z = torch.cat([z1, z2], dim=0) # 2B x T x C 17 | z = z.transpose(0, 1) # T x 2B x C 18 | sim = torch.matmul(z, z.transpose(1, 2)) # T x 2B x 2B 19 | logits = torch.tril(sim, diagonal=-1)[:, :, :-1] # T x 2B x (2B-1) 20 | logits += torch.triu(sim, diagonal=1)[:, :, 1:] 21 | logits = -F.log_softmax(logits, dim=-1) 22 | i = torch.arange(B, device=z1.device) 23 | loss = torch.sum(logits[:,i]*soft_labels_L) 24 | loss += torch.sum(logits[:,B + i]*soft_labels_R) 25 | loss /= (2*B*T) 26 | return loss 27 | 28 | #------------------------------------------------------------------------------------------# 29 | # (2) Temporal CL 30 | #------------------------------------------------------------------------------------------# 31 | def temp_CL_soft(z1, z2, timelag_L, timelag_R): 32 | B, T = z1.size(0), z1.size(1) 33 | if T == 1: 34 | return z1.new_tensor(0.) 35 | z = torch.cat([z1, z2], dim=1) # B x 2T x C 36 | sim = torch.matmul(z, z.transpose(1, 2)) # B x 2T x 2T 37 | logits = torch.tril(sim, diagonal=-1)[:, :, :-1] # B x 2T x (2T-1) 38 | logits += torch.triu(sim, diagonal=1)[:, :, 1:] 39 | logits = -F.log_softmax(logits, dim=-1) 40 | t = torch.arange(T, device=z1.device) 41 | loss = torch.sum(logits[:,t]*timelag_L) 42 | loss += torch.sum(logits[:,T + t]*timelag_R) 43 | loss /= (2*B*T) 44 | return loss 45 | 46 | #------------------------------------------------------------------------------------------# 47 | # (3) Hierarchical CL = Instance CL + Temporal CL 48 | # (The below differs by the way it generates timelag for temporal CL ) 49 | ## 3-1) hier_CL_soft : sigmoid 50 | ## 3-2) hier_CL_soft_window : window 51 | ## 3-3) hier_CL_soft_thres : threshold 52 | ## 3-4) hier_CL_soft_gaussian : gaussian 53 | ## 3-5) hier_CL_soft_interval : same interval 54 | ## 3-6) hier_CL_soft_wo_inst : 3-1) w/o instance CL 55 | #------------------------------------------------------------------------------------------# 56 | 57 | def hier_CL_soft(z1, z2, soft_labels, tau_temp=2, lambda_=0.5, temporal_unit=0, 58 | soft_temporal=False, soft_instance=False, temporal_hierarchy=True): 59 | 60 | if soft_labels is not None: 61 | soft_labels = torch.tensor(soft_labels, device=z1.device) 62 | soft_labels_L, soft_labels_R = dup_matrix(soft_labels) 63 | loss = torch.tensor(0., device=z1.device) 64 | d = 0 65 | while z1.size(1) > 1: 66 | if lambda_ != 0: 67 | if soft_instance: 68 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 69 | else: 70 | loss += lambda_ * inst_CL_hard(z1, z2) 71 | if d >= temporal_unit: 72 | if 1 - lambda_ != 0: 73 | if soft_temporal: 74 | if temporal_hierarchy: 75 | timelag = timelag_sigmoid(z1.shape[1],tau_temp*(2**d)) 76 | else: 77 | timelag = timelag_sigmoid(z1.shape[1],tau_temp) 78 | timelag = torch.tensor(timelag, device=z1.device) 79 | timelag_L, timelag_R = dup_matrix(timelag) 80 | loss += (1 - lambda_) * temp_CL_soft(z1, z2, timelag_L, timelag_R) 81 | else: 82 | loss += (1 - lambda_) * temp_CL_hard(z1, z2) 83 | d += 1 84 | z1 = F.max_pool1d(z1.transpose(1, 2), kernel_size=2).transpose(1, 2) 85 | z2 = F.max_pool1d(z2.transpose(1, 2), kernel_size=2).transpose(1, 2) 86 | 87 | if z1.size(1) == 1: 88 | if lambda_ != 0: 89 | if soft_instance: 90 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 91 | else: 92 | loss += lambda_ * inst_CL_hard(z1, z2) 93 | d += 1 94 | 95 | return loss / d 96 | 97 | 98 | def hier_CL_soft_window(z1, z2, soft_labels, window_ratio, tau_temp=2, lambda_=0.5, 99 | temporal_unit=0, soft_temporal=False, soft_instance=False): 100 | soft_labels = torch.tensor(soft_labels, device=z1.device) 101 | soft_labels_L, soft_labels_R = dup_matrix(soft_labels) 102 | loss = torch.tensor(0., device=z1.device) 103 | d = 0 104 | while z1.size(1) > 1: 105 | if lambda_ != 0: 106 | if soft_instance: 107 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 108 | else: 109 | loss += lambda_ * inst_CL_hard(z1, z2) 110 | if d >= temporal_unit: 111 | if 1 - lambda_ != 0: 112 | if soft_temporal: 113 | timelag = timelag_sigmoid_window(z1.shape[1],tau_temp*(2**d),window_ratio) 114 | timelag = torch.tensor(timelag, device=z1.device) 115 | timelag_L, timelag_R = dup_matrix(timelag) 116 | loss += (1 - lambda_) * temp_CL_soft(z1, z2, timelag_L, timelag_R) 117 | else: 118 | loss += (1 - lambda_) * temp_CL_hard(z1, z2) 119 | d += 1 120 | z1 = F.max_pool1d(z1.transpose(1, 2), kernel_size=2).transpose(1, 2) 121 | z2 = F.max_pool1d(z2.transpose(1, 2), kernel_size=2).transpose(1, 2) 122 | 123 | if z1.size(1) == 1: 124 | if lambda_ != 0: 125 | if soft_instance: 126 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 127 | else: 128 | loss += lambda_ * inst_CL_hard(z1, z2) 129 | d += 1 130 | 131 | return loss / d 132 | 133 | def hier_CL_soft_thres(z1, z2, soft_labels, threshold, lambda_=0.5, temporal_unit=0, soft_temporal=False, soft_instance=False): 134 | soft_labels = torch.tensor(soft_labels, device=z1.device) 135 | soft_labels_L, soft_labels_R = dup_matrix(soft_labels) 136 | loss = torch.tensor(0., device=z1.device) 137 | d = 0 138 | while z1.size(1) > 1: 139 | if lambda_ != 0: 140 | if soft_instance: 141 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 142 | else: 143 | loss += lambda_ * inst_CL_hard(z1, z2) 144 | if d >= temporal_unit: 145 | if 1 - lambda_ != 0: 146 | if soft_temporal: 147 | timelag = timelag_sigmoid_threshold(z1.shape[1], threshold) 148 | timelag = torch.tensor(timelag, device=z1.device) 149 | timelag_L, timelag_R = dup_matrix(timelag) 150 | loss += (1 - lambda_) * temp_CL_soft(z1, z2, timelag_L, timelag_R) 151 | else: 152 | loss += (1 - lambda_) * temp_CL_hard(z1, z2) 153 | d += 1 154 | z1 = F.max_pool1d(z1.transpose(1, 2), kernel_size=2).transpose(1, 2) 155 | z2 = F.max_pool1d(z2.transpose(1, 2), kernel_size=2).transpose(1, 2) 156 | 157 | if z1.size(1) == 1: 158 | if lambda_ != 0: 159 | if soft_instance: 160 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 161 | else: 162 | loss += lambda_ * inst_CL_hard(z1, z2) 163 | d += 1 164 | 165 | return loss / d 166 | 167 | 168 | def hier_CL_soft_gaussian(z1, z2, soft_labels, tau_temp=2, lambda_=0.5, temporal_unit=0, soft_temporal=False, soft_instance=False, temporal_hierarchy=True): 169 | soft_labels = torch.tensor(soft_labels, device=z1.device) 170 | soft_labels_L, soft_labels_R = dup_matrix(soft_labels) 171 | loss = torch.tensor(0., device=z1.device) 172 | d = 0 173 | while z1.size(1) > 1: 174 | if lambda_ != 0: 175 | if soft_instance: 176 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 177 | else: 178 | loss += lambda_ * inst_CL_hard(z1, z2) 179 | if d >= temporal_unit: 180 | if 1 - lambda_ != 0: 181 | if soft_temporal: 182 | if temporal_hierarchy: 183 | timelag = timelag_gaussian(z1.shape[1],tau_temp/(2**d)) 184 | else: 185 | timelag = timelag_gaussian(z1.shape[1],tau_temp) 186 | timelag = torch.tensor(timelag, device=z1.device) 187 | timelag_L, timelag_R = dup_matrix(timelag) 188 | loss += (1 - lambda_) * temp_CL_soft(z1, z2, timelag_L, timelag_R) 189 | else: 190 | loss += (1 - lambda_) * temp_CL_hard(z1, z2) 191 | d += 1 192 | z1 = F.max_pool1d(z1.transpose(1, 2), kernel_size=2).transpose(1, 2) 193 | z2 = F.max_pool1d(z2.transpose(1, 2), kernel_size=2).transpose(1, 2) 194 | 195 | if z1.size(1) == 1: 196 | if lambda_ != 0: 197 | if soft_instance: 198 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 199 | else: 200 | loss += lambda_ * inst_CL_hard(z1, z2) 201 | d += 1 202 | 203 | return loss / d 204 | 205 | 206 | def hier_CL_soft_interval(z1, z2, soft_labels, tau_temp=2, lambda_=0.5, temporal_unit=0, soft_temporal=False, soft_instance=False): 207 | soft_labels = torch.tensor(soft_labels, device=z1.device) 208 | soft_labels_L, soft_labels_R = dup_matrix(soft_labels) 209 | loss = torch.tensor(0., device=z1.device) 210 | d = 0 211 | while z1.size(1) > 1: 212 | if lambda_ != 0: 213 | if soft_instance: 214 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 215 | else: 216 | loss += lambda_ * inst_CL_hard(z1, z2) 217 | if d >= temporal_unit: 218 | if 1 - lambda_ != 0: 219 | if soft_temporal: 220 | timelag = timelag_same_interval(z1.shape[1],tau_temp/(2**d)) 221 | timelag = torch.tensor(timelag, device=z1.device) 222 | timelag_L, timelag_R = dup_matrix(timelag) 223 | loss += (1 - lambda_) * temp_CL_soft(z1, z2, timelag_L, timelag_R) 224 | else: 225 | loss += (1 - lambda_) * temp_CL_hard(z1, z2) 226 | d += 1 227 | z1 = F.max_pool1d(z1.transpose(1, 2), kernel_size=2).transpose(1, 2) 228 | z2 = F.max_pool1d(z2.transpose(1, 2), kernel_size=2).transpose(1, 2) 229 | 230 | if z1.size(1) == 1: 231 | if lambda_ != 0: 232 | if soft_instance: 233 | loss += lambda_ * inst_CL_soft(z1, z2, soft_labels_L, soft_labels_R) 234 | else: 235 | loss += lambda_ * inst_CL_hard(z1, z2) 236 | d += 1 237 | 238 | return loss / d 239 | 240 | def hier_CL_soft_wo_inst(z1, z2, soft_labels, tau_temp=2, lambda_=0.5, temporal_unit=0, soft_temporal=False, soft_instance=False): 241 | soft_labels = torch.tensor(soft_labels, device=z1.device) 242 | soft_labels_L, soft_labels_R = dup_matrix(soft_labels) 243 | loss = torch.tensor(0., device=z1.device) 244 | d = 0 245 | while z1.size(1) > 1: 246 | if d >= temporal_unit: 247 | if 1 - lambda_ != 0: 248 | if soft_temporal: 249 | timelag = timelag_sigmoid(z1.shape[1],tau_temp*(2**d)) 250 | timelag = torch.tensor(timelag, device=z1.device) 251 | timelag_L, timelag_R = dup_matrix(timelag) 252 | loss += (1 - lambda_) * temp_CL_soft(z1, z2, timelag_L, timelag_R) 253 | else: 254 | loss += (1 - lambda_) * temp_CL_hard(z1, z2) 255 | d += 1 256 | z1 = F.max_pool1d(z1.transpose(1, 2), kernel_size=2).transpose(1, 2) 257 | z2 = F.max_pool1d(z2.transpose(1, 2), kernel_size=2).transpose(1, 2) 258 | 259 | if z1.size(1) == 1: 260 | d += 1 261 | 262 | return loss / d 263 | 264 | -------------------------------------------------------------------------------- /softclt_catcc/trainer/trainer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | sys.path.append("..") 5 | import torch 6 | import torch.nn as nn 7 | 8 | from trainer.train_utils import * 9 | from sklearn.metrics.pairwise import cosine_similarity, euclidean_distances 10 | 11 | # 1) Trainer 12 | # 2) Trainer_wo_DTW 13 | # 3) Trainer_wo_val 14 | 15 | def Trainer(args, DTW, model, temporal_contr_model, model_optimizer, temp_cont_optimizer, train_dl, valid_dl, test_dl, device, 16 | logger, config, experiment_log_dir, training_mode): 17 | logger.debug("Training started ....") 18 | 19 | # (1) Loss Function & LR Scheduler & Epochs 20 | criterion = nn.CrossEntropyLoss() 21 | scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(model_optimizer, 'min') 22 | 23 | if training_mode =='self_supervised': 24 | num_epochs = args.num_epochs 25 | else: 26 | num_epochs = args.load_epoch 27 | ''' 28 | if ('linear' in training_mode)|('tl' in training_mode): 29 | num_epochs = args.num_epochs_linear 30 | else: 31 | num_epochs = args.num_epochs 32 | ''' 33 | # (2) Train 34 | os.makedirs(os.path.join(experiment_log_dir, "saved_models"), exist_ok=True) 35 | for epoch in range(1, num_epochs + 1): 36 | train_loss, train_acc = model_train(DTW, model, temporal_contr_model, model_optimizer, temp_cont_optimizer, 37 | criterion, train_dl, config, device, training_mode, args.lambda_aux) 38 | valid_loss, valid_acc, _, _ = model_evaluate(model, temporal_contr_model, valid_dl, device, training_mode) 39 | 40 | if (training_mode != "self_supervised") and (training_mode != "SupCon"): 41 | scheduler.step(valid_loss) 42 | 43 | logger.debug(f'\nEpoch : {epoch}\n' 44 | f'Train Loss : {train_loss:2.4f}\t | \tTrain Accuracy : {train_acc:2.4f}\n' 45 | f'Valid Loss : {valid_loss:2.4f}\t | \tValid Accuracy : {valid_acc:2.4f}') 46 | 47 | if epoch%args.save_epoch==0: 48 | chkpoint = {'model_state_dict': model.state_dict(), 49 | 'temporal_contr_model_state_dict': temporal_contr_model.state_dict()} 50 | torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", f'ckp_{epoch}.pt')) 51 | #if (training_mode != "self_supervised"): 52 | # torch.save(chkpoint, os.path.join(experiment_log_dir, f'ep_pretrain_{args.load_epoch}',"saved_models", f'ckp_{epoch}.pt')) 53 | #else: 54 | # torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", f'ckp_{epoch}.pt')) 55 | 56 | # (3) Save Results 57 | 58 | chkpoint = {'model_state_dict': model.state_dict(), 59 | 'temporal_contr_model_state_dict': temporal_contr_model.state_dict()} 60 | torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", f'ckp_last.pt')) 61 | #if (training_mode != "self_supervised"): 62 | # torch.save(chkpoint, os.path.join(experiment_log_dir, f'ep_pretrain_{args.load_epoch}',"saved_models", f'ckp_last.pt')) 63 | #else: 64 | # torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", 'ckp_last.pt')) 65 | 66 | # (4) (Optional) Evaluation 67 | if (training_mode != "self_supervised") and (training_mode != "SupCon"): 68 | # evaluate on the test set 69 | logger.debug('\nEvaluate on the Test set:') 70 | test_loss, test_acc, _, _ = model_evaluate(model, temporal_contr_model, test_dl, device, training_mode) 71 | logger.debug(f'Test loss :{test_loss:2.4f}\t | Test Accuracy : {test_acc:2.4f}') 72 | 73 | logger.debug("\n################## Training is Done! #########################") 74 | 75 | def Trainer_wo_DTW(args, model, temporal_contr_model, model_optimizer, temp_cont_optimizer, train_dl, valid_dl, test_dl, device, 76 | logger, config, experiment_log_dir, training_mode): 77 | logger.debug("Training started ....") 78 | 79 | # (1) Loss Function & LR Scheduler & Epochs 80 | criterion = nn.CrossEntropyLoss() 81 | scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(model_optimizer, 'min') 82 | if training_mode =='self_supervised': 83 | num_epochs = args.num_epochs 84 | else: 85 | num_epochs = args.load_epoch 86 | ''' 87 | if ('linear' in training_mode)|('tl' in training_mode): 88 | num_epochs = args.num_epochs_linear 89 | else: 90 | num_epochs = args.num_epochs 91 | ''' 92 | ######################################################## 93 | if args.dist == 'cos': 94 | dist_func = cosine_similarity 95 | elif args.dist == 'euc': 96 | dist_func = euclidean_distances 97 | ######################################################## 98 | 99 | # (2) Train 100 | os.makedirs(os.path.join(experiment_log_dir, "saved_models"), exist_ok=True) 101 | for epoch in range(1, num_epochs + 1): 102 | train_loss, train_acc = model_train_wo_DTW(dist_func, args.dist, args.tau_inst, model, temporal_contr_model, model_optimizer, temp_cont_optimizer, 103 | criterion, train_dl, config, device, training_mode, args.lambda_aux) 104 | 105 | valid_loss, valid_acc, _, _ = model_evaluate(model, temporal_contr_model, valid_dl, device, training_mode) 106 | 107 | if (training_mode != "self_supervised") and (training_mode != "SupCon"): 108 | scheduler.step(valid_loss) 109 | 110 | logger.debug(f'\nEpoch : {epoch}\n' 111 | f'Train Loss : {train_loss:2.4f}\t | \tTrain Accuracy : {train_acc:2.4f}\n' 112 | f'Valid Loss : {valid_loss:2.4f}\t | \tValid Accuracy : {valid_acc:2.4f}') 113 | 114 | if epoch%args.save_epoch==0: 115 | chkpoint = {'model_state_dict': model.state_dict(), 116 | 'temporal_contr_model_state_dict': temporal_contr_model.state_dict()} 117 | #torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", f'ckp_{epoch}.pt')) 118 | #torch.save(chkpoint, os.path.join(experiment_log_dir, f'ep_pretrain_{args.num_epochs}_load_{args.load_epoch}',"saved_models", f'ckp_{epoch}.pt')) 119 | torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", f'ckp_{epoch}.pt')) 120 | #if (training_mode != "self_supervised"): 121 | # torch.save(chkpoint, os.path.join(experiment_log_dir, f'ep_pretrain_{args.load_epoch}',"saved_models", f'ckp_{epoch}.pt')) 122 | #else: 123 | # torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", f'ckp_{epoch}.pt')) 124 | 125 | # (3) Save Results 126 | #os.makedirs(os.path.join(experiment_log_dir, "saved_models"), exist_ok=True) 127 | chkpoint = {'model_state_dict': model.state_dict(), 128 | 'temporal_contr_model_state_dict': temporal_contr_model.state_dict()} 129 | #torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", f'ckp_last.pt')) 130 | #torch.save(chkpoint, os.path.join(experiment_log_dir, f'ep_pretrain_{args.num_epochs}_load_{args.load_epoch}',"saved_models", f'ckp_last.pt')) 131 | torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", f'ckp_last.pt')) 132 | #if (training_mode != "self_supervised"): 133 | # torch.save(chkpoint, os.path.join(experiment_log_dir, f'ep_pretrain_{args.load_epoch}',"saved_models", f'ckp_last.pt')) 134 | #else: 135 | # torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", f'ckp_last.pt')) 136 | 137 | # (4) (Optional) Evaluation 138 | if (training_mode != "self_supervised") and (training_mode != "SupCon"): 139 | # evaluate on the test set 140 | logger.debug('\nEvaluate on the Test set:') 141 | test_loss, test_acc, _, _ = model_evaluate(model, temporal_contr_model, test_dl, device, training_mode) 142 | logger.debug(f'Test loss :{test_loss:2.4f}\t | Test Accuracy : {test_acc:2.4f}') 143 | 144 | logger.debug("\n################## Training is Done! #########################") 145 | 146 | 147 | def Trainer_wo_val(args, DTW, model, temporal_contr_model, model_optimizer, temp_cont_optimizer, train_dl, test_dl, device, 148 | logger, config, experiment_log_dir, training_mode): 149 | logger.debug("Training started ....") 150 | 151 | # (1) Loss Function & LR Scheduler & Epochs 152 | criterion = nn.CrossEntropyLoss() 153 | scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(model_optimizer, 'min') 154 | if training_mode =='self_supervised': 155 | num_epochs = args.num_epochs 156 | else: 157 | num_epochs = args.load_epoch 158 | ''' 159 | if ('linear' in training_mode)|('tl' in training_mode): 160 | num_epochs = args.num_epochs_linear 161 | else: 162 | num_epochs = args.num_epochs 163 | ''' 164 | # (2) Train 165 | os.makedirs(os.path.join(experiment_log_dir, "saved_models"), exist_ok=True) 166 | for epoch in range(1, num_epochs + 1): 167 | train_loss, train_acc = model_train(DTW, model, temporal_contr_model, model_optimizer, temp_cont_optimizer, 168 | criterion, train_dl, config, device, training_mode, args.lambda_aux) 169 | 170 | logger.debug(f'\nEpoch : {epoch}\n' 171 | f'Train Loss : {train_loss:2.4f}\t | \tTrain Accuracy : {train_acc:2.4f}') 172 | 173 | if epoch%args.save_epoch==0: 174 | chkpoint = {'model_state_dict': model.state_dict(), 175 | 'temporal_contr_model_state_dict': temporal_contr_model.state_dict()} 176 | #torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", f'ckp_{epoch}.pt')) 177 | #torch.save(chkpoint, os.path.join(experiment_log_dir, f'ep_pretrain_{args.num_epochs}_load_{args.load_epoch}',"saved_models", f'ckp_{epoch}.pt')) 178 | torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", f'ckp_{epoch}.pt')) 179 | #if (training_mode != "self_supervised"): 180 | # torch.save(chkpoint, os.path.join(experiment_log_dir, f'ep_pretrain_{args.load_epoch}',"saved_models", f'ckp_{epoch}.pt')) 181 | #else: 182 | # torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", f'ckp_{epoch}.pt')) 183 | 184 | # (3) Save Results 185 | #os.makedirs(os.path.join(experiment_log_dir, "saved_models"), exist_ok=True) 186 | chkpoint = {'model_state_dict': model.state_dict(), 187 | 'temporal_contr_model_state_dict': temporal_contr_model.state_dict()} 188 | #torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", f'ckp_last.pt')) 189 | #torch.save(chkpoint, os.path.join(experiment_log_dir, f'ep_pretrain_{args.num_epochs}_load_{args.load_epoch}',"saved_models", f'ckp_last.pt')) 190 | torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", f'ckp_last.pt')) 191 | #if (training_mode != "self_supervised"): 192 | # torch.save(chkpoint, os.path.join(experiment_log_dir, f'ep_pretrain_{args.load_epoch}',"saved_models", f'ckp_last.pt')) 193 | #else: 194 | # torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", f'ckp_last.pt')) 195 | 196 | # (4) (Optional) Evaluation 197 | if (training_mode != "self_supervised") and (training_mode != "SupCon"): 198 | # evaluate on the test set 199 | logger.debug('\nEvaluate on the Test set:') 200 | test_loss, test_acc, _, _ = model_evaluate(model, temporal_contr_model, test_dl, device, training_mode) 201 | logger.debug(f'Test loss :{test_loss:2.4f}\t | Test Accuracy : {test_acc:2.4f}') 202 | 203 | logger.debug("\n################## Training is Done! #########################") 204 | 205 | 206 | def Trainer_wo_DTW_wo_val(args, model, temporal_contr_model, model_optimizer, temp_cont_optimizer, train_dl, test_dl, device, 207 | logger, config, experiment_log_dir, training_mode): 208 | logger.debug("Training started ....") 209 | 210 | # (1) Loss Function & LR Scheduler & Epochs 211 | criterion = nn.CrossEntropyLoss() 212 | scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(model_optimizer, 'min') 213 | if training_mode =='self_supervised': 214 | num_epochs = args.num_epochs 215 | else: 216 | num_epochs = args.num_epochs_finetune 217 | 218 | ######################################################## 219 | if args.dist == 'cos': 220 | dist_func = cosine_similarity 221 | elif args.dist == 'euc': 222 | dist_func = euclidean_distances 223 | ######################################################## 224 | 225 | # (2) Train 226 | os.makedirs(os.path.join(experiment_log_dir, "saved_models"), exist_ok=True) 227 | for epoch in range(1, num_epochs + 1): 228 | train_loss, train_acc = model_train_wo_DTW(dist_func, args.dist, args.tau_inst, model, temporal_contr_model, model_optimizer, temp_cont_optimizer, 229 | criterion, train_dl, config, device, training_mode, args.lambda_aux) 230 | 231 | logger.debug(f'\nEpoch : {epoch}\n' 232 | f'Train Loss : {train_loss:2.4f}\t | \tTrain Accuracy : {train_acc:2.4f}') 233 | 234 | if epoch%args.save_epoch==0: 235 | chkpoint = {'model_state_dict': model.state_dict(), 236 | 'temporal_contr_model_state_dict': temporal_contr_model.state_dict()} 237 | torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", f'ckp_{epoch}.pt')) 238 | 239 | 240 | # (3) Save Results 241 | chkpoint = {'model_state_dict': model.state_dict(), 242 | 'temporal_contr_model_state_dict': temporal_contr_model.state_dict()} 243 | torch.save(chkpoint, os.path.join(experiment_log_dir, "saved_models", f'ckp_last.pt')) 244 | 245 | 246 | # (4) (Optional) Evaluation 247 | if (training_mode != "self_supervised") and (training_mode != "SupCon"): 248 | # evaluate on the test set 249 | logger.debug('\nEvaluate on the Test set:') 250 | test_loss, test_acc, _, _ = model_evaluate(model, temporal_contr_model, test_dl, device, training_mode) 251 | logger.debug(f'Test loss :{test_loss:2.4f}\t | Test Accuracy : {test_acc:2.4f}') 252 | 253 | logger.debug("\n################## Training is Done! #########################") 254 | --------------------------------------------------------------------------------