├── .gitignore ├── dataloaders ├── .DS_Store ├── __init__.py ├── dataloader_MOTION_Sense_har.py ├── dataloader_USCHAD_har.py ├── utils.py └── dataloader_uci_har.py ├── trainers ├── __init__.py └── maskcae_trainer.py ├── models ├── __init__.py ├── MaskedConvNet.py ├── ConvNet.py └── Baseline_CNN.py ├── train.py ├── configs ├── yaml │ ├── maskcaebase.yaml │ ├── maskcaeConv4Net.yaml │ ├── maskcaeConv4Net_LP.yaml │ ├── data.yaml │ └── data_sup.yaml ├── parser.py └── base_configs.py ├── requirements.txt ├── pretrain ├── script │ └── run.py ├── simpletest.py ├── models │ └── __init__.py ├── utils │ ├── lr_control.py │ ├── arg_util.py │ └── lamb.py ├── sampler.py ├── launch.py ├── dist.py ├── decoder.py ├── encoder.py └── main.py ├── scripts └── MaskCAE │ ├── Base.py │ └── supevisedlearning.py ├── common.py ├── utils ├── multi_crop.py ├── logger.py ├── metric.py ├── lamb.py ├── setup.py └── augmentations.py ├── README.md ├── HAR_sklearn.py ├── main.py ├── create_dataset.py └── MaskCAE └── pretrain └── utils ├── lamb.py └── misc.py /.gitignore: -------------------------------------------------------------------------------- 1 | /save 2 | **.pyc** -------------------------------------------------------------------------------- /dataloaders/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cheng-haha/MaskCAE/HEAD/dataloaders/.DS_Store -------------------------------------------------------------------------------- /trainers/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2023-03-18 14:25:30 4 | LastEditTime: 2024-08-13 19:27:46 5 | FilePath: /chengdongzhou/action/MaskCAE/trainers/__init__.py 6 | ''' 7 | from .maskcae_trainer import maskcae_train -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2023-02-06 09:50:31 4 | LastEditTime: 2024-08-13 16:04:12 5 | FilePath: /chengdongzhou/action/MaskCAE/models/__init__.py 6 | ''' 7 | from .ConvNet import Conv4Net,Conv4NetSup 8 | from .Baseline_CNN import BaseCNN,BaseCNNSup 9 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2023-03-18 14:25:30 4 | LastEditTime: 2024-08-13 19:28:11 5 | FilePath: /chengdongzhou/action/MaskCAE/train.py 6 | ''' 7 | from trainers import * 8 | 9 | def MetaTrain(args): 10 | ''' 11 | This is also an index. 12 | ''' 13 | if args.mode in [ 'maskcae','maskcae_diffmodel'] : 14 | return maskcae_train 15 | else: 16 | raise NotImplementedError 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /configs/yaml/maskcaebase.yaml: -------------------------------------------------------------------------------- 1 | 2 | # parameters 3 | epochs: 100 4 | learning_rate: 0.0001 5 | batch_size: 512 6 | device: 1 7 | times: 1 8 | 9 | model: 'BaseCNN' 10 | pre_trained_path: '' 11 | lr_scale: 0 12 | warm_up_epochs: 0 13 | train_portion: 1.0 14 | mask: -1 15 | 16 | # optim 17 | opt_type: 'adamw' 18 | decay_epochs: 20 19 | gamma: 0.1 20 | momentum: 0.9 21 | weight_decay: 0.0 22 | 23 | use_wandb: False 24 | multicrop: False 25 | 26 | chhander: True -------------------------------------------------------------------------------- /configs/yaml/maskcaeConv4Net.yaml: -------------------------------------------------------------------------------- 1 | 2 | # parameters 3 | epochs: 100 4 | learning_rate: 0.0001 5 | batch_size: 512 6 | device: 2 7 | times: 1 8 | 9 | model: 'Conv4Net' 10 | pre_trained_path: './MaskCAE/MaskCAE_EXP/' 11 | lr_scale: 0 12 | warm_up_epochs: 0 13 | train_portion: 1.0 14 | mask: 0.3 15 | # optim 16 | opt_type: 'adamw' 17 | decay_epochs: 20 18 | gamma: 0.1 19 | momentum: 0.9 20 | weight_decay: 0.0 21 | 22 | use_wandb: False 23 | multicrop: False 24 | linear_evaluation: False 25 | chhander: True -------------------------------------------------------------------------------- /configs/yaml/maskcaeConv4Net_LP.yaml: -------------------------------------------------------------------------------- 1 | 2 | # parameters 3 | epochs: 100 4 | learning_rate: 0.0001 5 | batch_size: 512 6 | device: 2 7 | times: 1 8 | 9 | model: 'Conv4Net' 10 | pre_trained_path: './MaskCAE/MaskCAE_EXP/' 11 | lr_scale: 0 12 | warm_up_epochs: 0 13 | train_portion: 1.0 14 | mask: 0.3 15 | # optim 16 | opt_type: 'adamw' 17 | decay_epochs: 20 18 | gamma: 0.1 19 | momentum: 0.9 20 | weight_decay: 0.0 21 | 22 | use_wandb: False 23 | multicrop: False 24 | linear_evaluation: True 25 | chhander: True -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | dataset==1.6.2 2 | einops==0.7.0 3 | h5py==3.10.0 4 | matplotlib==3.7.3 5 | numpy==1.24.4 6 | pandas==2.0.0 7 | pickle5==0.0.11 8 | Pillow 9 | Pillow==10.1.0 10 | plotly==5.18.0 11 | pyfiglet==0.8.post1 12 | pytz==2023.3.post1 13 | PyWavelets==1.4.1 14 | scikit_learn==1.3.2 15 | scipy 16 | seaborn==0.13.0 17 | scikit-image 18 | torch==1.10.0+cu111 19 | torchvision==0.11.0+cu111 20 | tqdm==4.66.1 21 | typed_argument_parser==1.8.1 22 | umap==0.1.1 23 | umap_learn==0.5.4 24 | tap 25 | timm==0.5.4 26 | thop 27 | tensorboard 28 | wandb 29 | tabulate 30 | -------------------------------------------------------------------------------- /pretrain/script/run.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2023-03-22 11:34:07 4 | LastEditTime: 2024-08-13 19:33:58 5 | FilePath: /chengdongzhou/action/MaskCAE/pretrain/script/run.py 6 | ''' 7 | import os 8 | import argparse 9 | parser = argparse.ArgumentParser() 10 | parser.add_argument('--dataset',type=str,default=None) 11 | parser.add_argument('--device',type=int,default=1) 12 | args = parser.parse_args() 13 | mode = 'Base' 14 | mask_radio = [0.3] 15 | ep = 1000 16 | for radio in mask_radio: 17 | os.system( 18 | f'python pretrain/main.py --dataset_name {args.dataset} --mask {radio} --ep {ep} --device {args.device}' 19 | ) 20 | -------------------------------------------------------------------------------- /configs/yaml/data.yaml: -------------------------------------------------------------------------------- 1 | 2 | # exp_mode: LOCV or Given 3 | 4 | uschad: 5 | filename: "USC-HAD" 6 | overlap_rate: 0.5 7 | down_sample: True 8 | sampling_freq: 30 9 | num_classes: 12 10 | num_channels: 6 11 | window_seconds: 1.0 12 | exp_mode: 'Given' 13 | valid_rate: 0.2 14 | 15 | motion: 16 | filename: "MotionSense" 17 | overlap_rate: 0.5 18 | down_sample: True 19 | sampling_freq: 30 20 | num_classes: 6 21 | num_channels: 12 22 | window_seconds: 1.0 23 | exp_mode: 'LOCV' 24 | valid_rate: 0.2 25 | 26 | ucihar: 27 | filename: "UCI HAR Dataset" 28 | overlap_rate: 0.5 29 | down_sample: True 30 | sampling_freq: 30 31 | num_classes: 6 32 | num_channels: 9 33 | window_seconds: 1.0 34 | exp_mode: 'LOCV' 35 | valid_rate: 0.2 36 | -------------------------------------------------------------------------------- /configs/yaml/data_sup.yaml: -------------------------------------------------------------------------------- 1 | 2 | # exp_mode: LOCV or Given 3 | 4 | uschad: 5 | filename: "USC-HAD" 6 | overlap_rate: 0.5 7 | down_sample: True 8 | sampling_freq: 30 9 | num_classes: 12 10 | num_channels: 6 11 | window_seconds: 1.0 12 | exp_mode: 'Given' 13 | valid_rate: 0.2 14 | 15 | motion: 16 | filename: "MotionSense" 17 | overlap_rate: 0.5 18 | down_sample: False 19 | sampling_freq: 50 20 | num_classes: 6 21 | num_channels: 12 22 | window_seconds: 2.56 23 | exp_mode: 'Given' 24 | valid_rate: 0.2 25 | 26 | ucihar: 27 | filename: "UCI HAR Dataset" 28 | overlap_rate: 0.5 29 | down_sample: False 30 | sampling_freq: 50 31 | num_classes: 6 32 | num_channels: 9 33 | window_seconds: 2.56 34 | exp_mode: 'Given' 35 | valid_rate: 0.2 36 | -------------------------------------------------------------------------------- /scripts/MaskCAE/Base.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2023-03-22 11:34:07 4 | LastEditTime: 2024-08-13 17:49:14 5 | FilePath: /chengdongzhou/action/MaskCAE/scripts/MaskCAE/Base.py 6 | ''' 7 | import os 8 | import argparse 9 | parser = argparse.ArgumentParser() 10 | parser.add_argument('--dataset',type=str,default=None) 11 | parser.add_argument('--config',type=str,default=None) 12 | parser.add_argument('--device',type=int,default=1) 13 | parser.add_argument('--times',type=int,default=5) 14 | parser.add_argument('--dataset_pre',type=str,default='self') 15 | 16 | args = parser.parse_args() 17 | mode = 'maskcae' 18 | datasets = ['motion','ucihar','uschad'] 19 | if args.dataset : 20 | assert args.dataset in datasets , 'not support this dataset' 21 | datasets = [ dataset.strip() for dataset in args.dataset.split(',')] 22 | 23 | for dataset in datasets: 24 | os.system( 25 | f'python main.py --dataset {dataset} --mode {mode} --config {args.config} --dataset_pre {args.dataset_pre} --device {args.device} --times {args.times}' 26 | ) 27 | -------------------------------------------------------------------------------- /configs/parser.py: -------------------------------------------------------------------------------- 1 | USABLE_TYPES = set([float, int]) 2 | 3 | 4 | def trim_preceding_hyphens(st): 5 | i = 0 6 | while st[i] == "-": 7 | i += 1 8 | 9 | return st[i:] 10 | 11 | 12 | def arg_to_varname(st: str): 13 | st = trim_preceding_hyphens(st) 14 | st = st.replace("-", "_") 15 | 16 | return st.split("=")[0] 17 | 18 | 19 | def argv_to_vars(argv): 20 | var_names = [] 21 | for arg in argv: 22 | if arg.startswith("-") and arg_to_varname(arg) != "config": 23 | var_names.append(arg_to_varname(arg)) 24 | 25 | return var_names 26 | 27 | 28 | def produce_override_string(args, override_args): 29 | lines = [] 30 | for v in override_args: 31 | if v != "multigpu": 32 | v_arg = getattr(args, v) 33 | if type(v_arg) in USABLE_TYPES: 34 | lines.append(v + ": " + str(v_arg)) 35 | else: 36 | lines.append(v + ": " + f'"{str(v_arg)}"') 37 | else: 38 | lines.append("multigpu: " + str(args.multigpu)) 39 | 40 | return "\n# ===== Overrided ===== #\n" + "\n".join(lines) 41 | -------------------------------------------------------------------------------- /common.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2024-08-13 13:08:07 4 | LastEditTime: 2024-08-13 19:09:44 5 | FilePath: /chengdongzhou/action/MaskCAE/common.py 6 | ''' 7 | 8 | channel_list = { 9 | 'ucihar': [ 16, 32, 64, 128, 6 ], 10 | 'motion': [ 16, 32, 64, 128, 6 ], 11 | 'uschad': [ 16, 32, 64, 128, 12 ], 12 | } 13 | 14 | conv_list = { 15 | 'ucihar': [ (5,1), (1,1), (2,0) ], 16 | 'motion': [ (5,1), (1,1), (2,0) ], 17 | 'uschad': [ (5,1), (1,1), (2,0) ], 18 | 19 | } 20 | 21 | maxp_list = { 22 | 'ucihar': [ (5,1), (2,1) , (2,0) ], 23 | 'motion': [ (5,1), (2,1) , (2,0) ], 24 | 'uschad': [ (5,1), (2,1) , (2,0) ], 25 | } 26 | 27 | first_maxp_list = { 28 | 'ucihar': [ (5,1), (4,1), (2,0) ], 29 | 'uschad': [ (5,1), (4,1), (2,0) ], 30 | 'motion': [ (5,1), (4,1), (2,0) ], 31 | } 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /pretrain/simpletest.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2024-08-13 13:08:14 4 | LastEditTime: 2024-08-13 16:09:47 5 | FilePath: /chengdongzhou/action/MaskCAE/pretrain/simpletest.py 6 | ''' 7 | 8 | import turtle 9 | import torch 10 | import encoder 11 | from decoder import LightDecoder 12 | from models import build_sparse_encoder 13 | from MaskCAE import MaskCAE 14 | class dotdict(dict): 15 | """dot.notation access to dictionary attributes""" 16 | __getattr__ = dict.get 17 | __setattr__ = dict.__setitem__ 18 | __delattr__ = dict.__delitem__ 19 | args = dotdict() 20 | args.model = 'Conv4Net' 21 | args.input_size: turtle = (33,12) 22 | args.sbn = False 23 | args.dp: float = 0.0 24 | args.mask = 0.6 25 | args.densify_norm: str = '' 26 | enc: encoder.SparseEncoder = build_sparse_encoder(args.model, input_size=args.input_size, sbn=args.sbn, drop_path_rate=args.dp, verbose=False) 27 | dec = LightDecoder(enc.downsample_raito, sbn=args.sbn) 28 | model = MaskCAE( 29 | sparse_encoder = enc, 30 | dense_decoder = dec, 31 | mask_ratio = args.mask, 32 | densify_norm = args.densify_norm, 33 | sbn = args.sbn, 34 | ).to(args.device) 35 | inp = torch.randn(2,1,128,12) 36 | loss = model(inp, active_b1fs=None, vis=False) 37 | print(loss) -------------------------------------------------------------------------------- /scripts/MaskCAE/supevisedlearning.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2023-03-22 11:34:07 4 | LastEditTime: 2024-08-13 19:11:15 5 | FilePath: /chengdongzhou/action/MaskCAE/scripts/MaskCAE/supevisedlearning.py 6 | ''' 7 | import os 8 | import argparse 9 | parser = argparse.ArgumentParser() 10 | parser.add_argument('--dataset',type=str,default=None) 11 | parser.add_argument('--config',type=str,default=None) 12 | parser.add_argument('--device',type=int,default=2) 13 | parser.add_argument('--times',type=int,default=5) 14 | parser.add_argument('--dataset_pre',type=str,default='Sup') 15 | 16 | args = parser.parse_args() 17 | mode = 'maskcae_diffmodel' 18 | datasets = ['motion','ucihar','uschad'] 19 | models = ['Transformer_HAR','DeepConvLstmAttn','SA_HAR'] 20 | # models = ['BaseCNNSup','Conv4NetSup'] 21 | if args.dataset : 22 | assert args.dataset in datasets , 'not support this dataset' 23 | datasets = [ dataset.strip() for dataset in args.dataset.split(',')] 24 | 25 | for dataset in datasets: 26 | for model in models: 27 | if model == 'Conv4NetSup': 28 | os.system( 29 | f'python main.py --dataset {dataset} --mode {mode} --config {args.config} --dataset_pre {args.dataset_pre} --model {model} --times {args.times} --mask {0.3}' 30 | ) 31 | 32 | else: 33 | os.system( 34 | f'python main.py --dataset {dataset} --mode {mode} --config {args.config} --dataset_pre {args.dataset_pre} --model {model} --times {args.times}' 35 | ) 36 | -------------------------------------------------------------------------------- /pretrain/models/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2023-06-04 12:00:53 4 | LastEditTime: 2024-08-13 19:42:18 5 | FilePath: /chengdongzhou/action/MaskCAE/pretrain/models/__init__.py 6 | ''' 7 | # Copyright (c) ByteDance, Inc. and its affiliates. 8 | # All rights reserved. 9 | # 10 | # This source code is licensed under the license found in the 11 | # LICENSE file in the root directory of this source tree. 12 | 13 | import torch 14 | from timm import create_model 15 | from timm.loss import SoftTargetCrossEntropy 16 | from timm.models.layers import DropPath 17 | 18 | 19 | from models.custom import ConvNet4, ConvNet3,ConvNetSup 20 | 21 | 22 | # log more 23 | def _ex_repr(self): 24 | return ', '.join( 25 | f'{k}=' + (f'{v:g}' if isinstance(v, float) else str(v)) 26 | for k, v in vars(self).items() 27 | if not k.startswith('_') and k != 'training' 28 | and not isinstance(v, (torch.nn.Module, torch.Tensor)) 29 | ) 30 | for clz in (torch.nn.CrossEntropyLoss, SoftTargetCrossEntropy, DropPath): 31 | if hasattr(clz, 'extra_repr'): 32 | clz.extra_repr = _ex_repr 33 | else: 34 | clz.__repr__ = lambda self: f'{type(self).__name__}({_ex_repr(self)})' 35 | 36 | 37 | pretrain_default_model_kwargs = { 38 | 'Conv4Net': dict(), 39 | 'Conv4NetSup': dict(), 40 | 'Conv3Net': dict(), 41 | } 42 | for kw in pretrain_default_model_kwargs.values(): 43 | kw['pretrained'] = False 44 | kw['num_classes'] = 0 45 | # kw['global_pool'] = '' 46 | 47 | 48 | def build_sparse_encoder(name: str, dataset_name:str ,input_size: tuple, sbn=False, drop_path_rate=0.0, verbose=False,mode=''): 49 | from encoder import SparseEncoder 50 | 51 | kwargs = pretrain_default_model_kwargs[name] 52 | kwargs['data_name'] = dataset_name 53 | if drop_path_rate != 0: 54 | kwargs['drop_path_rate'] = drop_path_rate 55 | print(f'[build_sparse_encoder] model kwargs={kwargs}') 56 | cnn = create_model(name, **kwargs) 57 | return SparseEncoder(cnn, input_size=input_size, sbn=sbn, verbose=verbose,mode=mode) 58 | 59 | -------------------------------------------------------------------------------- /pretrain/utils/lr_control.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) ByteDance, Inc. and its affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import math 8 | from pprint import pformat 9 | 10 | 11 | def lr_wd_annealing(optimizer, peak_lr, wd, wd_end, cur_it, wp_it, max_it): 12 | wp_it = round(wp_it) 13 | if cur_it < wp_it: 14 | cur_lr = 0.005 * peak_lr + 0.995 * peak_lr * cur_it / wp_it 15 | else: 16 | ratio = (cur_it - wp_it) / (max_it - 1 - wp_it) 17 | cur_lr = 0.001 * peak_lr + 0.999 * peak_lr * (0.5 + 0.5 * math.cos(math.pi * ratio)) 18 | 19 | ratio = cur_it / (max_it - 1) 20 | cur_wd = wd_end + (wd - wd_end) * (0.5 + 0.5 * math.cos(math.pi * ratio)) 21 | 22 | min_lr, max_lr = cur_lr, cur_lr 23 | min_wd, max_wd = cur_wd, cur_wd 24 | for param_group in optimizer.param_groups: 25 | scaled_lr = param_group['lr'] = cur_lr * param_group.get('lr_scale', 1) # 'lr_scale' could be assigned 26 | min_lr, max_lr = min(min_lr, scaled_lr), max(max_lr, scaled_lr) 27 | scaled_wd = param_group['weight_decay'] = cur_wd * param_group.get('weight_decay_scale', 1) # 'weight_decay_scale' could be assigned 28 | min_wd, max_wd = min(min_wd, scaled_wd), max(max_wd, scaled_wd) 29 | return min_lr, max_lr, min_wd, max_wd 30 | 31 | 32 | def get_param_groups(model, nowd_keys=()): 33 | para_groups, para_groups_dbg = {}, {} 34 | 35 | for name, para in model.named_parameters(): 36 | if not para.requires_grad: 37 | continue # frozen weights 38 | if len(para.shape) == 1 or name.endswith('.bias') or any(k in name for k in nowd_keys): 39 | wd_scale, group_name = 0., 'no_decay' 40 | else: 41 | wd_scale, group_name = 1., 'decay' 42 | 43 | if group_name not in para_groups: 44 | para_groups[group_name] = {'params': [], 'weight_decay_scale': wd_scale, 'lr_scale': 1.} 45 | para_groups_dbg[group_name] = {'params': [], 'weight_decay_scale': wd_scale, 'lr_scale': 1.} 46 | para_groups[group_name]['params'].append(para) 47 | para_groups_dbg[group_name]['params'].append(name) 48 | 49 | for g in para_groups_dbg.values(): 50 | g['params'] = pformat(', '.join(g['params']), width=200) 51 | 52 | print(f'[get_ft_param_groups] param groups = \n{pformat(para_groups_dbg, indent=2, width=250)}\n') 53 | return list(para_groups.values()) 54 | -------------------------------------------------------------------------------- /utils/multi_crop.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2023-05-23 16:02:49 4 | LastEditTime: 2023-05-30 13:34:05 5 | FilePath: /chengdongzhou/action/tRexHAR/utils/multi_crop.py 6 | ''' 7 | ################################################## 8 | # Multi-crop related code re-used from DINO 9 | # https://github.com/facebookresearch/dino 10 | ################################################## 11 | 12 | import random 13 | 14 | import torch 15 | import torch.nn as nn 16 | import torchvision.transforms as transforms 17 | import numpy as np 18 | from configs.base_configs import args 19 | from utils.augmentations import gen_aug 20 | 21 | 22 | class MultiCropWrapper(nn.Module): 23 | """ 24 | Perform forward pass separately on each resolution input. 25 | The inputs corresponding to a single resolution are clubbed and single 26 | forward is run on the same resolution inputs. Hence we do several 27 | forward passes = number of different resolutions used. We then 28 | concatenate all the output features and run the head forward on these 29 | concatenated features. 30 | """ 31 | 32 | def __init__(self, encoder, head): 33 | super().__init__() 34 | # disable layers dedicated to ImageNet labels classification 35 | self.encoder = encoder 36 | self.head = head 37 | 38 | def forward(self, x ): 39 | # convert to list 40 | if self.training: 41 | if not isinstance(x, list): 42 | x = [x] 43 | idx_crops = torch.cumsum( 44 | torch.unique_consecutive( 45 | torch.tensor([inp.shape[2] for inp in x]), 46 | return_counts=True, 47 | )[1], 48 | 0, 49 | ) 50 | 51 | start_idx, output_list,_cls_out = 0, [torch.empty(0).to(x[0].device) for i in range(3) ] , torch.empty(0).to(x[0].device) 52 | for end_idx in idx_crops: 53 | if start_idx < args.mc_global_number: 54 | _out = self.encoder(torch.cat(x[start_idx:end_idx])) 55 | else: 56 | _out = self.encoder( gen_aug( torch.cat(x[start_idx:end_idx]).cpu().squeeze(1), ssh_type=args.data_aug ).unsqueeze(1).cuda().float() ) 57 | # accumulate outputs 58 | _rep_list, _cls = _out['rep_list'] , _out['cls'] 59 | _cls_out = torch.cat((_cls_out,_cls)) 60 | output_list = [ torch.cat((output_list[x], rep)) for x , rep in enumerate( _rep_list ) ] 61 | start_idx = end_idx 62 | # Run the head forward on the concatenated features 63 | return [self.head(output) for output in output_list ] , _cls_out 64 | else: 65 | return None , self.encoder(x)['cls'] 66 | 67 | 68 | 69 | 70 | if __name__ == '__main__': 71 | x = torch.randn(1,3,224,224) -------------------------------------------------------------------------------- /pretrain/sampler.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) ByteDance, Inc. and its affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import random 8 | 9 | import numpy as np 10 | import torch 11 | from torch.utils.data.sampler import Sampler 12 | 13 | 14 | def worker_init_fn(worker_id): 15 | # https://pytorch.org/docs/stable/notes/randomness.html#dataloader 16 | worker_seed = torch.initial_seed() % 2 ** 32 17 | np.random.seed(worker_seed) 18 | random.seed(worker_seed) 19 | 20 | 21 | class DistInfiniteBatchSampler(Sampler): 22 | def __init__(self, world_size, rank, dataset_len, glb_batch_size, seed=1, filling=False, shuffle=True): 23 | assert glb_batch_size % world_size == 0 24 | self.world_size, self.rank = world_size, rank 25 | self.dataset_len = dataset_len 26 | self.glb_batch_size = glb_batch_size 27 | self.batch_size = glb_batch_size // world_size 28 | 29 | self.iters_per_ep = (dataset_len + glb_batch_size - 1) // glb_batch_size 30 | self.filling = filling 31 | self.shuffle = shuffle 32 | self.epoch = 0 33 | self.seed = seed 34 | self.indices = self.gener_indices() 35 | 36 | def gener_indices(self): 37 | global_max_p = self.iters_per_ep * self.glb_batch_size # global_max_p % world_size must be 0 cuz glb_batch_size % world_size == 0 38 | if self.shuffle: 39 | g = torch.Generator() 40 | g.manual_seed(self.epoch + self.seed) 41 | global_indices = torch.randperm(self.dataset_len, generator=g) 42 | else: 43 | global_indices = torch.arange(self.dataset_len) 44 | filling = global_max_p - global_indices.shape[0] 45 | if filling > 0 and self.filling: 46 | global_indices = torch.cat((global_indices, global_indices[:filling])) 47 | global_indices = tuple(global_indices.numpy().tolist()) 48 | 49 | seps = torch.linspace(0, len(global_indices), self.world_size + 1, dtype=torch.int) 50 | local_indices = global_indices[seps[self.rank]:seps[self.rank + 1]] 51 | self.max_p = len(local_indices) 52 | return local_indices 53 | 54 | def __iter__(self): 55 | self.epoch = 0 56 | while True: 57 | self.epoch += 1 58 | p, q = 0, 0 59 | while p < self.max_p: 60 | q = p + self.batch_size 61 | yield self.indices[p:q] 62 | p = q 63 | if self.shuffle: 64 | self.indices = self.gener_indices() 65 | 66 | def __len__(self): 67 | return self.iters_per_ep 68 | 69 | 70 | if __name__ == '__main__': 71 | W = 16 72 | for rk in range(W): 73 | ind = DistInfiniteBatchSampler(W, rk, 5024, 5024).gener_indices() 74 | print(rk, len(ind)) 75 | -------------------------------------------------------------------------------- /utils/logger.py: -------------------------------------------------------------------------------- 1 | 2 | import logging 3 | import numpy as np 4 | import os 5 | import pathlib 6 | import pandas as pd 7 | import time 8 | from configs.base_configs import args 9 | import h5py 10 | from utils.setup import GetModel 11 | from utils.metric import GetInferredSpeed,GetFlopsAndParams 12 | 13 | import re 14 | 15 | def Statistics(stat, Test_losses, Acc_tests, mF1_tests, wF1_tests , Recall_tests, Precision_tests, sum_time = 0): 16 | stat['time'] = sum_time 17 | stat['Test_losses'] = Test_losses 18 | stat['Acc_tests'] = Acc_tests 19 | stat['mF1_tests'] = mF1_tests 20 | stat['wF1_tests'] = wF1_tests 21 | stat['Recall_tests'] = Recall_tests 22 | stat['Precision_test'] = Precision_tests 23 | return stat 24 | 25 | class AverageMeter(object): 26 | """Computes and stores the average and current value""" 27 | def __init__(self, name, fmt=':f'): 28 | self.name = name 29 | self.fmt = fmt 30 | self.reset() 31 | 32 | def reset(self): 33 | self.val = 0 34 | self.avg = 0 35 | self.sum = 0 36 | self.count = 0 37 | 38 | def update(self, val, n=1): 39 | self.val = val 40 | self.sum += val * n 41 | self.count += n 42 | self.avg = self.sum / self.count 43 | 44 | def __str__(self): 45 | fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' 46 | return fmtstr.format(**self.__dict__) 47 | 48 | 49 | 50 | def initialize_logger(file_dir): 51 | """Print the results in the log file.""" 52 | logger = logging.getLogger() 53 | fhandler = logging.FileHandler(filename=file_dir, mode='a') 54 | chhander = logging.StreamHandler() if args.chhander else None 55 | formatter = logging.Formatter(fmt='[%(asctime)s] %(message)s', 56 | datefmt ='%m-%d %H:%M') 57 | fhandler.setFormatter(formatter) 58 | chhander.setFormatter(formatter) if args.chhander else None 59 | logger.addHandler(fhandler) 60 | logger.addHandler(chhander) if args.chhander else None 61 | logger.setLevel(logging.INFO) 62 | return logger 63 | 64 | 65 | def record_result(result, epoch, acc, mf1,wf1 ,recall , precision, c_mat , record_flag = 0): 66 | """ Record evaluation results.""" 67 | if record_flag == 0: 68 | result.write('Best validation epoch | accuracy: {:.4f}, mF1: {:.4f}, wF1: {:.4f}, Rec: {:.4f}, Pre: {:.4f} (at epoch {})\n'.format(acc, mf1, wf1 ,recall,precision, epoch)) 69 | elif record_flag == -1: 70 | result.write('\n\nTest (Best) | accuracy: {:.4f}, mF1: {:.4f}, wF1: {:.4f}, Rec: {:.4f}, Pre: {:.4f}\n'.format(acc, mf1,wf1,recall , precision, epoch)) 71 | result.write(np.array2string(c_mat)) 72 | result.flush() 73 | result.close 74 | elif record_flag == -2: 75 | result.write('\n\nTest (Final) | accuracy: {:.4f}, mF1: {:.4f}, wF1: {:.4f}, Rec: {:.4f}, Pre: {:.4f}\n'.format(acc, mf1,wf1,recall , precision, epoch)) 76 | elif record_flag == -3: 77 | result.write('\n\nFinal validation epoch | accuracy: {:.4f}, mF1: {:.4f}, wF1: {:.4f}, Rec: {:.4f}, Pre: {:.4f}\n'.format(acc, mf1,wf1,recall , precision, epoch)) 78 | 79 | 80 | -------------------------------------------------------------------------------- /pretrain/launch.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2024-08-13 13:08:14 4 | LastEditTime: 2024-08-13 17:38:42 5 | FilePath: /chengdongzhou/action/MaskCAE/pretrain/launch.py 6 | ''' 7 | import argparse 8 | import functools 9 | import os 10 | import socket 11 | import subprocess 12 | import sys 13 | from typing import List 14 | 15 | os_system = functools.partial(subprocess.call, shell=True) 16 | echo = lambda info: os_system(f'echo "[$(date "+%m-%d-%H:%M:%S")] ({os.path.basename(sys._getframe().f_back.f_code.co_filename)}, line{sys._getframe().f_back.f_lineno})=> {info}"') 17 | 18 | 19 | def __find_free_port(): 20 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 21 | sock.bind(("", 0)) 22 | port = sock.getsockname()[1] 23 | sock.close() 24 | return port 25 | 26 | 27 | if __name__ == '__main__': 28 | parser = argparse.ArgumentParser(description='PyTorch Distributed Launcher') 29 | parser.add_argument('--main_py_relpath', type=str, default='main.py', 30 | help='specify launcher script.') 31 | 32 | # distributed environment 33 | parser.add_argument('--num_nodes', type=int, default=1) 34 | parser.add_argument('--ngpu_per_node', type=int, default=1) 35 | parser.add_argument('--node_rank', type=int, default=0, 36 | help='node rank, ranged from 0 to [dist_num_nodes]-1') 37 | parser.add_argument('--master_address', type=str, default='128.0.0.0', 38 | help='master address for distributed communication') 39 | parser.add_argument('--master_port', type=int, default=30001, 40 | help='master port for distributed communication') 41 | 42 | args_for_this, args_for_python = parser.parse_known_args() 43 | args_for_python: List[str] 44 | 45 | echo(f'[initial args_for_python]: {args_for_python}') 46 | # auto-complete: update args like `--sbn` to `--sbn=1` 47 | kwargs = args_for_python[-1] 48 | kwargs = '='.join(map(str.strip, kwargs.split('='))) 49 | kwargs = kwargs.split(' ') 50 | for i, a in enumerate(kwargs): 51 | if len(a) and '=' not in a: 52 | kwargs[i] = f'{a}=1' 53 | args_for_python[-1] = ' '.join(kwargs) 54 | echo(f'[final args_for_python]: {args_for_python}') 55 | 56 | if args_for_this.num_nodes > 1: # distributed 57 | os.environ['NPROC_PER_NODE'] = str(args_for_this.ngpu_per_node) 58 | cmd = ( 59 | f'python3 -m torch.distributed.launch' 60 | f' --nnodes={args_for_this.num_nodes}' 61 | f' --nproc_per_node={args_for_this.ngpu_per_node}' 62 | f' --node_rank={args_for_this.node_rank}' 63 | f' --master_addr={args_for_this.master_address}' 64 | f' --master_port={args_for_this.master_port}' 65 | f' {args_for_this.main_py_relpath}' 66 | f' {" ".join(args_for_python)}' 67 | ) 68 | else: # single machine with multiple GPUs 69 | cmd = ( 70 | f'python3 -m torch.distributed.launch' 71 | f' --nproc_per_node={args_for_this.ngpu_per_node}' 72 | f' --master_port={__find_free_port()}' 73 | f' {args_for_this.main_py_relpath}' 74 | f' {" ".join(args_for_python)}' 75 | ) 76 | 77 | exit_code = subprocess.call(cmd, shell=True) 78 | sys.exit(exit_code) 79 | -------------------------------------------------------------------------------- /pretrain/dist.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) ByteDance, Inc. and its affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import os 8 | from typing import List 9 | from typing import Union 10 | 11 | import sys 12 | import torch 13 | import torch.distributed as tdist 14 | import torch.multiprocessing as mp 15 | 16 | __rank, __local_rank, __world_size, __device = 0, 0, 1, 'cpu' 17 | __initialized = False 18 | 19 | 20 | def initialized(): 21 | return __initialized 22 | 23 | 24 | def initialize(backend='nccl'): 25 | global __device 26 | if not torch.cuda.is_available(): 27 | print(f'[dist initialize] cuda is not available, use cpu instead', file=sys.stderr) 28 | return 29 | elif 'RANK' not in os.environ: 30 | __device = torch.empty(1).cuda().device 31 | print(f'[dist initialize] RANK is not set, use 1 GPU instead', file=sys.stderr) 32 | return 33 | 34 | # ref: https://github.com/open-mmlab/mmcv/blob/master/mmcv/runner/dist_utils.py#L29 35 | if mp.get_start_method(allow_none=True) is None: 36 | mp.set_start_method('spawn') 37 | global_rank, num_gpus = int(os.environ['RANK']), torch.cuda.device_count() 38 | local_rank = global_rank % num_gpus 39 | torch.cuda.set_device(local_rank) 40 | tdist.init_process_group(backend=backend) 41 | 42 | global __rank, __local_rank, __world_size, __initialized 43 | __local_rank = local_rank 44 | __rank, __world_size = tdist.get_rank(), tdist.get_world_size() 45 | __device = torch.empty(1).cuda().device 46 | __initialized = True 47 | 48 | assert tdist.is_initialized(), 'torch.distributed is not initialized!' 49 | 50 | 51 | def get_rank(): 52 | return __rank 53 | 54 | 55 | def get_local_rank(): 56 | return __local_rank 57 | 58 | 59 | def get_world_size(): 60 | return __world_size 61 | 62 | 63 | def get_device(): 64 | return __device 65 | 66 | 67 | def is_master(): 68 | return __rank == 0 69 | 70 | 71 | def is_local_master(): 72 | return __local_rank == 0 73 | 74 | 75 | def barrier(): 76 | if __initialized: 77 | tdist.barrier() 78 | 79 | 80 | def parallelize(net, syncbn=False): 81 | if syncbn: 82 | net = torch.nn.SyncBatchNorm.convert_sync_batchnorm(net) 83 | net = net.cuda() 84 | net = torch.nn.parallel.DistributedDataParallel(net, device_ids=[get_local_rank()], find_unused_parameters=False, broadcast_buffers=False) 85 | return net 86 | 87 | 88 | def allreduce(t: torch.Tensor) -> None: 89 | if __initialized: 90 | if not t.is_cuda: 91 | cu = t.detach().cuda() 92 | tdist.all_reduce(cu) 93 | t.copy_(cu.cpu()) 94 | else: 95 | tdist.all_reduce(t) 96 | 97 | 98 | def allgather(t: torch.Tensor, cat=True) -> Union[List[torch.Tensor], torch.Tensor]: 99 | if __initialized: 100 | if not t.is_cuda: 101 | t = t.cuda() 102 | ls = [torch.empty_like(t) for _ in range(__world_size)] 103 | tdist.all_gather(ls, t) 104 | else: 105 | ls = [t] 106 | if cat: 107 | ls = torch.cat(ls, dim=0) 108 | return ls 109 | 110 | 111 | def broadcast(t: torch.Tensor, src_rank) -> None: 112 | if __initialized: 113 | if not t.is_cuda: 114 | cu = t.detach().cuda() 115 | tdist.broadcast(cu, src=src_rank) 116 | t.copy_(cu.cpu()) 117 | else: 118 | tdist.broadcast(t, src=src_rank) 119 | -------------------------------------------------------------------------------- /models/MaskedConvNet.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2024-08-13 13:08:13 4 | LastEditTime: 2024-08-13 13:14:10 5 | FilePath: /chengdongzhou/action/MaskCAE/models/MaskedConvNet.py 6 | ''' 7 | import os 8 | import sys 9 | sys.path.append(os.path.dirname(os.path.dirname(__file__))) 10 | 11 | import torch 12 | import torch.nn as nn 13 | from typing import List 14 | from common import channel_list,conv_list,maxp_list,first_maxp_list 15 | from utils.metric import GetFeatureMapSize 16 | from configs.base_configs import args 17 | 18 | class ConvNet(nn.Module): 19 | def __init__(self, data_name='ucihar' ,use_adativeavg=False, sub_number = 4 ): 20 | super(ConvNet, self).__init__() 21 | self.channel= channel = channel_list[data_name] 22 | conv_params = conv_list[data_name] 23 | max_params = maxp_list[data_name] 24 | first_max = first_maxp_list[data_name] 25 | 26 | self.layer1 = self.maker_layers( 1 , channel[0], conv_params=conv_params, pooling_params=first_max ) 27 | self.layer2 = self.maker_layers( channel[0], channel[1], conv_params=conv_params, pooling_params=max_params ) 28 | self.layer3 = self.maker_layers( channel[1], channel[2], conv_params=conv_params, pooling_params=max_params ) 29 | self.layer4 = self.maker_layers( channel[2], channel[3], conv_params=conv_params, pooling_params=max_params ) 30 | h,w = GetFeatureMapSize(data_name,sub_number) 31 | self.adaptiveAvg = nn.AdaptiveAvgPool2d( ( 1, w ) ) if use_adativeavg else nn.Identity() 32 | self.classifier = nn.Linear( channel[2]*w if use_adativeavg else channel[2]* h * w , channel[-1] ) 33 | 34 | def maker_layers(self,inp,oup,conv_params=None,pooling_params=None): 35 | assert isinstance(conv_params,list),print('the format of kernel params is error') 36 | assert isinstance(pooling_params,list) or pooling_params == None ,print('the format of pooling params is error') 37 | return nn.Sequential( 38 | nn.Conv2d( inp, oup, *(conv_params) ), 39 | nn.BatchNorm2d( oup ), 40 | nn.ReLU( True ), 41 | nn.MaxPool2d( *(pooling_params) ) if pooling_params else nn.Identity() 42 | ) 43 | 44 | def get_downsample_ratio(self) -> int: 45 | return 32 46 | 47 | def get_feature_map_channels(self) -> List[int]: 48 | return self.channel[:-1] 49 | 50 | def forward(self, x: torch.Tensor, hierarchical=False): 51 | B,_,_,_ = x.size() 52 | if hierarchical: 53 | ls = [] 54 | x = self.layer1(x); ls.append(x) 55 | x = self.layer2(x); ls.append(x) 56 | x = self.layer3(x); ls.append(x) 57 | x = self.layer4(x); ls.append(x) 58 | return ls 59 | else: 60 | x = self.adaptiveAvg(x) 61 | x = x.view(B,-1) 62 | x = self.classifier( x ) 63 | return x 64 | 65 | 66 | @torch.no_grad() 67 | def convnet_test(): 68 | cnn = ConvNet() 69 | print('get_downsample_ratio:', cnn.get_downsample_ratio()) 70 | print('get_feature_map_channels:', cnn.get_feature_map_channels()) 71 | 72 | downsample_ratio = cnn.get_downsample_ratio() 73 | feature_map_channels = cnn.get_feature_map_channels() 74 | 75 | # check the forward function 76 | B, C, H, W = 4, 1, 128, 9 77 | inp = torch.rand(B, C, H, W) 78 | feats = cnn(inp, hierarchical=True) 79 | assert isinstance(feats, list) 80 | assert len(feats) == len(feature_map_channels) 81 | print([tuple(t.shape) for t in feats]) 82 | 83 | # check the downsample ratio 84 | feats = cnn(inp, hierarchical=True) 85 | assert feats[-1].shape[-2] == H // downsample_ratio 86 | assert feats[-1].shape[-1] == W // 1 87 | 88 | # check the channel number 89 | for feat, ch in zip(feats, feature_map_channels): 90 | assert feat.ndim == 4 91 | assert feat.shape[1] == ch 92 | 93 | 94 | if __name__ == '__main__': 95 | convnet_test() 96 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

🌟MaskCAE: Masked Convolutional AutoEncoder for HAR

2 |

Paper

3 |

Dongzhou Cheng

4 | 5 | ## 🔆🚀If you use this project, please cite our papers. 6 | ``` 7 | @article{cheng2024maskcae, 8 | title={MaskCAE: Masked Convolutional AutoEncoder via Sensor Data Reconstruction for Self-Supervised Human Activity Recognition}, 9 | author={Cheng, Dongzhou and Zhang, Lei and Qin, Lutong and Wang, Shuoyuan and Wu, Hao and Song, Aiguo}, 10 | journal={IEEE Journal of Biomedical and Health Informatics}, 11 | year={2024}, 12 | publisher={IEEE} 13 | } 14 | ``` 15 | ``` 16 | @article{cheng2023protohar, 17 | title={Protohar: Prototype guided personalized federated learning for human activity recognition}, 18 | author={Cheng, Dongzhou and Zhang, Lei and Bu, Can and Wang, Xing and Wu, Hao and Song, Aiguo}, 19 | journal={IEEE Journal of Biomedical and Health Informatics}, 20 | volume={27}, 21 | number={8}, 22 | pages={3900--3911}, 23 | year={2023}, 24 | publisher={IEEE} 25 | } 26 | ``` 27 | ``` 28 | @article{cheng2023learning, 29 | title={Learning hierarchical time series data augmentation invariances via contrastive supervision for human activity recognition}, 30 | author={Cheng, Dongzhou and Zhang, Lei and Bu, Can and Wu, Hao and Song, Aiguo}, 31 | journal={Knowledge-Based Systems}, 32 | volume={276}, 33 | pages={110789}, 34 | year={2023}, 35 | publisher={Elsevier} 36 | } 37 | 38 | ``` 39 | 40 | 41 | # Pretrain environment 42 | ``` 43 | conda activate -n maskcae python=3.8 44 | 45 | python -m pip install torch==1.10.0+cu111 -f https://download.pytorch.org/whl/cu111/torch_stable.html 46 | 47 | python -m pip install torchvision==0.11.0+cu111 -f https://download.pytorch.org/whl/cu111/torch_stable.html 48 | 49 | python -m pip uninstall setuptools 50 | 51 | python -m pip install setuptools==59.5.0 52 | 53 | python -m pip install -r requirements.txt 54 | ``` 55 | Activate your environment 56 | ``` 57 | conda activate maskcae 58 | ``` 59 | 60 | ## Settings 61 | 1. enter the folder 62 | ``` 63 | cd /MaskCAE 64 | ``` 65 | 2. check the yaml configs: `pre_trained_path`, this is the path to save the pre-trained weights 66 | 3. check the `WINDOWS_SAVE_PATH` and `ROOT_PATH` 67 | * `WINDOWS_SAVE_PATH` is the path to save the pre-processed data 68 | * `ROOT_PATH` is the path to save the raw dataset 69 | 70 | ### Get all pretrained weights 71 | ``` 72 | python pretrain/script/run.py --dataset ucihar 73 | 74 | python pretrain/script/run.py --dataset uschad 75 | 76 | python pretrain/script/run.py --dataset motion 77 | ``` 78 | 79 | 80 | # Self-Supervised Learning 81 | *FCN Base (without MaskCAE)* 82 | ``` 83 | # ucihar LOCV mode 84 | python scripts/MaskCAE/Base.py --config configs/yaml/maskcaebase.yaml --dataset ucihar --device 0 --times 1 85 | 86 | # uschad Given mode 87 | python scripts/MaskCAE/Base.py --config configs/yaml/maskcaebase.yaml --dataset uschad --device 2 88 | 89 | # motion LOCV mode 90 | python scripts/MaskCAE/Base.py --config configs/yaml/maskcaebase.yaml --dataset motion --device 1 --times 1 91 | ``` 92 | *FCN fully finetune* 93 | ``` 94 | # ucihar LOCV mode 95 | python scripts/MaskCAE/Base.py --config configs/yaml/maskcaeConv4Net.yaml --dataset ucihar --device 2 --times 1 96 | 97 | # uschad Given mode 98 | python scripts/MaskCAE/Base.py --config configs/yaml/maskcaeConv4Net.yaml --dataset uschad --device 1 99 | 100 | # motion LOCV mode 101 | python scripts/MaskCAE/Base.py --config configs/yaml/maskcaeConv4Net.yaml --dataset motion --device 2 --times 1 102 | ``` 103 | *FCN Linear Evalution* 104 | ``` 105 | # ucihar LOCV mode 106 | python scripts/MaskCAE/Base.py --config configs/yaml/maskcaeConv4Net_LP.yaml --dataset ucihar --device 0 --times 1 107 | 108 | # uschad Given mode 109 | python scripts/MaskCAE/Base.py --config configs/yaml/maskcaeConv4Net_LP.yaml --dataset uschad --device 1 110 | 111 | # motion LOCV mode 112 | python scripts/MaskCAE/Base.py --config configs/yaml/maskcaeConv4Net_LP.yaml --dataset motion --device 2 --times 1 113 | 114 | ``` 115 | -------------------------------------------------------------------------------- /HAR_sklearn.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import numpy as np 4 | import pandas as pd 5 | import matplotlib.pyplot as plt 6 | import sklearn 7 | from sklearn import metrics 8 | from sklearn import svm 9 | from sklearn.neural_network import MLPClassifier 10 | from sklearn.naive_bayes import GaussianNB 11 | from sklearn.neighbors import KNeighborsClassifier 12 | from sklearn.ensemble import BaggingClassifier, RandomForestClassifier, AdaBoostClassifier 13 | from sklearn.metrics import f1_score 14 | from sklearn.manifold import TSNE 15 | import matplotlib.pyplot as plt 16 | import seaborn as sns 17 | import pandas as pd 18 | 19 | from dataset.HAR_dataset import HARDataset 20 | 21 | def read_data(file): 22 | data = pd.read_csv(file) 23 | 24 | # suffle data 25 | data = sklearn.utils.shuffle(data) 26 | 27 | X_data = data.drop(['subject', 'Activity', 'ActivityName'], axis=1) 28 | y_data = data.ActivityName 29 | 30 | return np.array(X_data), np.array(y_data) 31 | 32 | def train_model(train_x, train_y, model_name='NaiveBayes', validation=None): 33 | """ 34 | Possible model names: ['NaiveBayes', 'SVM', 'MLP', 'AdaBoost', 'BAG', 'RandomForest'] 35 | default = 'NB' 36 | 37 | validation: (val_x, val_y) tupple for validation accuracy score. 38 | 39 | return: trained model 40 | """ 41 | model = None 42 | if model_name == 'SVM': 43 | model = svm.SVC(gamma='scale', probability=True) 44 | elif model_name == 'MLP': 45 | model = MLPClassifier(hidden_layer_sizes=(100,100), max_iter=200, alpha=0.0001, 46 | solver='sgd', verbose=10, tol=0.000000001) 47 | elif model_name == 'AdaBoost': 48 | model = AdaBoostClassifier(n_estimators=50) 49 | elif model_name == 'BAG': 50 | model = BaggingClassifier(n_jobs=2, n_estimators=50) 51 | elif model_name == 'RandomForest': 52 | model = RandomForestClassifier(n_estimators=200, max_depth=10) 53 | elif model_name == 'KNN': 54 | model = KNeighborsClassifier(n_neighbors=5, weights='distance', algorithm='auto', leaf_size=30, p=2, metric='minkowski', metric_params=None, n_jobs=None) 55 | elif model_name == 'NaiveBayes': 56 | model = GaussianNB() 57 | else: 58 | raise RuntimeError(f"{model_name} not supported") 59 | model.fit(train_x, train_y) 60 | 61 | if validation is not None: 62 | y_hat = model.predict(validation[0]) 63 | acc = metrics.accuracy_score(validation[1], y_hat) 64 | print(f"Validation Accuracy in '{model_name}' = {acc}") 65 | cm = metrics.confusion_matrix(validation[1], y_hat) 66 | print(cm) 67 | f1 = f1_score(validation[1], y_hat, average='weighted') 68 | print(f"F1 Score in '{model_name}' = {f1}") 69 | 70 | return model 71 | 72 | def visualize(data, label): 73 | bs, nc, ws = data.shape 74 | 75 | accel = data[:, :nc // 2, :].reshape(bs, -1) 76 | gyro = data[:, nc//2:, :].reshape(bs, -1) 77 | 78 | tsne = TSNE(n_components=2) 79 | tsne_feat = tsne.fit_transform(np.concatenate((accel, gyro))) 80 | df = pd.DataFrame() 81 | df["y"] = label 82 | df["accel_comp-1"] = tsne_feat[:len(label), 0] 83 | df["accel_comp-2"] = tsne_feat[:len(label), 1] 84 | df["gyro_comp-1"] = tsne_feat[len(label):, 0] 85 | df["gyro_comp-2"] = tsne_feat[len(label):, 1] 86 | 87 | fig, ax = plt.subplots(ncols=2, sharex=True, sharey=True, figsize=(16, 8)) 88 | sns.scatterplot(x="accel_comp-1", y="accel_comp-2", hue=df.y.tolist(), 89 | palette=sns.color_palette("hls", 6), 90 | data=df, ax=ax[0]).set(title="Accel feature T-SNE projection") 91 | sns.scatterplot(x="gyro_comp-1", y="gyro_comp-2", hue=df.y.tolist(), 92 | palette=sns.color_palette("hls", 6), 93 | data=df, ax=ax[1]).set(title="Gyro feature T-SNE projection") 94 | plt.savefig("tsne_visualize/UCI_HAR_raw.png") 95 | 96 | if __name__ == "__main__": 97 | train = HARDataset('UCI_HAR', split='train', window_width=128) 98 | test = HARDataset('UCI_HAR', split='test', window_width=128) 99 | print(len(train)) 100 | print(len(test)) 101 | # train_model(train.data.reshape(train.data.shape[0], -1), train.label, model_name='RandomForest', validation=(test.data.reshape(test.data.shape[0], -1), test.label)) 102 | visualize(test.data, test.label) -------------------------------------------------------------------------------- /pretrain/decoder.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2023-06-04 12:00:53 4 | LastEditTime: 2024-08-13 13:17:13 5 | FilePath: /chengdongzhou/action/MaskCAE/pretrain/decoder.py 6 | ''' 7 | import math 8 | from typing import List 9 | 10 | import torch 11 | import torch.nn as nn 12 | from timm.models.layers import trunc_normal_ 13 | 14 | from utils.misc import is_pow2n 15 | 16 | 17 | class UNetBlock(nn.Module): 18 | def __init__(self, cin, cout, bn2d,mode): 19 | """ 20 | a UNet block with 2x up sampling 21 | """ 22 | super().__init__() 23 | self.up_sample = nn.ConvTranspose2d(cin, cin, kernel_size=(4,1), stride=(2,1), padding=(1,0), bias=True) 24 | if mode == 'PlainDecoder': 25 | self.conv = nn.Sequential( 26 | nn.Conv2d(cin, cin, kernel_size=(5,1), stride=(1,1), padding=(2,0), bias=False), bn2d(cin), 27 | nn.ReLU6(inplace=True), 28 | nn.Conv2d(cin, cout, kernel_size=(5,1), stride=(1,1), padding=(2,0), bias=False), bn2d(cout), 29 | ) 30 | elif mode == '9x9IFD': 31 | self.conv = nn.Sequential( 32 | nn.Conv2d(cin, cin, kernel_size=9, stride=1, padding=4, bias=False), bn2d(cin), 33 | nn.ReLU6(inplace=True), 34 | nn.Conv2d(cin, cout, kernel_size=9, stride=1, padding=4, bias=False), bn2d(cout), 35 | ) 36 | elif mode == '7x7IFD': 37 | self.conv = nn.Sequential( 38 | nn.Conv2d(cin, cin, kernel_size=7, stride=1, padding=3, bias=False), bn2d(cin), 39 | nn.ReLU6(inplace=True), 40 | nn.Conv2d(cin, cout, kernel_size=7, stride=1, padding=3, bias=False), bn2d(cout), 41 | ) 42 | elif mode == '3x3IFD': 43 | self.conv = nn.Sequential( 44 | nn.Conv2d(cin, cin, kernel_size=3, stride=1, padding=1, bias=False), bn2d(cin), 45 | nn.ReLU6(inplace=True), 46 | nn.Conv2d(cin, cout, kernel_size=3, stride=1, padding=1, bias=False), bn2d(cout), 47 | ) 48 | else: 49 | self.conv = nn.Sequential( 50 | nn.Conv2d(cin, cin, kernel_size=5, stride=1, padding=2, bias=False), bn2d(cin), 51 | nn.ReLU6(inplace=True), 52 | nn.Conv2d(cin, cout, kernel_size=5, stride=1, padding=2, bias=False), bn2d(cout), 53 | ) 54 | def forward(self, x): 55 | x = self.up_sample(x) 56 | return self.conv(x) 57 | 58 | 59 | class LightDecoder(nn.Module): 60 | def __init__(self, up_sample_ratio, width=256, sbn=False,mode=''): 61 | super().__init__() 62 | self.width = width 63 | self.mode = mode 64 | assert is_pow2n(up_sample_ratio) 65 | n = round(math.log2(up_sample_ratio)) 66 | channels = [self.width // 2 ** i for i in range(n + 1)] 67 | bn2d = nn.SyncBatchNorm if sbn else nn.BatchNorm2d 68 | self.dec = nn.ModuleList([UNetBlock(cin, cout, bn2d, mode) for (cin, cout) in zip(channels[:-1], channels[1:])]) 69 | self.proj = nn.Conv2d(channels[-1], 1, kernel_size=1, stride=1, bias=True) 70 | 71 | self.initialize() 72 | 73 | def forward(self, to_dec: List[torch.Tensor]): 74 | x = 0 75 | for i, d in enumerate(self.dec): 76 | if i < len(to_dec) and to_dec[i] is not None: 77 | x = x + to_dec[i] 78 | x = self.dec[i](x) 79 | return self.proj(x) 80 | 81 | def extra_repr(self) -> str: 82 | return f'width={self.width}' 83 | 84 | def initialize(self): 85 | for m in self.modules(): 86 | if isinstance(m, nn.Linear): 87 | trunc_normal_(m.weight, std=.02) 88 | if m.bias is not None: 89 | nn.init.constant_(m.bias, 0) 90 | elif isinstance(m, nn.Conv2d): 91 | trunc_normal_(m.weight, std=.02) 92 | if m.bias is not None: 93 | nn.init.constant_(m.bias, 0) 94 | elif isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)): 95 | nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') 96 | if m.bias is not None: 97 | nn.init.constant_(m.bias, 0.) 98 | elif isinstance(m, (nn.LayerNorm, nn.BatchNorm1d, nn.BatchNorm2d, nn.SyncBatchNorm)): 99 | nn.init.constant_(m.bias, 0) 100 | nn.init.constant_(m.weight, 1.0) 101 | -------------------------------------------------------------------------------- /pretrain/utils/arg_util.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) ByteDance, Inc. and its affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import json 8 | import os 9 | import sys 10 | 11 | from tap import Tap 12 | import dist 13 | 14 | 15 | class Args(Tap): 16 | # environment 17 | exp_name: str = 'MaskCAE' 18 | exp_dir: str = 'MaskCAE_EXP' # will be created if not exists 19 | dataset_name: str = '' 20 | resume_from: str = '' # resume from some checkpoint.pth 21 | DATASET_PRE:str = 'self' 22 | # MaskCAE hyperparameters 23 | mask: float = None # mask ratio, should be in (0, 1) 24 | is_sup: bool = False 25 | # encoder hyperparameters 26 | model: str = 'Conv4Net' 27 | sbn: bool = False 28 | device: str = '1' 29 | # data hyperparameters 30 | bs: int = 512 31 | dataloader_workers: int = 8 32 | ablation_mode:str = '' 33 | # pre-training hyperparameters 34 | dp: float = 0.0 35 | base_lr: float = 2e-3 36 | wd: float = 0.04 37 | wde: float = 0.2 38 | ep: int = 1000 39 | wp_ep: int = 40 40 | clip: int = 5. 41 | opt: str = 'lamb' 42 | ada: float = 0. 43 | 44 | # NO NEED TO SPECIFIED; each of these args would be updated in runtime automatically 45 | lr: float = None 46 | batch_size_per_gpu: int = 0 47 | glb_batch_size: int = 0 48 | densify_norm: str = '' 49 | 50 | local_rank: int = 0 51 | cmd: str = ' '.join(sys.argv[1:]) 52 | commit_id: str = os.popen(f'git rev-parse HEAD').read().strip() or '[unknown]' 53 | commit_msg: str = (os.popen(f'git log -1').read().strip().splitlines() or ['[unknown]'])[-1].strip() 54 | last_loss: float = 0. 55 | cur_ep: str = '' 56 | remain_time: str = '' 57 | finish_time: str = '' 58 | first_logging: bool = True 59 | log_txt_name: str = '{args.exp_dir}/pretrain_log.txt' 60 | tb_lg_dir: str = '' # tensorboard log directory 61 | 62 | @property 63 | def is_convnet(self): 64 | return 'Conv4Net' in self.model 65 | 66 | def log_epoch(self): 67 | if not dist.is_local_master(): 68 | return 69 | 70 | if self.first_logging: 71 | self.first_logging = False 72 | with open(self.log_txt_name, 'w') as fp: 73 | json.dump({ 74 | 'name': self.exp_name, 'cmd': self.cmd, 'git_commit_id': self.commit_id, 'git_commit_msg': self.commit_msg, 75 | 'model': self.model, 76 | }, fp) 77 | fp.write('\n\n') 78 | 79 | with open(self.log_txt_name, 'a') as fp: 80 | json.dump({ 81 | 'cur_ep': self.cur_ep, 82 | 'last_L': self.last_loss, 83 | 'rema': self.remain_time, 'fini': self.finish_time, 84 | }, fp) 85 | fp.write('\n') 86 | 87 | 88 | def init_dist_and_get_args(): 89 | from utils import misc 90 | 91 | # initialize 92 | args = Args(explicit_bool=True).parse_args() 93 | e = os.path.abspath(args.exp_dir) 94 | d, e = os.path.dirname(e), os.path.basename(e) 95 | e = ''.join(ch if (ch.isalnum() or ch == '-') else '_' for ch in e) 96 | args.exp_dir = os.path.join(d, e) 97 | 98 | os.makedirs(args.exp_dir, exist_ok=True) 99 | args.log_txt_name = os.path.join(args.exp_dir, 'pretrain_log.txt') 100 | args.tb_lg_dir = args.tb_lg_dir or os.path.join(args.exp_dir, 'tensorboard_log') 101 | try: 102 | os.makedirs(args.tb_lg_dir, exist_ok=True) 103 | except: 104 | pass 105 | 106 | misc.init_distributed_environ(exp_dir=args.exp_dir) 107 | 108 | # update args 109 | if not dist.initialized(): 110 | args.sbn = False 111 | args.first_logging = True 112 | args.device = dist.get_device() 113 | args.batch_size_per_gpu = args.bs // dist.get_world_size() 114 | args.glb_batch_size = args.batch_size_per_gpu * dist.get_world_size() 115 | 116 | if args.is_convnet: 117 | args.ada = args.ada or 0.999 118 | args.densify_norm = 'bn' 119 | 120 | args.opt = args.opt.lower() 121 | args.lr = args.base_lr * args.glb_batch_size / 256 122 | args.wde = args.wde or args.wd 123 | 124 | return args 125 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2023-05-07 18:23:46 4 | LastEditTime: 2024-08-13 20:06:35 5 | FilePath: /chengdongzhou/action/MaskCAE/main.py 6 | ''' 7 | 8 | import os 9 | import torch 10 | from configs.base_configs import args, dict_to_markdown 11 | from torch.utils.data import DataLoader 12 | 13 | os.environ["CUDA_VISIBLE_DEVICES"] = str(args.device) 14 | 15 | import numpy as np 16 | 17 | from create_dataset import GetHarDataset, dotdict 18 | from dataloaders import data_set 19 | from utils.logger import initialize_logger 20 | from utils.setup import set_seed,GetModel 21 | from train import MetaTrain 22 | from trainers.evalution import evaluate 23 | import warnings 24 | import wandb 25 | warnings.filterwarnings("ignore") 26 | import pyfiglet 27 | print('------------------------------------------------------------------') 28 | result = pyfiglet.figlet_format(text="MaskCAE", font="slant") 29 | print(result) 30 | print('------------------------------------------------------------------') 31 | use_cuda = torch.cuda.is_available() 32 | 33 | DATASET_PRE = args.dataset_pre 34 | 35 | 36 | 37 | if __name__ == "__main__": 38 | print(dict_to_markdown(vars(args))) 39 | args.save_folder = os.path.join(args.model_path, args.dataset, args.model, args.mode , args.trial) 40 | if not os.path.isdir(args.save_folder): 41 | os.makedirs(args.save_folder) 42 | # logging 43 | log_dir = os.path.join(args.save_folder, 'train.log') 44 | logger = initialize_logger(log_dir) 45 | # writer = SummaryWriter(args.save_folder + '/run') 46 | total_losses, acc_tests, mf1_tests, wf1_tests, recall_tests, precision_tests = [ [ [] for i in range(args.times)] for i in range(6) ] 47 | for i in range(args.times): 48 | logger.info(f'/n-------run time {i}--------\n') 49 | args.time = i 50 | set_seed(i+1) 51 | 52 | data_args = dotdict() 53 | 54 | data_args.yaml_path = 'configs/yaml/data.yaml' 55 | args.is_sup = False 56 | 57 | data_args, dataset\ 58 | = GetHarDataset(dataset_name=args.dataset,data_args=data_args) 59 | for currenct_cv in range(dataset.num_of_cv): 60 | print("================ {} Mode ====================".format(dataset.exp_mode)) 61 | print("================ {} CV ====================".format(dataset.num_of_cv)) 62 | dataset.update_train_val_test_keys() 63 | args.Part = dataset.index_of_cv 64 | args.num_of_cv = dataset.num_of_cv 65 | 66 | train_data = data_set(data_args,dataset,"train") 67 | test_data = data_set(data_args,dataset,"test") 68 | vali_data = data_set(data_args,dataset,"vali") 69 | 70 | wandb.init( 71 | project=f'{args.mode}', 72 | reinit=True, 73 | config=vars(args), 74 | # entity='NanjingNormalUniversity', 75 | # settings=wandb.Settings(start_method="fork") 76 | # mode='offline' 77 | # name=recorder.save_alg+f'_{i}seed', 78 | # id=recorder.save_alg+f'_{i}seed', 79 | # resume="allow" 80 | ) if args.use_wandb else None 81 | #train 82 | 83 | train = MetaTrain(args) 84 | print(f'The training method is {train.__name__}') 85 | stat = train(train_data,vali_data,logger) 86 | wandb.join() if args.use_wandb else None 87 | model = GetModel() 88 | total_loss, acc_test, mf1_test, wf1_test, recall_test, precision_test , stat = evaluate(model,logger=logger,epoch = stat['best_epoch'],eval_loader=test_data,stat=stat) 89 | for res ,save_lsit in zip( [total_loss, acc_test, mf1_test, wf1_test, recall_test, precision_test], [total_losses, acc_tests, mf1_tests, wf1_tests, recall_tests, precision_tests]): 90 | save_lsit[i].append(res) 91 | 92 | logger.info(f'==>LOCV Acc List is {acc_tests}\n') 93 | logger.info(f'==>LOCV mF1 List is {mf1_tests}\n') 94 | logger.info(f'==>LOCV wF1 List is {wf1_tests}\n') 95 | print('====================== Averaging Results ======================') 96 | average_times = lambda lst : list(np.mean(lst,axis=0)) if args.dataset != 'uschad' else list(np.squeeze(lst)) 97 | total_losses = average_times(total_losses) 98 | acc_tests = average_times(acc_tests) 99 | mf1_tests = average_times(mf1_tests) 100 | wf1_tests = average_times(wf1_tests) 101 | recall_tests = average_times(recall_tests) 102 | precision_tests = average_times(precision_tests) 103 | 104 | 105 | logger.info(f'==>LOCV Averaged Acc is {np.around( np.mean( acc_tests ) * 100 , 3 )}\n') 106 | logger.info(f'==>LOCV Averaged mF1 is {np.around( np.mean( mf1_tests ) * 100 , 3 )}\n') 107 | logger.info(f'==>LOCV Averaged wF1 is {np.around( np.mean( wf1_tests ) * 100 , 3 )}\n') 108 | 109 | -------------------------------------------------------------------------------- /pretrain/encoder.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | _cur_active: torch.Tensor = None # B1fs 5 | def _get_active_ex_or_ii(H, returning_active_ex=True): 6 | downsample_raito = H // _cur_active.shape[2] 7 | # B1ff -> B,1,f*downsample_raito,f*downsample_raito，对mask进行重复扩张，也就是说每个层级的特征图所乘的mask网格是不一致的，_cur_active为布尔张量， 8 | # 但仍然需要注意的是每个层级的特征图的mask radio永远都是一致的 9 | active_ex = _cur_active.repeat_interleave(downsample_raito, 2) 10 | return active_ex if returning_active_ex else active_ex.squeeze(1).nonzero(as_tuple=True) # ii: bi, hi, wi 11 | 12 | 13 | def sp_conv_forward(self, x: torch.Tensor): 14 | x = super(type(self), self).forward(x) 15 | x *= _get_active_ex_or_ii(H=x.shape[2], returning_active_ex=True) # (BCHW) *= (B1HW), mask the output of conv 16 | return x 17 | 18 | 19 | def sp_bn_forward(self, x: torch.Tensor): 20 | ii = _get_active_ex_or_ii(H=x.shape[2], returning_active_ex=False) 21 | 22 | bhwc = x.permute(0, 2, 3, 1) 23 | nc = bhwc[ii] # select the features on non-masked positions to form a flatten feature `nc` 24 | nc = super(type(self), self).forward(nc) # use BN1d to normalize this flatten feature `nc` 25 | 26 | bchw = torch.zeros_like(bhwc) 27 | bchw[ii] = nc 28 | bchw = bchw.permute(0, 3, 1, 2) 29 | return bchw 30 | 31 | 32 | class SparseConv2d(nn.Conv2d): 33 | forward = sp_conv_forward # hack: override the forward function; see `sp_conv_forward` above for more details 34 | 35 | 36 | class SparseMaxPooling(nn.MaxPool2d): 37 | forward = sp_conv_forward # hack: override the forward function; see `sp_conv_forward` above for more details 38 | 39 | 40 | class SparseAvgPooling(nn.AvgPool2d): 41 | forward = sp_conv_forward # hack: override the forward function; see `sp_conv_forward` above for more details 42 | 43 | 44 | class SparseBatchNorm2d(nn.BatchNorm1d): 45 | forward = sp_bn_forward # hack: override the forward function; see `sp_bn_forward` above for more details 46 | 47 | 48 | class SparseSyncBatchNorm2d(nn.SyncBatchNorm): 49 | forward = sp_bn_forward # hack: override the forward function; see `sp_bn_forward` above for more details 50 | 51 | 52 | 53 | class SparseEncoder(nn.Module): 54 | def __init__(self, cnn, input_size, sbn=False, verbose=False, mode='' ): 55 | super(SparseEncoder, self).__init__() 56 | if mode != 'plain_conv': 57 | print('==>sparse convolution') 58 | self.sp_cnn = SparseEncoder.dense_model_to_sparse(m=cnn, verbose=verbose, sbn=sbn) 59 | else:# plain convolution, do not 60 | self.sp_cnn = cnn 61 | self.dataset_name= cnn.dataset_name 62 | self.input_size, self.downsample_raito, self.enc_feat_map_chs = input_size, cnn.get_downsample_ratio(), cnn.get_feature_map_channels() 63 | 64 | @staticmethod 65 | def dense_model_to_sparse(m: nn.Module, verbose=False, sbn=False): 66 | oup = m 67 | if isinstance(m, nn.Conv2d): 68 | m: nn.Conv2d 69 | bias = m.bias is not None 70 | oup = SparseConv2d( 71 | m.in_channels, m.out_channels, 72 | kernel_size=m.kernel_size, stride=m.stride, padding=m.padding, 73 | dilation=m.dilation, groups=m.groups, bias=bias, padding_mode=m.padding_mode, 74 | ) 75 | oup.weight.data.copy_(m.weight.data) 76 | if bias: 77 | oup.bias.data.copy_(m.bias.data) 78 | elif isinstance(m, nn.MaxPool2d): 79 | m: nn.MaxPool2d 80 | oup = SparseMaxPooling(m.kernel_size, stride=m.stride, padding=m.padding, dilation=m.dilation, return_indices=m.return_indices, ceil_mode=m.ceil_mode) 81 | elif isinstance(m, nn.AvgPool2d): 82 | m: nn.AvgPool2d 83 | oup = SparseAvgPooling(m.kernel_size, m.stride, m.padding, ceil_mode=m.ceil_mode, count_include_pad=m.count_include_pad, divisor_override=m.divisor_override) 84 | elif isinstance(m, (nn.BatchNorm2d, nn.SyncBatchNorm)): 85 | m: nn.BatchNorm2d 86 | oup = (SparseSyncBatchNorm2d if sbn else SparseBatchNorm2d)(m.weight.shape[0], eps=m.eps, momentum=m.momentum, affine=m.affine, track_running_stats=m.track_running_stats) 87 | oup.weight.data.copy_(m.weight.data) 88 | oup.bias.data.copy_(m.bias.data) 89 | oup.running_mean.data.copy_(m.running_mean.data) 90 | oup.running_var.data.copy_(m.running_var.data) 91 | oup.num_batches_tracked.data.copy_(m.num_batches_tracked.data) 92 | if hasattr(m, "qconfig"): 93 | oup.qconfig = m.qconfig 94 | elif isinstance(m, (nn.Conv1d,)): 95 | raise NotImplementedError 96 | 97 | for name, child in m.named_children(): 98 | oup.add_module(name, SparseEncoder.dense_model_to_sparse(child, verbose=verbose, sbn=sbn)) 99 | del m 100 | return oup 101 | 102 | def forward(self, x): 103 | return self.sp_cnn(x, hierarchical=True) 104 | -------------------------------------------------------------------------------- /trainers/maskcae_trainer.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2023-03-18 16:59:36 4 | LastEditTime: 2024-08-13 17:54:05 5 | FilePath: /chengdongzhou/action/MaskCAE/trainers/maskcae_trainer.py 6 | ''' 7 | from torch.utils.data import DataLoader 8 | from sklearn import metrics 9 | from configs.base_configs import args 10 | from utils.setup import create_model_opt , lr_wd_annealing,load_checkpoint ,LrSchedulerSet 11 | import torch.nn as nn 12 | from utils.metric import Timer 13 | from utils.logger import Statistics 14 | from tqdm import tqdm, trange 15 | import torch 16 | from torch.nn import functional as F 17 | import wandb 18 | from trainers.evalution import evaluate 19 | import os 20 | 21 | def freeze_modules_by_name(model, module_names): 22 | for name, module in model.named_modules(): 23 | if name in module_names: 24 | for param in module.parameters(): 25 | param.requires_grad = False 26 | return model 27 | 28 | def LinearEavlution(model): 29 | return freeze_modules_by_name(model,['layer1','layer2','layer3','layer4']) 30 | 31 | def Partial_fine_tuning(model,layer_numbers): 32 | freeze_layer_numbers = 4 - layer_numbers 33 | freeze_layers = [f'layer{ln+1}' for ln in range(freeze_layer_numbers)] 34 | return freeze_modules_by_name(model,freeze_layers) 35 | 36 | def GetEvaMode(model,logger): 37 | if args.linear_evaluation: 38 | logger.info( '==> Linear Probing....') 39 | model = LinearEavlution(model) 40 | eva_flag = 'lp' 41 | else: 42 | if args.partial_ft: 43 | logger.info( '==> Partial Fine Tuning....') 44 | model = Partial_fine_tuning(model,args.ft_layernumbers) 45 | eva_flag = 'pft' 46 | else: 47 | logger.info( '==> Fine Tuning....') 48 | eva_flag = 'ft' 49 | return model,eva_flag 50 | 51 | def maskcae_train(train_set,valid_set,logger): 52 | train_loader = DataLoader( train_set, batch_size=args.batch_size, shuffle=True, drop_last=False, 53 | num_workers=args.num_workers, pin_memory=True ) 54 | val_loader = DataLoader( valid_set, batch_size=args.batch_size, shuffle=False, drop_last=False, 55 | num_workers=args.num_workers, pin_memory=True ) 56 | 57 | model, optimizer, criterion_cls = create_model_opt() 58 | 59 | stat = {} 60 | stat['best_mf1'] = 0.0 61 | # wandb.watch(model,log_freq=50,log="all") 62 | # logging 63 | logger.info( args ) 64 | print( f"==> load pre-trained model..." ) 65 | load_checkpoint(args,resume_from=args.pre_trained_path,model=model,optimizer=optimizer) 66 | model,eva_flag = GetEvaMode(model=model,logger=logger) 67 | timer = Timer() 68 | iters_train = len(train_loader) 69 | Test_losses ,Acc_tests, mF1_tests,wF1_tests, Recall_tests, Precision_tests = [],[],[],[],[],[] 70 | for epoch in trange( args.epochs, desc='Training_epoch' ): 71 | model.train() 72 | total_num, total_loss = 0, 0 73 | label_list, predicted_list = [], [] 74 | for idx, (data, label) in enumerate(train_loader): 75 | # adjust lr and wd 76 | cur_it = idx + epoch * iters_train 77 | min_lr, max_lr, min_wd, max_wd = lr_wd_annealing(optimizer, args.learning_rate, args.weight_decay, cur_it, args.warm_up_epochs * iters_train, args.epochs * iters_train) 78 | timer.start() 79 | inputs, label = data.cuda().float(), label.cuda().long() 80 | output = model(inputs)['output'] 81 | loss = criterion_cls(output, label) 82 | optimizer.zero_grad() 83 | loss.backward() 84 | # orig_norm = nn.utils.clip_grad_norm_( model.parameters(), 100 ) 85 | optimizer.step() 86 | 87 | timer.stop() 88 | total_loss += loss.detach().item() 89 | with torch.no_grad(): 90 | _, predicted = torch.max( output, 1 ) 91 | label_list.append(label) 92 | predicted_list.append( predicted ) 93 | total_num = total_num + len( label ) 94 | 95 | label = torch.cat(label_list).cpu().detach().numpy() 96 | predicted = torch.cat(predicted_list).cpu().detach().numpy() 97 | acc_train = ( predicted == label ).sum() / total_num 98 | f1_train = metrics.f1_score( label, predicted, average='macro' ) 99 | loss_train = total_loss/len(train_loader) 100 | 101 | logger.info( f'Epoch:[{epoch}/{args.epochs}] - {eva_flag}_loss:{loss_train:.7f}, {eva_flag}_train@Acc: {acc_train:.5f}, {eva_flag}_train@F1: {f1_train:.5f}') 102 | 103 | wandb.log({f'{eva_flag} train loss' :loss_train , 104 | f'{eva_flag} train Acc' :acc_train , 105 | f'{eva_flag} train F1' :f1_train , 106 | 'min_lr':min_lr, 107 | 'max_lr':max_lr, 108 | 'min_wd':min_wd, 109 | 'max_wd':max_wd 110 | } , 111 | commit=False) if args.use_wandb else None 112 | 113 | Test_loss , Acc_test, mF1_test,wF1_test, Recall_test, Precision_test, stat = evaluate(model, logger=logger, eval_loader = val_loader , epoch = epoch , is_test=False, stat=stat) 114 | 115 | for elem , save_res in zip((Test_loss,Acc_test, mF1_test, wF1_test, Recall_test, Precision_test),(Test_losses,Acc_tests, mF1_tests,wF1_tests, Recall_tests, Precision_tests)): 116 | save_res.append(elem) 117 | 118 | stat = Statistics(stat,Test_losses, Acc_tests, mF1_tests, wF1_tests , Recall_tests, Precision_tests,sum_time= timer.sum() ) 119 | # wandb.save('model.h5') 120 | 121 | return stat -------------------------------------------------------------------------------- /models/ConvNet.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2023-05-22 16:58:22 4 | LastEditTime: 2024-08-13 19:31:13 5 | FilePath: /chengdongzhou/action/MaskCAE/models/ConvNet.py 6 | ''' 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | import os 10 | import sys 11 | sys.path.append(os.path.dirname(os.path.dirname(__file__))) 12 | import torch 13 | from common import channel_list,conv_list,maxp_list,first_maxp_list 14 | from utils.metric import get_data_size,GetFlopsAndParams,GetFeatureMapSize 15 | 16 | 17 | 18 | class Conv4Net(nn.Module): 19 | def __init__(self, data_name='ucihar' ,use_adativeavg=False, sub_number = 4 ): 20 | super(Conv4Net, self).__init__() 21 | self.dataset = data_name 22 | self.channel= channel = channel_list[data_name] 23 | conv_params = conv_list[data_name] 24 | max_params = maxp_list[data_name] 25 | first_max = first_maxp_list[data_name] 26 | 27 | self.layer1 = self.maker_layers( 1 , channel[0], conv_params=conv_params, pooling_params=max_params ) 28 | self.layer2 = self.maker_layers( channel[0], channel[1], conv_params=conv_params, pooling_params=max_params ) 29 | self.layer3 = self.maker_layers( channel[1], channel[2], conv_params=conv_params, pooling_params=max_params ) 30 | self.layer4 = self.maker_layers( channel[2], channel[3], conv_params=conv_params, pooling_params=max_params ) 31 | h,w = GetFeatureMapSize(data_name,sub_number,padding = True) 32 | self.adaptiveAvg = nn.AdaptiveAvgPool2d( ( 1, w ) ) if use_adativeavg else nn.Identity() 33 | self.classifier = nn.Linear( channel[3]*w if use_adativeavg else channel[3]* h * w , channel[-1] ) 34 | 35 | def maker_layers(self,inp,oup,conv_params=None,pooling_params=None): 36 | assert isinstance(conv_params,list),print('the format of kernel params is error') 37 | assert isinstance(pooling_params,list) or pooling_params == None ,print('the format of pooling params is error') 38 | return nn.Sequential( 39 | nn.Conv2d( inp, oup, *(conv_params) ), 40 | nn.BatchNorm2d( oup ), 41 | nn.ReLU( True ), 42 | nn.MaxPool2d( *(pooling_params) ) if pooling_params else nn.Identity() 43 | ) 44 | def forward(self, x: torch.Tensor): 45 | B,_,_,_ = x.size() 46 | x = F.pad(x, pad=(0,0,1,1), mode='replicate') 47 | x = self.layer1(x) 48 | x = self.layer2(x) 49 | x = self.layer3(x) 50 | x = self.layer4(x) 51 | x = self.adaptiveAvg(x) 52 | x = x.view(B,-1) 53 | x = self.classifier( x ) 54 | res = {} 55 | res['output'] = x 56 | return res 57 | 58 | 59 | class Conv4NetSup(nn.Module): 60 | def __init__(self, data_name='ucihar' ,use_adativeavg=False, sub_number = 4 ): 61 | super(Conv4NetSup, self).__init__() 62 | self.dataset = data_name 63 | self.channel= channel = channel_list[data_name] 64 | conv_params = conv_list[data_name] 65 | max_params = maxp_list[data_name] 66 | first_max_params = first_maxp_list[data_name] 67 | 68 | self.layer1 = self.maker_layers( 1 , channel[0], conv_params=conv_params, pooling_params=first_max_params ) 69 | self.layer2 = self.maker_layers( channel[0], channel[1], conv_params=conv_params, pooling_params=max_params ) 70 | self.layer3 = self.maker_layers( channel[1], channel[2], conv_params=conv_params, pooling_params=max_params ) 71 | self.layer4 = self.maker_layers( channel[2], channel[3], conv_params=conv_params, pooling_params=max_params ) 72 | h,w = GetFeatureMapSize(data_name,sub_number,padding = True, larger_max= True , is_sup= True ) 73 | self.adaptiveAvg = nn.AdaptiveAvgPool2d( ( 1, w ) ) if use_adativeavg else nn.Identity() 74 | self.classifier = nn.Linear( channel[3]*w if use_adativeavg else channel[3]* h * w , channel[-1] ) 75 | 76 | def maker_layers(self,inp,oup,conv_params=None,pooling_params=None): 77 | assert isinstance(conv_params,list),print('the format of kernel params is error') 78 | assert isinstance(pooling_params,list) or pooling_params == None ,print('the format of pooling params is error') 79 | return nn.Sequential( 80 | nn.Conv2d( inp, oup, *(conv_params) ), 81 | nn.BatchNorm2d( oup ), 82 | nn.ReLU( True ), 83 | nn.MaxPool2d( *(pooling_params) ) if pooling_params else nn.Identity(), 84 | ) 85 | 86 | def forward(self, x: torch.Tensor): 87 | B,_,_,_ = x.size() 88 | if self.dataset in ['uschad']: 89 | x = F.pad(x, pad=(0,0,1,1), mode='replicate') 90 | x = self.layer1(x) 91 | x = self.layer2(x) 92 | x = self.layer3(x) 93 | x = self.layer4(x) 94 | x = self.adaptiveAvg(x) 95 | x = x.view(B,-1) 96 | x = self.classifier( x ) 97 | res = {} 98 | res['output'] = x 99 | return res 100 | 101 | 102 | if __name__ == "__main__": 103 | import torch 104 | f_size = { 105 | 'ucihar': (30 , 9 ), 106 | 'uschad': (30 , 6 ), 107 | 'motion': (30 , 12 ) 108 | } 109 | dataset = 'motion' 110 | x = torch.randn(64,1,*f_size[dataset]) 111 | model = Conv4Net(dataset) 112 | # print(model,model.layer2[1]) 113 | GetFlopsAndParams(dataset,model) 114 | print(model(x)['output'].shape) 115 | n_all = sum(p.numel() for p in model.parameters()) 116 | n_trainable = sum(p.numel() for p in model.parameters() if p.requires_grad) 117 | print("Parameter Count: all {:,d}; trainable {:,d}".format(n_all, n_trainable)) -------------------------------------------------------------------------------- /utils/metric.py: -------------------------------------------------------------------------------- 1 | from thop.profile import profile 2 | from thop import clever_format 3 | import torch 4 | import numpy as np 5 | import time 6 | from common import maxp_list,channel_list,conv_list,first_maxp_list 7 | 8 | def Reture_F_SIZE(dataset , is_sup=False): 9 | F_SIZE = { 10 | 'ucihar': (30 , 9 ) , 11 | 'uschad': (30 , 6 ) , 12 | 'motion': (30 , 12 ) 13 | } 14 | SUP_F_SIZE = { 15 | 'ucihar': (128 , 9 ) , 16 | 'uschad': (30 , 6 ) , 17 | 'motion': (128 , 12 ) 18 | } 19 | if not is_sup: 20 | return F_SIZE[dataset] 21 | else: 22 | return SUP_F_SIZE[dataset] 23 | 24 | class Timer: 25 | """Record multiple running times.""" 26 | def __init__(self): 27 | self.times = [] 28 | self.start() 29 | 30 | def start(self): 31 | """Start the timer.""" 32 | self.tik = time.time() 33 | 34 | def stop(self): 35 | """Stop the timer and record the time in a list.""" 36 | self.times.append(time.time() - self.tik) 37 | return self.times[-1] 38 | 39 | def avg(self): 40 | """Return the average time.""" 41 | return sum(self.times) / len(self.times) 42 | 43 | def sum(self): 44 | """Return the sum of time.""" 45 | return sum(self.times) 46 | 47 | def cumsum(self): 48 | """Return the accumulated time.""" 49 | return np.array(self.times).cumsum().tolist() 50 | 51 | def get_data_size(data_name, is_sup = False ): 52 | Model_Seen_SSL_F_Size = { 53 | 'ucihar': ( 1, 1, 32, 9 ) , 54 | 'motion': ( 1, 1, 32, 12 ) , 55 | 'uschad': ( 1, 1, 32, 6 ) , 56 | } 57 | Model_Seen_Sup_F_Size = { 58 | 'ucihar': ( 1, 1, 128, 9 ) , 59 | 'motion': ( 1, 1, 128, 12 ) , 60 | 'uschad': ( 1, 1, 32, 6 ) , 61 | } 62 | if is_sup: 63 | size_dict = Model_Seen_Sup_F_Size 64 | else: 65 | size_dict = Model_Seen_SSL_F_Size 66 | if data_name in size_dict: 67 | pass 68 | else: 69 | raise Exception( 'please input correct data name') 70 | return size_dict[data_name] 71 | 72 | def get_classes(data_name): 73 | if data_name == 'ucihar': 74 | classes = 6 75 | elif data_name == 'motion': 76 | classes = 6 77 | elif data_name == 'uschad': 78 | classes = 12 79 | else: 80 | raise Exception( 'please input correct data name') 81 | return classes 82 | 83 | 84 | def GetFeatureMapSize(data_name,idex_layer,padding=False, larger_max=False , is_sup = False ): 85 | size = get_data_size(data_name, is_sup)[2:] 86 | maxpooling_size = maxp_list[data_name] 87 | first_maxp_size = first_maxp_list[data_name] 88 | h,w = size 89 | if idex_layer > 0: 90 | for i in range(idex_layer): 91 | if padding: 92 | if larger_max and i == 0: 93 | h = (( h - first_maxp_size[0][0] + first_maxp_size[2][0] * 2 + first_maxp_size[1][0] ) // first_maxp_size[1][0]) 94 | w = (( w - first_maxp_size[0][1] + first_maxp_size[2][1] * 2 + first_maxp_size[1][1] ) // first_maxp_size[1][1]) 95 | else: 96 | h = (( h - maxpooling_size[0][0] + maxpooling_size[2][0] * 2 + maxpooling_size[1][0] ) // maxpooling_size[1][0]) 97 | w = (( w - maxpooling_size[0][1] + maxpooling_size[2][1] * 2 + maxpooling_size[1][1] ) // maxpooling_size[1][1]) 98 | else: 99 | h //= maxpooling_size[0][0] 100 | w //= maxpooling_size[0][1] 101 | return ( h , w ) 102 | elif idex_layer == 0: 103 | return ( h , w ) 104 | else: 105 | raise ValueError(f'check your idex_layer') 106 | 107 | def GetFeatureMapSizeByConv(data_name,idex_layer): 108 | size = get_data_size(data_name)[2:] 109 | conv_size = conv_list[data_name] 110 | h,w = size 111 | if idex_layer > 0: 112 | for i in range(idex_layer): 113 | h = h - conv_size[0][0] + 1 114 | w = w - conv_size[0][1] + 1 115 | return ( h , w ) 116 | else: 117 | raise ValueError(f'check your idex_layer') 118 | 119 | def GetFeatureMapSizeBySelfMax(data_name , max_size, idex_layer): 120 | size = get_data_size(data_name)[2:] 121 | h,w = size 122 | if idex_layer > 0: 123 | for i in range(idex_layer): 124 | h //= max_size[0] 125 | w //= max_size[1] 126 | return ( h , w ) 127 | else: 128 | raise ValueError(f'check your idex_layer') 129 | 130 | def GetFlopsAndParams(dataset,net,logger = None , is_sup = False ): 131 | X = torch.randn(1,1,*Reture_F_SIZE(dataset,is_sup)) 132 | total_ops, total_params= profile(net, (X,), verbose=False) 133 | flops, params = clever_format([total_ops*2, total_params], "%.3f") 134 | if logger: 135 | logger.info(f'==>FLOPs of model that runs the {dataset} is {flops}') 136 | logger.info(f'==>Params of model that runs the {dataset} is {params}' ) 137 | else: 138 | print(f'==>FLOPs of model that runs the {dataset} is {flops}') 139 | print(f'==>Params of model that runs the {dataset} is {params}' ) 140 | return flops,params 141 | 142 | def DeConvOrthDist(kernel, stride = 2, padding = 1): 143 | [o_c, i_c, w, h] = kernel.shape 144 | output = torch.conv2d(kernel, kernel, stride=stride, padding=padding) 145 | target = torch.zeros((o_c, o_c, output.shape[-2], output.shape[-1])).cuda() 146 | ct = int(np.floor(output.shape[-1]/2)) 147 | target[:,:,ct,ct] = torch.eye(o_c).cuda() 148 | # print(target.shape) 149 | return torch.norm( output - target ) 150 | 151 | def GetInferredSpeed(dataset,model,times, is_sup = False ): 152 | x = torch.randn(1,1,*Reture_F_SIZE(dataset,is_sup)) 153 | each_times = [] 154 | model.eval() 155 | for i in range(times): 156 | start_time = time.time() 157 | model(x) 158 | stop_time = time.time() 159 | each_times.append((stop_time-start_time)*1000) 160 | return np.mean(each_times) -------------------------------------------------------------------------------- /configs/base_configs.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | import pandas as pd 4 | from configs import parser as _parser 5 | import sys 6 | import yaml 7 | 8 | args = None 9 | 10 | def parse_arguments(): 11 | parser = argparse.ArgumentParser() 12 | parser.add_argument('--seed', type=int, default=0, metavar='S', 13 | help='random seed (default: 0). if seed=-1, seed is not fixed.') 14 | parser.add_argument('--batch_size', type=int, default=128, help='batch_size') 15 | parser.add_argument( 16 | "--config", help="Config file to use (see configs dir)", default=None 17 | ) 18 | parser.add_argument('--times', type=int, default=1, help='num of different seed') 19 | parser.add_argument('--num_workers', type=int, default=4, help='num of workers to use') 20 | parser.add_argument('--device', type=int, default=2, choices=[0,1,2]) 21 | parser.add_argument('--epochs', type=int, default=200, help='number of training epochs') 22 | parser.add_argument('--window_width', type=int, default=0, help='window width') 23 | parser.add_argument('--normalize', action='store_true', default=False, help='normalize signal based on mean/std of training samples') 24 | parser.add_argument('--pretrain', action='store_true', default=False) 25 | parser.add_argument('--not_save_res','--nsr', default=False, action='store_true', 26 | help='The default value is saved.') 27 | parser.add_argument('--mode', type=str, default='ce', help='mode') 28 | parser.add_argument('--dataset_pre', type=str, default='self') 29 | parser.add_argument('--trial', type=str, default='default', help='trial id') 30 | parser.add_argument('--not_avg_exp', action='store_false', default=True) 31 | parser.add_argument('--opt_type', type=str, default='sgd') 32 | 33 | # optimization 34 | parser.add_argument('--learning_rate','--lr', type=float, default=0.01, help='learning rate') 35 | parser.add_argument('--weight_decay', type=float, default = 0.0005 , help='weight decay') 36 | parser.add_argument('--weight_decay_la', type=float, default = 0.0005 , help='weight decay for layer attention') 37 | parser.add_argument('--momentum', type=float, default=0.9, help='momentum') 38 | parser.add_argument('--lr_scheduler', type=str, default='S', help='lr_scheduler') 39 | parser.add_argument('--warm_up_epochs', type=int, default=5, help='linear warm up epochs') 40 | parser.add_argument('--handcrafted_lr', action='store_true', default=False, help='learning rate with warm up') 41 | parser.add_argument('--chhander', type=bool, default=False) 42 | 43 | 44 | # MultiStepLR or ExponentialLR 45 | parser.add_argument('--gamma',type=float,default=0.1,help='gamma')## 46 | parser.add_argument('--milestones',type=list,default=[40,80,120,160],help='optimizer milestones') 47 | parser.add_argument('--decay_epochs',type=float,default=40,help='n_epochs for CosineAnnealingLR or StepLR') 48 | 49 | # CosineAnnealingLR 50 | parser.add_argument('--n_epochs',type=float,default=20,help='n_epochs for CosineAnnealingLR') 51 | 52 | # dataset and model 53 | parser.add_argument('--model', type=str, default='EarlyFusion') 54 | parser.add_argument('--dataset', type=str, default='ucihar') 55 | parser.add_argument('--no_clean', action='store_false', default=False) 56 | parser.add_argument('--no_null', action='store_false', default=True) 57 | parser.add_argument('--train_portion', type=float, default=1.0, help='use portion of trainset') 58 | parser.add_argument('--model_path', type=str, default='save', help='path to save model') 59 | parser.add_argument('--load_model', type=str, default='', help='load the pretrained model') 60 | 61 | # coefficients 62 | parser.add_argument('--lam', type=float, default=0.0, help='An alternate measure coefficient, which defaults to 0') 63 | parser.add_argument('--p', type=float, default=0.0, help='An alternate measure coefficient, which defaults to 0') 64 | parser.add_argument('--beta', type=float, default=0.0, help='An alternate measure coefficient, which defaults to 0') 65 | 66 | 67 | 68 | # for wandb 69 | parser.add_argument('--use_wandb', default=False, action='store_true') 70 | 71 | # for maskcae 72 | parser.add_argument('--partial_ft', type=bool, default=False) 73 | parser.add_argument('--ft_layernumbers', type=int, default=0) 74 | parser.add_argument('--semi_number', type=int, default=0) 75 | parser.add_argument('--mask', type=float, default=-1) 76 | parser.add_argument('--ablationMode', type=str,default='') 77 | parser.add_argument('--pre_epochs', type=int) 78 | parser.add_argument('--decoder_type', type=str) 79 | parser.add_argument('--IP_type', type=str) 80 | parser.add_argument('--loss_type', type=str) 81 | parser.add_argument('--linear_evaluation', type=bool, default=False) 82 | parser.add_argument('--vis_feature', action='store_true', default=False) 83 | args = parser.parse_args() 84 | 85 | get_config(args) 86 | 87 | return args 88 | 89 | 90 | def get_config(args): 91 | # get commands from command line 92 | override_args = _parser.argv_to_vars(sys.argv) 93 | yaml_path = os.path.join(os.path.dirname(os.path.dirname(__file__)) , args.config ) 94 | print(f'==> local yaml path:{yaml_path}') 95 | # load yaml file 96 | yaml_txt = open( yaml_path).read() 97 | 98 | # override args 99 | loaded_yaml = yaml.load(yaml_txt, Loader=yaml.FullLoader) 100 | for v in override_args: 101 | loaded_yaml[v] = getattr(args, v) 102 | 103 | print(f"==> Reading YAML config from {args.config}") 104 | args.__dict__.update(loaded_yaml) 105 | 106 | 107 | def run_args(): 108 | global args 109 | if args is None: 110 | args = parse_arguments() 111 | 112 | 113 | run_args() 114 | 115 | if args.pretrain: 116 | args.lambda_cls = 0.0 117 | args.lambda_ssl = 1.0 118 | 119 | def dict_to_markdown(d, max_str_len=120): 120 | # convert list into its str representation 121 | d = {k: v.__repr__() if isinstance(v, list) else v for k, v in d.items()} 122 | # truncate string that is longer than max_str_len 123 | if max_str_len is not None: 124 | d = {k: v[-max_str_len:] if isinstance(v, str) else v for k, v in d.items()} 125 | return pd.DataFrame(d, index=[0]).transpose().to_markdown() 126 | 127 | #Display settings 128 | #print(dict_to_markdown(vars(args), max_str_len=120)) -------------------------------------------------------------------------------- /dataloaders/__init__.py: -------------------------------------------------------------------------------- 1 | from torch.utils.data import Dataset 2 | import torch 3 | import numpy as np 4 | import os 5 | import pickle 6 | from tqdm import tqdm 7 | 8 | from .dataloader_uci_har import UCI_HAR_DATA 9 | from .dataloader_MOTION_Sense_har import MotionSense_HAR_DATA 10 | from .dataloader_USCHAD_har import USC_HAD_HAR_DATA 11 | data_dict = { 12 | "ucihar" : UCI_HAR_DATA, 13 | "motion" : MotionSense_HAR_DATA, 14 | "uschad" : USC_HAD_HAR_DATA, 15 | } 16 | 17 | class data_set(Dataset): 18 | def __init__(self, args, dataset, flag): 19 | """ 20 | args : a dict , In addition to the parameters for building the model, the parameters for reading the data are also in here 21 | dataset : It should be implmented dataset object, it contarins train_x, train_y, vali_x,vali_y,test_x,test_y 22 | flag : (str) "train","test","vali" 23 | """ 24 | self.args = args 25 | self.flag = flag 26 | self.load_all = args.load_all 27 | self.data_x = dataset.normalized_data_x 28 | self.data_y = dataset.data_y 29 | self.slidingwindows = dataset.slidingwindows 30 | self.act_weights = dataset.act_weights 31 | 32 | if self.args.representation_type in ["freq","time_freq"]: 33 | if flag in ["train","vali"]: 34 | self.freq_path = dataset.train_freq_path 35 | self.freq_file_name = dataset.train_freq_file_name 36 | if self.load_all : 37 | self.data_freq = dataset.data_freq 38 | else: 39 | self.freq_path = dataset.test_freq_path 40 | self.freq_file_name = dataset.test_freq_file_name 41 | self.load_all = False 42 | 43 | if self.flag == "train": 44 | # load train 45 | self.window_index = dataset.train_window_index 46 | print(f"================ Train data number: {len(self.window_index)} =================") 47 | 48 | 49 | elif self.flag == "vali": 50 | # load vali 51 | 52 | self.window_index = dataset.valid_window_index 53 | print(f"================ Validation data number: {len(self.window_index)} ================") 54 | 55 | 56 | else: 57 | # load test 58 | self.window_index = dataset.test_window_index 59 | print(f"================ Test data number: {len(self.window_index)} ================") 60 | 61 | 62 | all_labels = list(np.unique(dataset.data_y)) 63 | to_drop = list(dataset.drop_activities) 64 | label = [item for item in all_labels if item not in to_drop] 65 | self.nb_classes = len(label) 66 | assert self.nb_classes==len(dataset.no_drop_activites) 67 | 68 | classes = dataset.no_drop_activites 69 | self.class_transform = {x: i for i, x in enumerate(classes)} 70 | self.class_back_transform = {i: x for i, x in enumerate(classes)} 71 | self.input_length = self.slidingwindows[0][2]-self.slidingwindows[0][1] 72 | self.channel_in = self.data_x.shape[1]-2 73 | 74 | 75 | #if self.args.wavelet_filtering: 76 | # SelectedWavelet = PrepareWavelets(K=self.args.number_wavelet_filtering, length=self.args.windowsize) 77 | # self.ScaledFilter = FiltersExtention(SelectedWavelet) 78 | # if self.args.windowsize%2==1: 79 | # self.Filter_ReplicationPad1d = torch.nn.ReplicationPad1d(int((self.args.windowsize-1)/2)) 80 | # else: 81 | # self.Filter_ReplicationPad1d = torch.nn.ReplicationPad1d(int(self.args.windowsize/2)) 82 | 83 | if self.flag == "train": 84 | print("==>The number of classes is : ", self.nb_classes) 85 | print("==>The input_length is : ", self.input_length) 86 | print("==>The channel_in is : ", self.channel_in) 87 | 88 | 89 | def __getitem__(self, index): 90 | #print(index) 91 | index = self.window_index[index] 92 | start_index = self.slidingwindows[index][1] 93 | end_index = self.slidingwindows[index][2] 94 | # print(index,start_index,end_index) 95 | if self.args.representation_type == "time": 96 | 97 | if self.args.sample_wise ==True: 98 | sample_x = np.array(self.data_x.iloc[start_index:end_index, 1:-1].apply(lambda x: (x - np.mean(x)) / (np.max(x) - np.min(x)))) 99 | else: 100 | sample_x = self.data_x.iloc[start_index:end_index, 1:-1].values 101 | 102 | sample_y = self.class_transform[self.data_y.iloc[start_index:end_index].mode().loc[0]] 103 | 104 | sample_x = np.expand_dims(sample_x,0) 105 | 106 | if self.args.multicrop and self.flag == "train": 107 | if self.args.mc_local_number > 0: 108 | sample_x = [sample_x] + np.array_split(sample_x,self.args.mc_local_number,1) 109 | else: 110 | pass 111 | return sample_x , sample_y 112 | 113 | elif self.args.representation_type == "freq": 114 | 115 | if self.load_all: 116 | sample_x = self.data_freq[self.freq_file_name[index]] 117 | else: 118 | with open(os.path.join(self.freq_path,"{}.pickle".format(self.freq_file_name[index])), 'rb') as handle: 119 | sample_x = pickle.load(handle) 120 | 121 | sample_y = self.class_transform[self.data_y.iloc[start_index:end_index].mode().loc[0]] 122 | 123 | return sample_x, sample_y,sample_y 124 | 125 | else: 126 | 127 | if self.args.sample_wise ==True: 128 | sample_ts_x = np.array(self.data_x.iloc[start_index:end_index, 1:-1].apply(lambda x: (x - np.mean(x)) / (np.max(x) - np.min(x)))) 129 | else: 130 | sample_ts_x = self.data_x.iloc[start_index:end_index, 1:-1].values 131 | 132 | 133 | if self.load_all: 134 | sample_fq_x = self.data_freq[self.freq_file_name[index]] 135 | else: 136 | with open(os.path.join(self.freq_path,"{}.pickle".format(self.freq_file_name[index])), 'rb') as handle: 137 | sample_fq_x = pickle.load(handle) 138 | 139 | sample_y = self.class_transform[self.data_y.iloc[start_index:end_index].mode().loc[0]] 140 | 141 | 142 | return sample_ts_x, sample_fq_x , sample_y 143 | 144 | def __len__(self): 145 | return len(self.window_index) 146 | 147 | -------------------------------------------------------------------------------- /create_dataset.py: -------------------------------------------------------------------------------- 1 | 2 | import warnings 3 | warnings.filterwarnings("ignore") 4 | 5 | import os 6 | import sys 7 | sys.path.append(os.path.dirname(__file__)) 8 | 9 | from torch.utils.data import DataLoader 10 | 11 | from dataloaders import data_set,data_dict 12 | import torch 13 | import yaml 14 | import os 15 | import pandas as pd 16 | import matplotlib.pyplot as plt 17 | import numpy as np 18 | import argparse 19 | 20 | WINDOWS_SAVE_PATH = r"./MaskCAE/datasets/Sliding_window" 21 | ROOT_PATH = r"./data/raw" 22 | 23 | class dotdict(dict): 24 | """dot.notation access to dictionary attributes""" 25 | __getattr__ = dict.get 26 | __setattr__ = dict.__setitem__ 27 | __delattr__ = dict.__delitem__ 28 | 29 | 30 | 31 | def GetHarDataset(dataset_name , data_args): 32 | 33 | data_args.freq_save_path = None 34 | data_args.window_save_path = WINDOWS_SAVE_PATH 35 | data_args.root_path = ROOT_PATH 36 | 37 | data_args.drop_transition = False 38 | 39 | data_args.batch_size = 128 40 | data_args.shuffle = True 41 | data_args.drop_last = False 42 | data_args.train_vali_quote = 0.8 43 | data_args.down_sample = True 44 | 45 | data_args.data_name = dataset_name # wisdm 46 | 47 | 48 | # 是否作augmentation difference 49 | data_args.difference = False 50 | 51 | # 是否作augmentation filtering 52 | data_args.filtering = False 53 | 54 | # 是否作augmentation magnitude 55 | data_args.magnitude = False 56 | # 是否按照权重抽取批次 57 | data_args.weighted_sampler = False 58 | 59 | data_args.load_all = None 60 | # 是在load数据的时候 wavelet_filtering 61 | data_args.wavelet_filtering = False 62 | data_args.number_wavelet_filtering = 10 63 | 64 | data_args.datanorm_type = "standardization" # None ,"standardization", "minmax" 65 | 66 | data_args.pos_select = None 67 | data_args.sensor_select = None 68 | data_args.sample_wise = False 69 | data_args.multicrop = False 70 | 71 | data_args.representation_type = "time" 72 | 73 | 74 | config_file = open(data_args.yaml_path, mode='r') 75 | data_config = yaml.load(config_file, Loader=yaml.FullLoader) 76 | config = data_config[data_args.data_name] 77 | 78 | data_args.root_path = os.path.join(data_args.root_path,config["filename"]) 79 | data_args.sampling_freq = config["sampling_freq"] 80 | data_args.valid_rate = config["valid_rate"] 81 | data_args.overlap_rate = config["overlap_rate"] 82 | data_args.down_sample = config["down_sample"] 83 | window_seconds = config["window_seconds"] 84 | data_args.windowsize = int(window_seconds * data_args.sampling_freq) 85 | data_args.c_in = config["num_channels"] 86 | data_args.exp_mode = config["exp_mode"] 87 | 88 | data_args.input_length = data_args.windowsize 89 | # input information 90 | if data_args.wavelet_filtering : 91 | 92 | if data_args.windowsize%2==1: 93 | N_ds = int(torch.log2(torch.tensor(data_args.windowsize-1)).floor()) - 2 94 | else: 95 | N_ds = int(torch.log2(torch.tensor(data_args.windowsize)).floor()) - 2 96 | 97 | data_args.f_in = data_args.number_wavelet_filtering*N_ds+1 98 | else: 99 | data_args.f_in = 1 100 | 101 | return data_args , data_dict[data_args.data_name](data_args) 102 | 103 | def CheckDataloader(data_args , dataset): 104 | print("================ {} Mode ====================".format(dataset.exp_mode)) 105 | print("================ {} CV ======================".format(dataset.num_of_cv)) 106 | for i in range(dataset.num_of_cv): 107 | dataset.update_train_val_test_keys() 108 | train_data = data_set(data_args,dataset,"train") 109 | test_data = data_set(data_args,dataset,"test") 110 | vali_data = data_set(data_args,dataset,"vali") 111 | 112 | 113 | # form the dataloader 114 | train_data_loader = DataLoader(train_data, 115 | batch_size = data_args.batch_size, 116 | shuffle = data_args.shuffle, 117 | num_workers = 4, 118 | drop_last = data_args.drop_last) 119 | 120 | vali_data_loader = DataLoader(vali_data, 121 | batch_size = data_args.batch_size, 122 | shuffle = data_args.shuffle, 123 | num_workers = 4, 124 | drop_last = data_args.drop_last) 125 | 126 | test_data_loader = DataLoader(test_data, 127 | batch_size = data_args.batch_size, 128 | shuffle = data_args.shuffle, 129 | num_workers = 4, 130 | drop_last = data_args.drop_last) 131 | 132 | for datas,label in train_data_loader: 133 | if data_args.multicrop: 134 | [print(data.shape) for data in datas ] 135 | else: 136 | print(datas.shape) 137 | break 138 | 139 | 140 | 141 | 142 | 143 | 144 | def test(dataset): 145 | data_args = dotdict() 146 | data_args.all = False 147 | data_args.name = dataset 148 | if data_args.all: 149 | assert data_args.name == None, f'can not choice the dataset {data_args.name}' 150 | dataset_list = ['hapt','oppo','wisdm','dg','uschad','pamap2','rw','skoda','dsads','motion','ucihar','mhealth'] 151 | for dataset_name in dataset_list: 152 | print(f'==>generate {dataset_name} datset-----------------------') 153 | data_args , dataset = GetHarDataset(dataset_name,data_args) 154 | CheckDataloader(data_args , dataset) 155 | else: 156 | assert data_args.name , 'please give a dataset name' 157 | print(f'==>generate {data_args.name} datset-----------------------') 158 | data_args , dataset = GetHarDataset(data_args.name,data_args) 159 | CheckDataloader(data_args , dataset) 160 | 161 | 162 | if __name__ == '__main__': 163 | np.random.seed(1) 164 | parser = argparse.ArgumentParser() 165 | parser.add_argument('--dataset',type=str,default='motion') 166 | args = parser.parse_args() 167 | test(args.dataset) -------------------------------------------------------------------------------- /models/Baseline_CNN.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2023-05-22 16:58:22 4 | LastEditTime: 2024-08-13 18:48:44 5 | FilePath: /chengdongzhou/action/MaskCAE/models/Baseline_CNN.py 6 | ''' 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | import os 10 | import sys 11 | sys.path.append(os.path.dirname(os.path.dirname(__file__))) 12 | import torch 13 | from common import channel_list,conv_list,maxp_list,first_maxp_list 14 | from utils.metric import get_data_size,GetFlopsAndParams,GetInferredSpeed 15 | 16 | def GetFeatureMapSize(data_name,idex_layer,padding=False, larger_max=False , is_sup = False ): 17 | size = get_data_size(data_name, is_sup)[2:] 18 | maxpooling_size = maxp_list[data_name] 19 | first_maxp_size = first_maxp_list[data_name] 20 | h,w = size 21 | if idex_layer > 0: 22 | for i in range(idex_layer): 23 | if padding: 24 | if larger_max and i == 0: 25 | h = (( h - first_maxp_size[0][0] + first_maxp_size[2][0] * 2 + first_maxp_size[1][0] ) // first_maxp_size[1][0]) 26 | w = (( w - first_maxp_size[0][1] + first_maxp_size[2][1] * 2 + first_maxp_size[1][1] ) // first_maxp_size[1][1]) 27 | else: 28 | h = (( h - maxpooling_size[0][0] + maxpooling_size[2][0] * 2 + maxpooling_size[1][0] ) // maxpooling_size[1][0]) 29 | w = (( w - maxpooling_size[0][1] + maxpooling_size[2][1] * 2 + maxpooling_size[1][1] ) // maxpooling_size[1][1]) 30 | else: 31 | h //= maxpooling_size[0][0] 32 | w //= maxpooling_size[0][1] 33 | return ( h , w ) 34 | elif idex_layer == 0: 35 | return ( h , w ) 36 | else: 37 | raise ValueError(f'check your idex_layer') 38 | 39 | class BaseCNN(nn.Module): 40 | def __init__(self, data_name='ucihar' ,use_adativeavg=False, sub_number = 4 ): 41 | super(BaseCNN, self).__init__() 42 | self.dataset = data_name 43 | self.channel= channel = channel_list[data_name] 44 | conv_params = conv_list[data_name] 45 | max_params = maxp_list[data_name] 46 | 47 | self.layer1 = self.maker_layers( 1 , channel[0], conv_params=conv_params, pooling_params=max_params ) 48 | self.layer2 = self.maker_layers( channel[0], channel[1], conv_params=conv_params, pooling_params=max_params ) 49 | self.layer3 = self.maker_layers( channel[1], channel[2], conv_params=conv_params, pooling_params=max_params ) 50 | self.layer4 = self.maker_layers( channel[2], channel[3], conv_params=conv_params, pooling_params=max_params ) 51 | h,w = GetFeatureMapSize(data_name,sub_number,padding = True) 52 | self.adaptiveAvg = nn.AdaptiveAvgPool2d( ( 1, w ) ) if use_adativeavg else nn.Identity() 53 | self.classifier = nn.Linear( channel[3]*w if use_adativeavg else channel[3]* h * w , channel[-1] ) 54 | 55 | def maker_layers(self,inp,oup,conv_params=None,pooling_params=None): 56 | assert isinstance(conv_params,list),print('the format of kernel params is error') 57 | assert isinstance(pooling_params,list) or pooling_params == None ,print('the format of pooling params is error') 58 | return nn.Sequential( 59 | nn.Conv2d( inp, oup, *(conv_params) ), 60 | nn.BatchNorm2d( oup ), 61 | nn.ReLU( True ), 62 | nn.MaxPool2d( *(pooling_params) ) if pooling_params else nn.Identity(), 63 | ) 64 | 65 | def forward(self, x: torch.Tensor): 66 | B,_,_,_ = x.size() 67 | if self.dataset in ['uschad','motion','ucihar']: 68 | x = F.pad(x, pad=(0,0,1,1), mode='replicate') 69 | x = self.layer1(x) 70 | x = self.layer2(x) 71 | x = self.layer3(x) 72 | x = self.layer4(x) 73 | x = self.adaptiveAvg(x) 74 | x = x.view(B,-1) 75 | x = self.classifier( x ) 76 | res = {} 77 | res['output'] = x 78 | return res 79 | 80 | 81 | 82 | class BaseCNNSup(nn.Module): 83 | def __init__(self, data_name='ucihar' ,use_adativeavg=False, sub_number = 4 ): 84 | super(BaseCNNSup, self).__init__() 85 | self.dataset = data_name 86 | self.channel= channel = channel_list[data_name] 87 | conv_params = conv_list[data_name] 88 | max_params = maxp_list[data_name] 89 | first_max_params = first_maxp_list[data_name] 90 | 91 | self.layer1 = self.maker_layers( 1 , channel[0], conv_params=conv_params, pooling_params=first_max_params ) 92 | self.layer2 = self.maker_layers( channel[0], channel[1], conv_params=conv_params, pooling_params=max_params ) 93 | self.layer3 = self.maker_layers( channel[1], channel[2], conv_params=conv_params, pooling_params=max_params ) 94 | self.layer4 = self.maker_layers( channel[2], channel[3], conv_params=conv_params, pooling_params=max_params ) 95 | h,w = GetFeatureMapSize(data_name,sub_number,padding = True, larger_max= True , is_sup= True ) 96 | self.adaptiveAvg = nn.AdaptiveAvgPool2d( ( 1, w ) ) if use_adativeavg else nn.Identity() 97 | self.classifier = nn.Linear( channel[3]*w if use_adativeavg else channel[3]* h * w , channel[-1] ) 98 | 99 | def maker_layers(self,inp,oup,conv_params=None,pooling_params=None): 100 | assert isinstance(conv_params,list),print('the format of kernel params is error') 101 | assert isinstance(pooling_params,list) or pooling_params == None ,print('the format of pooling params is error') 102 | return nn.Sequential( 103 | nn.Conv2d( inp, oup, *(conv_params) ), 104 | nn.BatchNorm2d( oup ), 105 | nn.ReLU( True ), 106 | nn.MaxPool2d( *(pooling_params) ) if pooling_params else nn.Identity(), 107 | ) 108 | 109 | def forward(self, x: torch.Tensor): 110 | B,_,_,_ = x.size() 111 | if self.dataset in ['uschad']: 112 | x = F.pad(x, pad=(0,0,1,1), mode='replicate') 113 | x = self.layer1(x) 114 | x = self.layer2(x) 115 | x = self.layer3(x) 116 | x = self.layer4(x) 117 | x = self.adaptiveAvg(x) 118 | x = x.view(B,-1) 119 | x = self.classifier( x ) 120 | res = {} 121 | res['output'] = x 122 | return res 123 | 124 | if __name__ == "__main__": 125 | import torch 126 | f_size = { 127 | 'ucihar': (30 , 9 ), 128 | 'uschad': (30 , 6 ), 129 | 'motion': (30 , 6 ) 130 | } 131 | dataset = 'ucihar' 132 | x = torch.randn(64,1,*f_size[dataset]) 133 | model = BaseCNN(dataset) 134 | # print(model,model.layer2[1]) 135 | GetFlopsAndParams(dataset,model) 136 | time = GetInferredSpeed(dataset,model,500) 137 | print(f'==>inferred time is {time}') 138 | print(model(x)['output'].shape) 139 | n_all = sum(p.numel() for p in model.parameters()) 140 | n_trainable = sum(p.numel() for p in model.parameters() if p.requires_grad) 141 | print("Parameter Count: all {:,d}; trainable {:,d}".format(n_all, n_trainable)) -------------------------------------------------------------------------------- /dataloaders/dataloader_MOTION_Sense_har.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import os 4 | 5 | from dataloaders.dataloader_base import BASE_DATA 6 | 7 | # ======================================== MotionSense_HAR_DATA ============================= 8 | class MotionSense_HAR_DATA(BASE_DATA): 9 | """ 10 | MotionSense Dataset 11 | https://github.com/mmalekzadeh/motion-sense 12 | 13 | Brief Description of the Dataset: 14 | --------------------------------- 15 | 16 | This dataset includes time-series data generated by accelerometer and gyroscope sensors 17 | (attitude, gravity, userAcceleration, and rotationRate). 18 | It is collected with an iPhone 6s kept in the participant's front pocket using SensingKit 19 | which collects information from Core Motion framework on iOS devices. 20 | All data collected in 50Hz sample rate. 21 | 22 | A total of 24 participants in a range of gender, age, weight, and height 23 | performed 6 activities in 15 trials in the same environment and conditions: 24 | 25 | downstairs, upstairs, walking, jogging, sitting, and standing. 26 | 27 | With this dataset, we aim to look for personal attributes fingerprints in time-series of sensor data, 28 | i.e. attribute-specific patterns that can be used to infer gender or personality of the data subjects in addition to their activities. 29 | 30 | 31 | There three different folders. Usually, you just need the folder (A) (DeviceMotion), 32 | because this folder includes everything that can be captured by both Accelerometer and Gyroscope. 33 | 34 | DeviceMotion_data 35 | This folder contains time-series collected by both Accelerometer and Gyroscope for all 15 trials. 36 | 37 | Thus, we have time-series with 12 features: 38 | 39 | attitude.roll 40 | attitude.pitch 41 | attitude.yaw 42 | gravity.x 43 | gravity.y 44 | gravity.z 45 | rotationRate.x 46 | rotationRate.y 47 | rotationRate.z 48 | userAcceleration.x 49 | userAcceleration.y 50 | userAcceleration.z 51 | 52 | 53 | Labels 54 | There are 6 different labels: 55 | 56 | dws: downstairs 57 | ups: upstairs 58 | sit: sitting 59 | std: standing 60 | wlk: walking 61 | jog: jogging 62 | """ 63 | 64 | def __init__(self, args): 65 | 66 | """ 67 | root_path : Root directory of the data set 68 | difference (bool) : Whether to calculate the first order derivative of the original data 69 | datanorm_type (str) : Methods of data normalization: "standardization", "minmax" , "per_sample_std", "per_sample_minmax" 70 | 71 | spectrogram (bool): Whether to convert raw data into frequency representations 72 | scales : Depends on the sampling frequency of the data （ UCI 数据的采样频率？？） 73 | wavelet : Methods of wavelet transformation 74 | 75 | """ 76 | 77 | self.used_cols = [] # no use , use the all columns, 78 | self.col_names = [] # the original files have column name 79 | 80 | self.label_map = [(0, 'dws'), # downstairs 81 | (1, "ups"), # upstairs 82 | (2, "sit"), # sitting 83 | (3, "std"), # standing 84 | (4, "wlk"), # walking 85 | (5, "jog"), # jogging 86 | ] 87 | self.drop_activities = [] 88 | 89 | # TODO , There all in total 25 subjects ! 90 | # The original train test is ## We consider long trials as training dataset and short trials as test dataset 91 | # The use trial to do the train test split 92 | 93 | self.train_keys = [1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,19,20,21,22,23] # 5,11,17,23 94 | self.vali_keys = [] 95 | self.test_keys = [6,12,18,24] 96 | 97 | self.exp_mode = args.exp_mode 98 | self.down_sample:bool \ 99 | = args.down_sample 100 | 101 | if self.exp_mode == "LOCV": 102 | self.split_tag = "sub" 103 | else: 104 | self.split_tag = "sub" 105 | 106 | self.all_keys = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24] 107 | self.LOCV_keys = [list(elem) for elem in np.array_split(self.all_keys,5)] 108 | self.sub_ids_of_each_sub = {} 109 | 110 | self.file_encoding = {} # no use 111 | 112 | 113 | self.labelToId = {int(x[0]): i for i, x in enumerate(self.label_map)} 114 | self.all_labels = list(range(len(self.label_map))) 115 | 116 | self.drop_activities = [self.labelToId[i] for i in self.drop_activities] 117 | self.no_drop_activites = [item for item in self.all_labels if item not in self.drop_activities] 118 | 119 | super(MotionSense_HAR_DATA, self).__init__(args) 120 | 121 | def load_all_the_data(self, root_path): 122 | print(" ----------------------- load all the data -------------------") 123 | 124 | trial_codes = {"dws":[1,2,11], "ups":[3,4,12], "wlk":[7,8,15], "jog":[9,16], "sit":[5,13], "std":[6,14]} 125 | 126 | df_dict = {} 127 | 128 | for sub in range(1,25): 129 | 130 | for act in ('dws', 'ups', 'wlk', 'jog', 'sit', 'std'): 131 | 132 | for trial in trial_codes[act]: 133 | 134 | fname = os.path.join(root_path,'A_DeviceMotion_data/'+act+'_'+str(trial)+'/sub_'+str(int(sub))+'.csv') 135 | sub_data = pd.read_csv(fname) 136 | sub_data = sub_data.drop(['Unnamed: 0'], axis=1) 137 | 138 | sub_data["activity_id"] = act 139 | 140 | sub_id = "{}_{}_{}".format(sub,act,trial) 141 | sub_data["sub_id"] = sub_id 142 | sub_data["sub"] = sub 143 | 144 | df_dict[sub_id] = sub_data 145 | 146 | if self.split_tag != 'sub': 147 | if trial > 10: 148 | self.test_keys.append(sub_id) 149 | else: 150 | self.train_keys.append(sub_id) 151 | 152 | if sub not in self.sub_ids_of_each_sub.keys(): 153 | self.sub_ids_of_each_sub[sub] = [] 154 | self.sub_ids_of_each_sub[sub].append(sub_id) 155 | 156 | df_all = pd.concat(df_dict) 157 | 158 | if self.down_sample: 159 | # Downsampling! form 50hz to 30hz 160 | df_all.reset_index(drop=True,inplace=True) 161 | index_list = list( np.arange(0,df_all.shape[0],5/3).astype(int) ) 162 | df_all = df_all.iloc[index_list] 163 | 164 | df_all = df_all.set_index('sub_id') 165 | 166 | label_mapping = {item[1]:item[0] for item in self.label_map} 167 | # because the activity label in the df is not encoded, thet are 'dws', 'ups', 'wlk', 'jog', 'sit', 'std' 168 | # first, map them in to nummeric number 169 | df_all["activity_id"] = df_all["activity_id"].map(label_mapping) 170 | df_all["activity_id"] = df_all["activity_id"].map(self.labelToId) 171 | 172 | # reorder the columns as sensor1, sensor2... sensorn, sub, activity_id 173 | df_all = df_all[list(df_all.columns[:-2])+["sub"]+["activity_id"]] 174 | 175 | data_y = df_all.iloc[:,-1] 176 | data_x = df_all.iloc[:,:-1] 177 | # drop partial sensors for special task. 178 | # data_x = df_all.drop(columns=['attitude.roll', 'attitude.pitch', 'attitude.yaw', 'rotationRate.x', 'rotationRate.y', 'rotationRate.z']) 179 | # data_x = data_x.loc[:, ['userAcceleration.x', 'userAcceleration.y' ,'userAcceleration.z', 'gravity.x' , 'gravity.y' , 'gravity.z' , 'sub' ]] 180 | # acc, gra 181 | data_x = data_x.reset_index() 182 | # sub_id, sensor1, sensor2... sensorn, sub, 183 | return data_x, data_y 184 | -------------------------------------------------------------------------------- /MaskCAE/pretrain/utils/lamb.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | """ PyTorch Lamb optimizer w/ behaviour similar to NVIDIA FusedLamb 4 | This optimizer code was adapted from the following (starting with latest) 5 | * https://github.com/HabanaAI/Model-References/blob/2b435114fe8e31f159b1d3063b8280ae37af7423/PyTorch/nlp/bert/pretraining/lamb.py 6 | * https://github.com/NVIDIA/DeepLearningExamples/blob/master/PyTorch/LanguageModeling/Transformer-XL/pytorch/lamb.py 7 | * https://github.com/cybertronai/pytorch-lamb 8 | Use FusedLamb if you can (GPU). The reason for including this variant of Lamb is to have a version that is 9 | similar in behaviour to APEX FusedLamb if you aren't using NVIDIA GPUs or cannot install/use APEX. 10 | In addition to some cleanup, this Lamb impl has been modified to support PyTorch XLA and has been tested on TPU. 11 | Original copyrights for above sources are below. 12 | Modifications Copyright 2021 Ross Wightman 13 | """ 14 | import math 15 | 16 | import torch 17 | from torch.optim.optimizer import Optimizer 18 | 19 | 20 | class TheSameAsTimmLAMB(Optimizer): 21 | """Implements a pure pytorch variant of FuseLAMB (NvLamb variant) optimizer from apex.optimizers.FusedLAMB 22 | reference: https://github.com/NVIDIA/DeepLearningExamples/blob/master/PyTorch/LanguageModeling/Transformer-XL/pytorch/lamb.py 23 | 24 | LAMB was proposed in `Large Batch Optimization for Deep Learning: Training BERT in 76 minutes`_. 25 | 26 | Arguments: 27 | params (iterable): iterable of parameters to optimize or dicts defining parameter groups. 28 | lr (float, optional): learning rate. (default: 1e-3) 29 | betas (Tuple[float, float], optional): coefficients used for computing 30 | running averages of gradient and its norm. (default: (0.9, 0.999)) 31 | eps (float, optional): term added to the denominator to improve 32 | numerical stability. (default: 1e-8) 33 | weight_decay (float, optional): weight decay (L2 penalty) (default: 0) 34 | grad_averaging (bool, optional): whether apply (1-beta2) to grad when 35 | calculating running averages of gradient. (default: True) 36 | max_grad_norm (float, optional): value used to clip global grad norm (default: 1.0) 37 | trust_clip (bool): enable LAMBC trust ratio clipping (default: False) 38 | always_adapt (boolean, optional): Apply adaptive learning rate to 0.0 39 | weight decay parameter (default: False) 40 | 41 | .. _Large Batch Optimization for Deep Learning - Training BERT in 76 minutes: 42 | https://arxiv.org/abs/1904.00962 43 | .. _On the Convergence of Adam and Beyond: 44 | https://openreview.net/forum?id=ryQu7f-RZ 45 | """ 46 | 47 | def __init__( 48 | self, params, lr=1e-3, bias_correction=True, betas=(0.9, 0.999), eps=1e-6, 49 | weight_decay=0.01, grad_averaging=True, max_grad_norm=2.0, trust_clip=False, always_adapt=False): 50 | defaults = dict( 51 | lr=lr, bias_correction=bias_correction, betas=betas, eps=eps, weight_decay=weight_decay, 52 | grad_averaging=grad_averaging, max_grad_norm=max_grad_norm, 53 | trust_clip=trust_clip, always_adapt=always_adapt) 54 | super().__init__(params, defaults) 55 | print(f'[lamb1] max_grad_norm={max_grad_norm}') 56 | self.global_grad_norm = 0 57 | 58 | @torch.no_grad() 59 | def step(self, closure=None): 60 | """Performs a single optimization step. 61 | Arguments: 62 | closure (callable, optional): A closure that reevaluates the model 63 | and returns the loss. 64 | """ 65 | loss = None 66 | if closure is not None: 67 | with torch.enable_grad(): 68 | loss = closure() 69 | 70 | device = self.param_groups[0]['params'][0].device 71 | one_tensor = torch.tensor(1.0, device=device) # because torch.where doesn't handle scalars correctly 72 | global_grad_norm = torch.zeros(1, device=device) 73 | for group in self.param_groups: 74 | for p in group['params']: 75 | if p.grad is None: 76 | continue 77 | grad = p.grad 78 | if grad.is_sparse: 79 | raise RuntimeError('Lamb does not support sparse gradients, consider SparseAdam instad.') 80 | global_grad_norm.add_(grad.pow(2).sum()) 81 | 82 | global_grad_norm = torch.sqrt(global_grad_norm) 83 | self.global_grad_norm = global_grad_norm.item() 84 | max_grad_norm = torch.tensor(self.defaults['max_grad_norm'], device=device) 85 | clip_global_grad_norm = 1 / torch.where( 86 | global_grad_norm > max_grad_norm, 87 | global_grad_norm / max_grad_norm, 88 | one_tensor) 89 | 90 | for group in self.param_groups: 91 | bias_correction = 1 if group['bias_correction'] else 0 92 | beta1, beta2 = group['betas'] 93 | grad_averaging = 1 if group['grad_averaging'] else 0 94 | beta3 = 1 - beta1 if grad_averaging else 1.0 95 | 96 | # assume same step across group now to simplify things 97 | # per parameter step can be easily support by making it tensor, or pass list into kernel 98 | if 'step' in group: 99 | group['step'] += 1 100 | else: 101 | group['step'] = 1 102 | 103 | if bias_correction: 104 | bias_correction1 = 1 - beta1 ** group['step'] 105 | bias_correction2 = 1 - beta2 ** group['step'] 106 | else: 107 | bias_correction1, bias_correction2 = 1.0, 1.0 108 | 109 | for p in group['params']: 110 | if p.grad is None: 111 | continue 112 | grad = p.grad.mul_(clip_global_grad_norm) 113 | state = self.state[p] 114 | 115 | # State initialization 116 | if len(state) == 0: 117 | # Exponential moving average of gradient valuesa 118 | state['exp_avg'] = torch.zeros_like(p) 119 | # Exponential moving average of squared gradient values 120 | state['exp_avg_sq'] = torch.zeros_like(p) 121 | 122 | exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] 123 | 124 | # Decay the first and second moment running average coefficient 125 | exp_avg.mul_(beta1).add_(grad, alpha=beta3) # m_t 126 | exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) # v_t 127 | 128 | denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(group['eps']) 129 | update = (exp_avg / bias_correction1).div_(denom) 130 | 131 | weight_decay = group['weight_decay'] 132 | if weight_decay != 0: 133 | update.add_(p, alpha=weight_decay) 134 | 135 | if weight_decay != 0 or group['always_adapt']: 136 | # Layer-wise LR adaptation. By default, skip adaptation on parameters that are 137 | # excluded from weight decay, unless always_adapt == True, then always enabled. 138 | w_norm = p.norm(2.0) 139 | g_norm = update.norm(2.0) 140 | # FIXME nested where required since logical and/or not working in PT XLA 141 | trust_ratio = torch.where( 142 | w_norm > 0, 143 | torch.where(g_norm > 0, w_norm / g_norm, one_tensor), 144 | one_tensor, 145 | ) 146 | if group['trust_clip']: 147 | # LAMBC trust clipping, upper bound fixed at one 148 | trust_ratio = torch.minimum(trust_ratio, one_tensor) 149 | update.mul_(trust_ratio) 150 | 151 | p.add_(update, alpha=-group['lr']) 152 | 153 | return loss 154 | -------------------------------------------------------------------------------- /dataloaders/dataloader_USCHAD_har.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import os 4 | 5 | import scipy.io as sio 6 | from dataloaders.dataloader_base import BASE_DATA 7 | 8 | # ================================= USC_HAD_HAR_DATA ============================================ 9 | class USC_HAD_HAR_DATA(BASE_DATA): 10 | """ 11 | 12 | ********************************************** 13 | Section 1: Device Configuration 14 | 15 | 16 | 2. Sampling rate: 100Hz 17 | 3. Accelerometer range: +-6g 18 | 4. Gyroscope range: +-500dps 19 | 20 | 21 | ********************************************** 22 | Section 2: Data Format 23 | Each activity trial is stored in an .mat file. 24 | 25 | The naming convention of each .mat file is defined as: 26 | a"m"t"n".mat, where 27 | "a" stands for activity 28 | "m" stands for activity number 29 | "t" stands for trial 30 | "n" stands for trial number 31 | 32 | Each .mat file contains 13 fields: 33 | 1. title: USC Human Motion Database 34 | 2. version: it is version 1.0 for this first round data collection 35 | 3. date 36 | 4. subject number 37 | 5. age 38 | 6. height 39 | 7. weight 40 | 8. activity name 41 | 9. activity number 42 | 10. trial number 43 | 11. sensor_location 44 | 12. sensor_orientation 45 | 13. sensor_readings 46 | 47 | For sensor_readings field, it consists of 6 readings: 48 | From left to right: 49 | 1. acc_x, w/ unit g (gravity) 50 | 2. acc_y, w/ unit g 51 | 3. acc_z, w/ unit g 52 | 4. gyro_x, w/ unit dps (degrees per second) 53 | 5. gyro_y, w/ unit dps 54 | 6. gyro_z, w/ unit dps 55 | 56 | ********************************************** 57 | Section 3: Activities 58 | 1. Walking Forward 59 | 2. Walking Left 60 | 3. Walking Right 61 | 4. Walking Upstairs 62 | 5. Walking Downstairs 63 | 6. Running Forward 64 | 7. Jumping Up 65 | 8. Sitting 66 | 9. Standing 67 | 10. Sleeping 68 | 11. Elevator Up 69 | 12. Elevator Down 70 | 71 | """ 72 | 73 | def __init__(self, args): 74 | 75 | """ 76 | root_path : Root directory of the data set 77 | difference (bool) : Whether to calculate the first order derivative of the original data 78 | datanorm_type (str) : Methods of data normalization: "standardization", "minmax" , "per_sample_std", "per_sample_minmax" 79 | 80 | spectrogram (bool): Whether to convert raw data into frequency representations 81 | scales : Depends on the sampling frequency of the data （ UCI 数据的采样频率？？） 82 | wavelet : Methods of wavelet transformation 83 | 84 | """ 85 | 86 | # !!!!!! Depending on the setting of each data set!!!!!! 87 | # because this dataset only has 6 columns, the label is saved in the file name, so this used cols will not be used 88 | self.used_cols = [0,1,2,3,4,5] 89 | # This dataset only has this 6 channels 90 | self.col_names = [ 'acc_x', 'acc_y', 'acc_z', 'gyro_x', 'gyro_y', 'gyro_z' ] 91 | 92 | 93 | # These two variables represent whether all sensors can be filtered according to position and sensor type 94 | # pos_filter ------- > filter according to position 95 | # sensor_filter -----> filter according to the sensor type 96 | self.pos_filter = None 97 | self.sensor_filter = ["acc","gyro"] 98 | 99 | # selected_cols will be updated according to user settings. User have to set -- args.pos_select, args.sensor_select--- 100 | self.selected_cols = None 101 | # Filtering channels according to the Position 102 | self.selected_cols = self.Sensor_filter_acoording_to_pos_and_type(args.pos_select, self.pos_filter, self.col_names, "position") 103 | # Filtering channels according to the Sensor Type 104 | if self.selected_cols is None: 105 | self.selected_cols = self.Sensor_filter_acoording_to_pos_and_type(args.sensor_select, self.sensor_filter, self.col_names, "Sensor Type") 106 | else: 107 | self.selected_cols = self.Sensor_filter_acoording_to_pos_and_type(args.sensor_select, self.sensor_filter, self.selected_cols, "Sensor Type") 108 | 109 | 110 | # The original labels are from 1 to 12 111 | self.label_map = [(1, "Walking Forward"), 112 | (2, "Walking Left"), 113 | (3, "Walking Right"), 114 | (4, "Walking Upstairs"), 115 | (5, "Walking Downstairs"), 116 | (6, "Running Forward"), 117 | (7, "Jumping Up"), 118 | (8, "Sitting"), 119 | (9, "Standing"), 120 | (10, "Sleeping"), 121 | (11, "Elevator Up"), 122 | (12, "Elevator Down")] 123 | 124 | # As can be seen from the readme 125 | self.drop_activities = [] 126 | 127 | 128 | 129 | self.train_keys = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 ] 130 | self.vali_keys = [ 11, 12 ] 131 | self.test_keys = [ 13, 14 ] 132 | 133 | self.exp_mode = args.exp_mode 134 | self.down_sample:bool \ 135 | = args.down_sample 136 | 137 | self.split_tag = "sub" 138 | 139 | self.LOCV_keys = [[1,2],[3,4],[5,6],[7,8],[9,10],[11,12],[13,14]] 140 | self.all_keys = [1,2,3,4,5,6,7,8,9,10,11,12,13,14] 141 | self.sub_ids_of_each_sub = {} 142 | 143 | self.file_encoding = {} # no use 144 | 145 | self.labelToId = {int(x[0]): i for i, x in enumerate(self.label_map)} 146 | self.all_labels = list(range(len(self.label_map))) 147 | 148 | self.drop_activities = [self.labelToId[i] for i in self.drop_activities] 149 | self.no_drop_activites = [item for item in self.all_labels if item not in self.drop_activities] 150 | 151 | super(USC_HAD_HAR_DATA, self).__init__(args) 152 | 153 | def load_all_the_data(self, root_path): 154 | print(" ----------------------- load all the data -------------------") 155 | 156 | activities = range(1, 13) 157 | 158 | df_dict = {} 159 | for sub in range(1, 15): 160 | 161 | for activity in activities: 162 | 163 | for trial in range(1, 6): 164 | 165 | sub_data = sio.loadmat("%s/Subject%d%sa%dt%d.mat" % (root_path, sub, os.sep, activity, trial)) 166 | sub_data = pd.DataFrame(np.array(sub_data['sensor_readings'])) 167 | 168 | sub_data =sub_data.iloc[:,self.used_cols] 169 | sub_data.columns = self.col_names 170 | 171 | sub_id = "{}_{}_{}".format(sub,activity,trial) 172 | sub_data["sub_id"] = sub_id 173 | sub_data["sub"] = sub 174 | sub_data["activity_id"] = activity 175 | 176 | df_dict[sub_id] = sub_data 177 | 178 | if sub not in self.sub_ids_of_each_sub.keys(): 179 | self.sub_ids_of_each_sub[sub] = [] 180 | self.sub_ids_of_each_sub[sub].append(sub_id) 181 | 182 | 183 | df_all = pd.concat(df_dict) 184 | 185 | if self.down_sample: 186 | # Downsampling! form 100hz to 30hz 187 | df_all.reset_index(drop=True,inplace=True) 188 | index_list = list( np.arange(0,df_all.shape[0],10/3).astype(int) ) 189 | df_all = df_all.iloc[index_list] 190 | 191 | df_all = df_all.set_index('sub_id') 192 | 193 | 194 | # reorder the columns as sensor1, sensor2... sensorn, sub, activity_id 195 | if self.selected_cols: 196 | df_all = df_all[self.selected_cols+["sub"]+["activity_id"]] 197 | else: 198 | df_all = df_all[self.col_names+["sub"]+["activity_id"]] 199 | 200 | # Label Transformation 201 | df_all["activity_id"] = df_all["activity_id"].map(self.labelToId) 202 | 203 | data_y = df_all.iloc[:,-1] 204 | data_x = df_all.iloc[:,:-1] 205 | 206 | data_x = data_x.reset_index() 207 | # sub_id, sensor1, sensor2... sensorn, sub, 208 | return data_x, data_y 209 | -------------------------------------------------------------------------------- /utils/lamb.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) ByteDance, Inc. and its affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | # 7 | # This file is basically a copy to: https://github.com/rwightman/pytorch-image-models/blob/v0.5.4/timm/optim/lamb.py 8 | # **The only modification** is adding the `global_grad_norm` member for debugging 9 | 10 | 11 | """ PyTorch Lamb optimizer w/ behaviour similar to NVIDIA FusedLamb 12 | This optimizer code was adapted from the following (starting with latest) 13 | * https://github.com/HabanaAI/Model-References/blob/2b435114fe8e31f159b1d3063b8280ae37af7423/PyTorch/nlp/bert/pretraining/lamb.py 14 | * https://github.com/NVIDIA/DeepLearningExamples/blob/master/PyTorch/LanguageModeling/Transformer-XL/pytorch/lamb.py 15 | * https://github.com/cybertronai/pytorch-lamb 16 | Use FusedLamb if you can (GPU). The reason for including this variant of Lamb is to have a version that is 17 | similar in behaviour to APEX FusedLamb if you aren't using NVIDIA GPUs or cannot install/use APEX. 18 | In addition to some cleanup, this Lamb impl has been modified to support PyTorch XLA and has been tested on TPU. 19 | Original copyrights for above sources are below. 20 | Modifications Copyright 2021 Ross Wightman 21 | """ 22 | import math 23 | 24 | import torch 25 | from torch.optim.optimizer import Optimizer 26 | 27 | 28 | class TheSameAsTimmLAMB(Optimizer): 29 | """Implements a pure pytorch variant of FuseLAMB (NvLamb variant) optimizer from apex.optimizers.FusedLAMB 30 | reference: https://github.com/NVIDIA/DeepLearningExamples/blob/master/PyTorch/LanguageModeling/Transformer-XL/pytorch/lamb.py 31 | 32 | LAMB was proposed in `Large Batch Optimization for Deep Learning: Training BERT in 76 minutes`_. 33 | 34 | Arguments: 35 | params (iterable): iterable of parameters to optimize or dicts defining parameter groups. 36 | lr (float, optional): learning rate. (default: 1e-3) 37 | betas (Tuple[float, float], optional): coefficients used for computing 38 | running averages of gradient and its norm. (default: (0.9, 0.999)) 39 | eps (float, optional): term added to the denominator to improve 40 | numerical stability. (default: 1e-8) 41 | weight_decay (float, optional): weight decay (L2 penalty) (default: 0) 42 | grad_averaging (bool, optional): whether apply (1-beta2) to grad when 43 | calculating running averages of gradient. (default: True) 44 | max_grad_norm (float, optional): value used to clip global grad norm (default: 1.0) 45 | trust_clip (bool): enable LAMBC trust ratio clipping (default: False) 46 | always_adapt (boolean, optional): Apply adaptive learning rate to 0.0 47 | weight decay parameter (default: False) 48 | 49 | .. _Large Batch Optimization for Deep Learning - Training BERT in 76 minutes: 50 | https://arxiv.org/abs/1904.00962 51 | .. _On the Convergence of Adam and Beyond: 52 | https://openreview.net/forum?id=ryQu7f-RZ 53 | """ 54 | 55 | def __init__( 56 | self, params, lr=1e-3, bias_correction=True, betas=(0.9, 0.999), eps=1e-6, 57 | weight_decay=0.01, grad_averaging=True, max_grad_norm=2.0, trust_clip=False, always_adapt=False): 58 | defaults = dict( 59 | lr=lr, bias_correction=bias_correction, betas=betas, eps=eps, weight_decay=weight_decay, 60 | grad_averaging=grad_averaging, max_grad_norm=max_grad_norm, 61 | trust_clip=trust_clip, always_adapt=always_adapt) 62 | super().__init__(params, defaults) 63 | print(f'[lamb1] max_grad_norm={max_grad_norm}') 64 | self.global_grad_norm = 0 65 | 66 | @torch.no_grad() 67 | def step(self, closure=None): 68 | """Performs a single optimization step. 69 | Arguments: 70 | closure (callable, optional): A closure that reevaluates the model 71 | and returns the loss. 72 | """ 73 | loss = None 74 | if closure is not None: 75 | with torch.enable_grad(): 76 | loss = closure() 77 | 78 | device = self.param_groups[0]['params'][0].device 79 | one_tensor = torch.tensor(1.0, device=device) # because torch.where doesn't handle scalars correctly 80 | global_grad_norm = torch.zeros(1, device=device) 81 | for group in self.param_groups: 82 | for p in group['params']: 83 | if p.grad is None: 84 | continue 85 | grad = p.grad 86 | if grad.is_sparse: 87 | raise RuntimeError('Lamb does not support sparse gradients, consider SparseAdam instad.') 88 | global_grad_norm.add_(grad.pow(2).sum()) 89 | 90 | global_grad_norm = torch.sqrt(global_grad_norm) 91 | self.global_grad_norm = global_grad_norm.item() 92 | max_grad_norm = torch.tensor(self.defaults['max_grad_norm'], device=device) 93 | clip_global_grad_norm = 1 / torch.where( 94 | global_grad_norm > max_grad_norm, 95 | global_grad_norm / max_grad_norm, 96 | one_tensor) 97 | 98 | for group in self.param_groups: 99 | bias_correction = 1 if group['bias_correction'] else 0 100 | beta1, beta2 = group['betas'] 101 | grad_averaging = 1 if group['grad_averaging'] else 0 102 | beta3 = 1 - beta1 if grad_averaging else 1.0 103 | 104 | # assume same step across group now to simplify things 105 | # per parameter step can be easily support by making it tensor, or pass list into kernel 106 | if 'step' in group: 107 | group['step'] += 1 108 | else: 109 | group['step'] = 1 110 | 111 | if bias_correction: 112 | bias_correction1 = 1 - beta1 ** group['step'] 113 | bias_correction2 = 1 - beta2 ** group['step'] 114 | else: 115 | bias_correction1, bias_correction2 = 1.0, 1.0 116 | 117 | for p in group['params']: 118 | if p.grad is None: 119 | continue 120 | grad = p.grad.mul_(clip_global_grad_norm) 121 | state = self.state[p] 122 | 123 | # State initialization 124 | if len(state) == 0: 125 | # Exponential moving average of gradient valuesa 126 | state['exp_avg'] = torch.zeros_like(p) 127 | # Exponential moving average of squared gradient values 128 | state['exp_avg_sq'] = torch.zeros_like(p) 129 | 130 | exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] 131 | 132 | # Decay the first and second moment running average coefficient 133 | exp_avg.mul_(beta1).add_(grad, alpha=beta3) # m_t 134 | exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) # v_t 135 | 136 | denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(group['eps']) 137 | update = (exp_avg / bias_correction1).div_(denom) 138 | 139 | weight_decay = group['weight_decay'] 140 | if weight_decay != 0: 141 | update.add_(p, alpha=weight_decay) 142 | 143 | if weight_decay != 0 or group['always_adapt']: 144 | # Layer-wise LR adaptation. By default, skip adaptation on parameters that are 145 | # excluded from weight decay, unless always_adapt == True, then always enabled. 146 | w_norm = p.norm(2.0) 147 | g_norm = update.norm(2.0) 148 | # FIXME nested where required since logical and/or not working in PT XLA 149 | trust_ratio = torch.where( 150 | w_norm > 0, 151 | torch.where(g_norm > 0, w_norm / g_norm, one_tensor), 152 | one_tensor, 153 | ) 154 | if group['trust_clip']: 155 | # LAMBC trust clipping, upper bound fixed at one 156 | trust_ratio = torch.minimum(trust_ratio, one_tensor) 157 | update.mul_(trust_ratio) 158 | 159 | p.add_(update, alpha=-group['lr']) 160 | 161 | return loss 162 | -------------------------------------------------------------------------------- /pretrain/utils/lamb.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) ByteDance, Inc. and its affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | # 7 | # This file is basically a copy to: https://github.com/rwightman/pytorch-image-models/blob/v0.5.4/timm/optim/lamb.py 8 | # **The only modification** is adding the `global_grad_norm` member for debugging 9 | 10 | 11 | """ PyTorch Lamb optimizer w/ behaviour similar to NVIDIA FusedLamb 12 | This optimizer code was adapted from the following (starting with latest) 13 | * https://github.com/HabanaAI/Model-References/blob/2b435114fe8e31f159b1d3063b8280ae37af7423/PyTorch/nlp/bert/pretraining/lamb.py 14 | * https://github.com/NVIDIA/DeepLearningExamples/blob/master/PyTorch/LanguageModeling/Transformer-XL/pytorch/lamb.py 15 | * https://github.com/cybertronai/pytorch-lamb 16 | Use FusedLamb if you can (GPU). The reason for including this variant of Lamb is to have a version that is 17 | similar in behaviour to APEX FusedLamb if you aren't using NVIDIA GPUs or cannot install/use APEX. 18 | In addition to some cleanup, this Lamb impl has been modified to support PyTorch XLA and has been tested on TPU. 19 | Original copyrights for above sources are below. 20 | Modifications Copyright 2021 Ross Wightman 21 | """ 22 | import math 23 | 24 | import torch 25 | from torch.optim.optimizer import Optimizer 26 | 27 | 28 | class TheSameAsTimmLAMB(Optimizer): 29 | """Implements a pure pytorch variant of FuseLAMB (NvLamb variant) optimizer from apex.optimizers.FusedLAMB 30 | reference: https://github.com/NVIDIA/DeepLearningExamples/blob/master/PyTorch/LanguageModeling/Transformer-XL/pytorch/lamb.py 31 | 32 | LAMB was proposed in `Large Batch Optimization for Deep Learning: Training BERT in 76 minutes`_. 33 | 34 | Arguments: 35 | params (iterable): iterable of parameters to optimize or dicts defining parameter groups. 36 | lr (float, optional): learning rate. (default: 1e-3) 37 | betas (Tuple[float, float], optional): coefficients used for computing 38 | running averages of gradient and its norm. (default: (0.9, 0.999)) 39 | eps (float, optional): term added to the denominator to improve 40 | numerical stability. (default: 1e-8) 41 | weight_decay (float, optional): weight decay (L2 penalty) (default: 0) 42 | grad_averaging (bool, optional): whether apply (1-beta2) to grad when 43 | calculating running averages of gradient. (default: True) 44 | max_grad_norm (float, optional): value used to clip global grad norm (default: 1.0) 45 | trust_clip (bool): enable LAMBC trust ratio clipping (default: False) 46 | always_adapt (boolean, optional): Apply adaptive learning rate to 0.0 47 | weight decay parameter (default: False) 48 | 49 | .. _Large Batch Optimization for Deep Learning - Training BERT in 76 minutes: 50 | https://arxiv.org/abs/1904.00962 51 | .. _On the Convergence of Adam and Beyond: 52 | https://openreview.net/forum?id=ryQu7f-RZ 53 | """ 54 | 55 | def __init__( 56 | self, params, lr=1e-3, bias_correction=True, betas=(0.9, 0.999), eps=1e-6, 57 | weight_decay=0.01, grad_averaging=True, max_grad_norm=2.0, trust_clip=False, always_adapt=False): 58 | defaults = dict( 59 | lr=lr, bias_correction=bias_correction, betas=betas, eps=eps, weight_decay=weight_decay, 60 | grad_averaging=grad_averaging, max_grad_norm=max_grad_norm, 61 | trust_clip=trust_clip, always_adapt=always_adapt) 62 | super().__init__(params, defaults) 63 | print(f'[lamb1] max_grad_norm={max_grad_norm}') 64 | self.global_grad_norm = 0 65 | 66 | @torch.no_grad() 67 | def step(self, closure=None): 68 | """Performs a single optimization step. 69 | Arguments: 70 | closure (callable, optional): A closure that reevaluates the model 71 | and returns the loss. 72 | """ 73 | loss = None 74 | if closure is not None: 75 | with torch.enable_grad(): 76 | loss = closure() 77 | 78 | device = self.param_groups[0]['params'][0].device 79 | one_tensor = torch.tensor(1.0, device=device) # because torch.where doesn't handle scalars correctly 80 | global_grad_norm = torch.zeros(1, device=device) 81 | for group in self.param_groups: 82 | for p in group['params']: 83 | if p.grad is None: 84 | continue 85 | grad = p.grad 86 | if grad.is_sparse: 87 | raise RuntimeError('Lamb does not support sparse gradients, consider SparseAdam instad.') 88 | global_grad_norm.add_(grad.pow(2).sum()) 89 | 90 | global_grad_norm = torch.sqrt(global_grad_norm) 91 | self.global_grad_norm = global_grad_norm.item() 92 | max_grad_norm = torch.tensor(self.defaults['max_grad_norm'], device=device) 93 | clip_global_grad_norm = 1 / torch.where( 94 | global_grad_norm > max_grad_norm, 95 | global_grad_norm / max_grad_norm, 96 | one_tensor) 97 | 98 | for group in self.param_groups: 99 | bias_correction = 1 if group['bias_correction'] else 0 100 | beta1, beta2 = group['betas'] 101 | grad_averaging = 1 if group['grad_averaging'] else 0 102 | beta3 = 1 - beta1 if grad_averaging else 1.0 103 | 104 | # assume same step across group now to simplify things 105 | # per parameter step can be easily support by making it tensor, or pass list into kernel 106 | if 'step' in group: 107 | group['step'] += 1 108 | else: 109 | group['step'] = 1 110 | 111 | if bias_correction: 112 | bias_correction1 = 1 - beta1 ** group['step'] 113 | bias_correction2 = 1 - beta2 ** group['step'] 114 | else: 115 | bias_correction1, bias_correction2 = 1.0, 1.0 116 | 117 | for p in group['params']: 118 | if p.grad is None: 119 | continue 120 | grad = p.grad.mul_(clip_global_grad_norm) 121 | state = self.state[p] 122 | 123 | # State initialization 124 | if len(state) == 0: 125 | # Exponential moving average of gradient valuesa 126 | state['exp_avg'] = torch.zeros_like(p) 127 | # Exponential moving average of squared gradient values 128 | state['exp_avg_sq'] = torch.zeros_like(p) 129 | 130 | exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] 131 | 132 | # Decay the first and second moment running average coefficient 133 | exp_avg.mul_(beta1).add_(grad, alpha=beta3) # m_t 134 | exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) # v_t 135 | 136 | denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(group['eps']) 137 | update = (exp_avg / bias_correction1).div_(denom) 138 | 139 | weight_decay = group['weight_decay'] 140 | if weight_decay != 0: 141 | update.add_(p, alpha=weight_decay) 142 | 143 | if weight_decay != 0 or group['always_adapt']: 144 | # Layer-wise LR adaptation. By default, skip adaptation on parameters that are 145 | # excluded from weight decay, unless always_adapt == True, then always enabled. 146 | w_norm = p.norm(2.0) 147 | g_norm = update.norm(2.0) 148 | # FIXME nested where required since logical and/or not working in PT XLA 149 | trust_ratio = torch.where( 150 | w_norm > 0, 151 | torch.where(g_norm > 0, w_norm / g_norm, one_tensor), 152 | one_tensor, 153 | ) 154 | if group['trust_clip']: 155 | # LAMBC trust clipping, upper bound fixed at one 156 | trust_ratio = torch.minimum(trust_ratio, one_tensor) 157 | update.mul_(trust_ratio) 158 | 159 | p.add_(update, alpha=-group['lr']) 160 | 161 | return loss 162 | -------------------------------------------------------------------------------- /dataloaders/utils.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | 4 | # necessary functions 5 | from scipy.fftpack import fft,fftfreq,ifft 6 | # Importing Scipy 7 | import scipy as sp 8 | import pywt 9 | import scipy.fft as F 10 | from sklearn.cluster import KMeans 11 | import torch 12 | 13 | 14 | def mag_3_signals(x,y,z):# magnitude function redefintion 15 | return np.array([math.sqrt((x[i]**2+y[i]**2+z[i]**2)) for i in range(len(x))]) 16 | 17 | def components_selection_one_signal(t_signal,freq1,freq2,sampling_freq): 18 | """ 19 | DC_component: f_signal values having freq between [-0.3 hz to 0 hz] and from [0 hz to 0.3hz] 20 | (-0.3 and 0.3 are included) 21 | 22 | noise components: f_signal values having freq between [-25 hz to 20 hz[ and from ] 20 hz to 25 hz] 23 | (-25 and 25 hz inculded 20hz and -20hz not included) 24 | 25 | selecting body_component: f_signal values having freq between [-20 hz to -0.3 hz] and from [0.3 hz to 20 hz] 26 | (-0.3 and 0.3 not included , -20hz and 20 hz included) 27 | """ 28 | 29 | t_signal=np.array(t_signal) 30 | t_signal_length=len(t_signal) # number of points in a t_signal 31 | 32 | # the t_signal in frequency domain after applying fft 33 | f_signal=fft(t_signal) # 1D numpy array contains complex values (in C) 34 | 35 | # generate frequencies associated to f_signal complex values 36 | freqs=np.array(sp.fftpack.fftfreq(t_signal_length, d=1/float(sampling_freq))) # frequency values between [-25hz:+25hz] 37 | 38 | 39 | 40 | 41 | f_DC_signal=[] # DC_component in freq domain 42 | f_body_signal=[] # body component in freq domain numpy.append(a, a[0]) 43 | f_noise_signal=[] # noise in freq domain 44 | 45 | for i in range(len(freqs)):# iterate over all available frequencies 46 | 47 | # selecting the frequency value 48 | freq=freqs[i] 49 | 50 | # selecting the f_signal value associated to freq 51 | value= f_signal[i] 52 | 53 | # Selecting DC_component values 54 | if abs(freq)>freq1:# testing if freq is outside DC_component frequency ranges 55 | f_DC_signal.append(float(0)) # add 0 to the list if it was the case (the value should not be added) 56 | else: # if freq is inside DC_component frequency ranges 57 | f_DC_signal.append(value) # add f_signal value to f_DC_signal list 58 | 59 | # Selecting noise component values 60 | if (abs(freq)<=freq2):# testing if freq is outside noise frequency ranges 61 | f_noise_signal.append(float(0)) # # add 0 to f_noise_signal list if it was the case 62 | else:# if freq is inside noise frequency ranges 63 | f_noise_signal.append(value) # add f_signal value to f_noise_signal 64 | 65 | # Selecting body_component values 66 | if (abs(freq)<=freq1 or abs(freq)>freq2):# testing if freq is outside Body_component frequency ranges 67 | f_body_signal.append(float(0))# add 0 to f_body_signal list 68 | else:# if freq is inside Body_component frequency ranges 69 | f_body_signal.append(value) # add f_signal value to f_body_signal list 70 | 71 | ################### Inverse the transformation of signals in freq domain ######################## 72 | # applying the inverse fft(ifft) to signals in freq domain and put them in float format 73 | t_DC_component= ifft(np.array(f_DC_signal)).real 74 | t_body_component= ifft(np.array(f_body_signal)).real 75 | #t_noise=ifft(np.array(f_noise_signal)).real 76 | 77 | #total_component=t_signal-t_noise # extracting the total component(filtered from noise) 78 | # # by substracting noise from t_signal (the original signal). 79 | 80 | 81 | #return (total_component,t_DC_component,t_body_component,t_noise) 82 | return (t_DC_component,t_body_component) 83 | 84 | 85 | 86 | class Normalizer(object): 87 | """ 88 | Normalizes dataframe across ALL contained rows (time steps). Different from per-sample normalization. 89 | """ 90 | 91 | def __init__(self, norm_type): 92 | """ 93 | Args: 94 | norm_type: choose from: 95 | "standardization", "minmax": normalizes dataframe across ALL contained rows (time steps) 96 | "per_sample_std", "per_sample_minmax": normalizes each sample separately (i.e. across only its own rows) 97 | mean, std, min_val, max_val: optional (num_feat,) Series of pre-computed values 98 | """ 99 | 100 | self.norm_type = norm_type 101 | 102 | def fit(self, df): 103 | if self.norm_type == "standardization": 104 | self.mean = df.mean(0) 105 | self.std = df.std(0) 106 | elif self.norm_type == "minmax": 107 | self.max_val = df.max() 108 | self.min_val = df.min() 109 | elif self.norm_type == "per_sample_std": 110 | self.max_val = None 111 | self.min_val = None 112 | elif self.norm_type == "per_sample_minmax": 113 | self.max_val = None 114 | self.min_val = None 115 | else: 116 | raise (NameError(f'Normalize method "{self.norm_type}" not implemented')) 117 | 118 | def normalize(self, df): 119 | """ 120 | Args: 121 | df: input dataframe 122 | Returns: 123 | df: normalized dataframe 124 | """ 125 | if self.norm_type == "standardization": 126 | return (df - self.mean) / (self.std + np.finfo(float).eps) 127 | 128 | elif self.norm_type == "minmax": 129 | return (df - self.min_val) / (self.max_val - self.min_val + np.finfo(float).eps) 130 | elif self.norm_type == "per_sample_std": 131 | grouped = df.groupby(by=df.index) 132 | return (df - grouped.transform('mean')) / grouped.transform('std') 133 | elif self.norm_type == "per_sample_minmax": 134 | grouped = df.groupby(by=df.index) 135 | min_vals = grouped.transform('min') 136 | return (df - min_vals) / (grouped.transform('max') - min_vals + np.finfo(float).eps) 137 | 138 | else: 139 | raise (NameError(f'Normalize method "{self.norm_type}" not implemented')) 140 | 141 | 142 | 143 | def PrepareWavelets(K, length=20, seed= 1): 144 | motherwavelets = [] 145 | for family in pywt.families(): 146 | for mother in pywt.wavelist(family): 147 | motherwavelets.append(mother) 148 | 149 | X = np.zeros([1,length]) 150 | PSI = np.zeros([1,length]) 151 | for mw_temp in motherwavelets: 152 | if mw_temp.startswith('gaus') or mw_temp.startswith('mexh') or mw_temp.startswith('morl') or mw_temp.startswith('cmor') or mw_temp.startswith('fbsp') or mw_temp.startswith('shan') or mw_temp.startswith('cgau'): 153 | pass 154 | else: 155 | param = pywt.Wavelet(mw_temp).wavefun(level=7) 156 | psi, x = param[1], param[-1] 157 | 158 | # normalization 159 | psi_sum = np.sum(psi) 160 | if np.abs(psi_sum) > 1: 161 | psi = psi / np.abs(psi_sum) 162 | x = x / max(x) 163 | 164 | # down sampling 165 | idx_ds = np.round(np.linspace(0, x.shape[0]-1, length)).astype(int) 166 | x = x[idx_ds] 167 | psi = psi[idx_ds] 168 | 169 | X = np.vstack((X, x.reshape(1,-1))) 170 | PSI = np.vstack((PSI, psi.reshape(1,-1))) 171 | 172 | X = X[1:,:] 173 | PSI = PSI[1:,:] 174 | 175 | # clustering 176 | FRE = np.zeros([1,length]) 177 | for i in range(PSI.shape[0]): 178 | FRE = np.vstack((FRE, np.real(F.fft(PSI[i,:])).reshape(1,-1))) 179 | FRE = FRE[1:,:] 180 | 181 | PSI_extended = np.hstack((PSI, FRE)) 182 | kmeans = KMeans(n_clusters=K, random_state=seed).fit(PSI_extended) 183 | label = kmeans.labels_ 184 | 185 | SelectedWavelet = np.zeros([1,length]) 186 | for k in range(K): 187 | wavesidx = np.where(label==k)[0][0] 188 | SelectedWavelet = np.vstack((SelectedWavelet, PSI[wavesidx,:])) 189 | 190 | return torch.tensor(SelectedWavelet[1:,:]) 191 | 192 | 193 | def FiltersExtention(Filters): 194 | K, WS = Filters.shape 195 | 196 | 197 | if WS%2==1: 198 | N_padding = int((WS-1)/2) 199 | N_ds = int(torch.log2(torch.tensor(WS-1)).floor()) - 2 200 | else: 201 | N_ds = int(torch.log2(torch.tensor(WS)).floor()) - 2 202 | N_padding = int(WS/2) 203 | 204 | Filter_temp = Filters.repeat(N_ds,1,1) 205 | m = torch.nn.ConstantPad1d(N_padding, 0) 206 | 207 | for n_ds in range(N_ds-1): 208 | filter_temp = Filter_temp[n_ds,:,:] 209 | filter_temp_pad = m(filter_temp) 210 | filter_ds = filter_temp_pad[:,::2] * 2. 211 | Filter_temp[n_ds+1,:,:] = filter_ds 212 | 213 | # formualte dimensionality 214 | Filter_temp = Filter_temp.view(K*N_ds,WS) 215 | Filter_temp = Filter_temp.repeat(1,1,1,1) 216 | Filter_temp = Filter_temp.permute(2,0,1,3) 217 | 218 | # normalization 219 | energy = torch.sum(Filter_temp, dim=3, keepdims=True) 220 | energy[torch.abs(energy)<=1] = 1. 221 | Filter_temp = Filter_temp / energy 222 | 223 | return Filter_temp 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | -------------------------------------------------------------------------------- /dataloaders/dataloader_uci_har.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import os 4 | 5 | from dataloaders.dataloader_base import BASE_DATA 6 | # ============================== UCI_HAR_DATA ====================================== 7 | class UCI_HAR_DATA(BASE_DATA): 8 | """ 9 | The experiments have been carried out with a group of 30 volunteers within an age bracket of 19-48 years. 10 | Each person performed six activities 11 | (WALKING, WALKING_UPSTAIRS, WALKING_DOWNSTAIRS, SITTING, STANDING, LAYING) 12 | wearing a smartphone (Samsung Galaxy S II) on the waist. Using its embedded accelerometer and gyroscope, 13 | we captured 3-axial linear acceleration and 3-axial angular velocity at a constant rate of 50Hz. 14 | The experiments have been video-recorded to label the data manually. 15 | The obtained dataset has been randomly partitioned into two sets, 16 | where 70% of the volunteers was selected for generating the training data and 30% the test data. 17 | 18 | The sensor signals (accelerometer and gyroscope) were pre-processed by applying noise filters 19 | and then sampled in fixed-width sliding windows of 2.56 sec and 50% overlap (128 readings/window). 20 | The sensor acceleration signal, which has gravitational and body motion components, 21 | was separated using a Butterworth low-pass filter into body acceleration and gravity. 22 | The gravitational force is assumed to have only low frequency components, 23 | therefore a filter with 0.3 Hz cutoff frequency was used. 24 | From each window, a vector of features was obtained by calculating variables from the time and frequency domain. 25 | See 'features_info.txt' for more details. 26 | 27 | 1 WALKING 28 | 2 WALKING_UPSTAIRS 29 | 3 WALKING_DOWNSTAIRS 30 | 4 SITTING 31 | 5 STANDING 32 | 6 LAYING 33 | """ 34 | def __init__(self, args): 35 | 36 | """ 37 | root_path : Root directory of the data set 38 | difference (bool) : Whether to calculate the first order derivative of the original data 39 | datanorm_type (str) : Methods of data normalization: "standardization", "minmax" , "per_sample_std", "per_sample_minmax" 40 | 41 | spectrogram (bool): Whether to convert raw data into frequency representations 42 | scales : Depends on the sampling frequency of the data （ UCI 数据的采样频率？？） 43 | wavelet : Methods of wavelet transformation 44 | 45 | """ 46 | self.used_cols = [] #no use , because this data format has save each sensor in one file 47 | self.col_names = ['body_acc_x_', 'body_acc_y_', 'body_acc_z_', 48 | 'body_gyro_x_', 'body_gyro_y_', 'body_gyro_z_', 49 | 'total_acc_x_', 'total_acc_y_', 'total_acc_z_'] 50 | 51 | self.label_map = [ 52 | (1, 'WALKING'), 53 | (2, 'WALKING_UPSTAIRS'), 54 | (3, 'WALKING_DOWNSTAIRS'), 55 | (4, 'SITTING'), 56 | (5, 'STANDING'), 57 | (6, 'LAYING'), 58 | ] 59 | 60 | self.drop_activities = [] 61 | 62 | # All ID used for training [ 1, 3, 5, 6, 7, 8, 11, 14, 15, 16, 17, 19, 21, 22, 23, 25, 26, 27, 28, 29, 30] 63 | # All ID used for Test [ 2, 4, 9, 10, 12, 13, 18, 20, 24] 64 | # this is given train test split 65 | self.train_keys = [ 1, 3, 5, 7, 8, 14, 15, 16, 17, 21, 22, 23, 26, 27, 28, 29, 6, 11, 19, 25, 30] 66 | self.vali_keys = [ ] 67 | self.test_keys = [ 2, 4, 9, 10, 12, 13, 18, 20, 24] 68 | 69 | self.exp_mode = args.exp_mode 70 | self.down_sample:bool \ 71 | = args.down_sample 72 | self.split_tag = "sub" 73 | 74 | self.all_keys = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30] 75 | all_keys = np.array_split(self.all_keys,5) 76 | self.LOCV_keys = [list(elem) for elem in all_keys] 77 | self.sub_ids_of_each_sub = {} 78 | 79 | self.file_encoding = {} 80 | 81 | 82 | self.labelToId = {int(x[0]): i for i, x in enumerate(self.label_map)} 83 | self.all_labels = list(range(len(self.label_map))) 84 | 85 | self.drop_activities = [self.labelToId[i] for i in self.drop_activities] 86 | self.no_drop_activites = [item for item in self.all_labels if item not in self.drop_activities] 87 | 88 | 89 | self.leave_one_out = True 90 | self.full_None_Overlapping = False 91 | self.Semi_None_Overlapping = True 92 | 93 | super(UCI_HAR_DATA, self).__init__(args) 94 | 95 | 96 | 97 | 98 | 99 | def load_all_the_data(self, root_path): 100 | 101 | print(" ----------------------- load all the data -------------------") 102 | # ==================== Load the sensor values ==================== 103 | train_vali_path = os.path.join(root_path, "train/Inertial Signals/") 104 | test_path = os.path.join(root_path, "test/Inertial Signals/") 105 | 106 | train_vali_dict = {} 107 | test_dict = {} 108 | 109 | file_list = os.listdir(train_vali_path) 110 | for file in file_list: 111 | 112 | train_vali = pd.read_csv(train_vali_path + file,header=None, delim_whitespace=True) 113 | test = pd.read_csv(test_path+file[:-9]+"test.txt",header=None, delim_whitespace=True) 114 | 115 | train_vali_dict[file[:-9]] = train_vali 116 | test_dict[file[:-9]] = test 117 | 118 | 119 | # =================== Define the sub id and the label for each segments FOR TRAIN VALI ================ 120 | train_vali = pd.DataFrame(np.stack([train_vali_dict[col].values.reshape(-1) for col in self.col_names], axis=1), columns = self.col_names) 121 | 122 | train_vali_subjects = pd.read_csv(os.path.join(root_path,"train/subject_train.txt"), header=None) 123 | train_vali_subjects.columns = ["subjects"] 124 | 125 | train_vali_label = pd.read_csv(os.path.join(root_path,"train/y_train.txt"),header=None) 126 | train_vali_label.columns = ["labels"] 127 | 128 | index = [] 129 | labels = [] 130 | sub_list = [] 131 | 132 | assert train_vali_dict["body_acc_x_"].shape[0] == train_vali_subjects.shape[0] 133 | 134 | # repeat the id and the label for each segs 128 tims 135 | for i in range(train_vali_dict["body_acc_x_"].shape[0]): 136 | sub = train_vali_subjects.loc[i,"subjects"] 137 | sub_id = "{}_{}".format(sub,i) 138 | 139 | ac_id = train_vali_label.loc[i,"labels"] 140 | if sub not in self.sub_ids_of_each_sub.keys(): 141 | self.sub_ids_of_each_sub[sub] = [] 142 | self.sub_ids_of_each_sub[sub].append(sub_id) 143 | 144 | index.extend(128*[sub_id]) 145 | labels.extend(128*[ac_id]) 146 | sub_list.extend(128*[sub]) 147 | 148 | train_vali["sub_id"] = index 149 | train_vali["sub"] = sub_list 150 | train_vali["activity_id"] = labels 151 | 152 | # =================== Define the sub id and the label for each segments FOR TEST ================ 153 | test = pd.DataFrame(np.stack([test_dict[col].values.reshape(-1) for col in self.col_names], axis=1), columns = self.col_names) 154 | 155 | test_subjects = pd.read_csv(os.path.join(root_path,"test/subject_test.txt"), header=None) 156 | test_subjects.columns = ["subjects"] 157 | 158 | test_label = pd.read_csv(os.path.join(root_path,"test/y_test.txt"),header=None) 159 | test_label.columns = ["labels"] 160 | 161 | index = [] 162 | labels = [] 163 | sub_list = [] 164 | assert test_dict["body_acc_x_"].shape[0] == test_subjects.shape[0] 165 | 166 | for i in range(test_dict["body_acc_x_"].shape[0]): 167 | sub = test_subjects.loc[i,"subjects"] 168 | sub_id = "{}_{}".format(sub,i) 169 | 170 | ac_id = test_label.loc[i,"labels"] 171 | 172 | if sub not in self.sub_ids_of_each_sub.keys(): 173 | self.sub_ids_of_each_sub[sub] = [] 174 | self.sub_ids_of_each_sub[sub].append(sub_id) 175 | 176 | index.extend(128*[sub_id]) 177 | labels.extend(128*[ac_id]) 178 | sub_list.extend(128*[sub]) 179 | 180 | test["sub_id"] = index 181 | test["sub"] = sub_list 182 | test["activity_id"] = labels 183 | 184 | 185 | 186 | # The split may be different as the default setting, so we concat all segs together 187 | df_all = pd.concat([train_vali,test]) 188 | 189 | 190 | df_dict = {} 191 | for i in df_all.groupby("sub_id"): 192 | df_dict[i[0]] = i[1] 193 | df_all = pd.concat(df_dict) 194 | 195 | if self.down_sample: 196 | # Downsampling! form 50hz to 30hz 197 | df_all.reset_index(drop=True,inplace=True) 198 | index_list = list( np.arange(0,df_all.shape[0],5/3).astype(int) ) 199 | df_all = df_all.iloc[index_list] 200 | 201 | # ================= Label Transformation =================== 202 | df_all["activity_id"] = df_all["activity_id"].map(self.labelToId) 203 | df_all = df_all.set_index('sub_id') 204 | data_y = df_all.iloc[:,-1] 205 | data_x = df_all.iloc[:,:-1] 206 | 207 | 208 | data_x = data_x.reset_index() 209 | # drop partial sensors for special task. 210 | # data_x = data_x.drop(columns=['total_acc_x_', 'total_acc_y_', 'total_acc_z_']) 211 | 212 | return data_x, data_y -------------------------------------------------------------------------------- /utils/setup.py: -------------------------------------------------------------------------------- 1 | from torch.optim.lr_scheduler import MultiStepLR,ExponentialLR,CosineAnnealingLR,StepLR,LambdaLR 2 | import torch 3 | from configs.base_configs import args 4 | import numpy as np 5 | import torch.optim as optim 6 | import torch.nn as nn 7 | from timm.models.layers import trunc_normal_ 8 | import models 9 | import random 10 | import math 11 | from bisect import bisect_right 12 | import numpy as np 13 | import os 14 | from pprint import pformat 15 | from typing import List, Tuple, Callable 16 | from timm.optim import Lamb 17 | from torch.optim import SGD,AdamW,Adam 18 | from functools import partial 19 | from timm.loss import SoftTargetCrossEntropy, BinaryCrossEntropy 20 | import sys 21 | 22 | def LrSchedulerSet(optimizer,args): 23 | ''' 24 | default: StepLR , gamma is 0.1 , step_size is 40 25 | ''' 26 | if args.lr_scheduler == 'C': 27 | print('==>LrScheduler Set CosineAnnealingLR') 28 | return CosineAnnealingLR( optimizer, T_max=args.n_epochs ) 29 | elif args.lr_scheduler == 'S': 30 | print(f'==>LrScheduler Set StepLR , decay epoch is {args.decay_epochs} , gamma is {args.gamma}') 31 | return StepLR( optimizer, step_size=args.decay_epochs, gamma=args.gamma ) 32 | elif args.lr_scheduler == 'E': 33 | print('==>LrScheduler Set ExponentialLR') 34 | return ExponentialLR( optimizer, gamma=args.gamma ) 35 | elif args.lr_scheduler == 'M': 36 | print(f'==>LrScheduler Set MultiStepLR , scale is {args.milestones} , gamma is {args.gamma} ') 37 | return MultiStepLR( optimizer , args.milestones , gamma=args.gamma ) 38 | elif args.lr_scheduler == 'W': 39 | print(f'==>LrScheduler Set MultiStepLR with Warm up {args.warm_up_epochs}, scale is {args.milestones} , gamma is {args.gamma} ') 40 | # args.linear_gap = args.learning_rate / args.warm_up_epochs 41 | warm_up_with_multistep_lr = lambda epoch: epoch / args.warm_up_epochs \ 42 | if epoch <= args.warm_up_epochs else args.gamma**len([m for m in args.milestones if m <= epoch]) 43 | return LambdaLR(optimizer, lr_lambda=warm_up_with_multistep_lr) 44 | elif args.lr_scheduler == None and args.handcrafted_lr: 45 | print(f'==>LrScheduler is set to manually adjust') 46 | else: 47 | raise NotImplementedError('Not Implement this Lr_Scheduler!') 48 | 49 | def adjust_lr(optimizer, epoch, args, step=0, all_iters_per_epoch=0): 50 | '''this version is MultiStepLR with warm up''' 51 | cur_lr = 0. 52 | if epoch < args.warm_up_epochs: 53 | cur_lr = args.learning_rate * float(1 + step + epoch*all_iters_per_epoch)/(args.warm_up_epochs *all_iters_per_epoch) 54 | else: 55 | epoch = epoch - args.warm_up_epochs 56 | # multistep: 57 | cur_lr = args.learning_rate * 0.1 ** bisect_right(args.milestones, epoch) 58 | 59 | for param_group in optimizer.param_groups: 60 | param_group['lr'] = cur_lr 61 | 62 | return cur_lr 63 | 64 | 65 | 66 | def set_seed(seed): 67 | # fix seed 68 | torch.manual_seed(seed) 69 | torch.cuda.manual_seed_all(seed) 70 | np.random.seed(seed) 71 | random.seed(seed) 72 | np.random.seed(seed) 73 | torch.backends.cudnn.deterministic = True 74 | 75 | def GetModel(): 76 | try: 77 | model = getattr(models, args.model)(args.dataset).cuda() 78 | except Exception as e: 79 | raise NotImplementedError("{} is not implemented".format(args.model)) 80 | return model 81 | 82 | 83 | def GetMOS(opt_type='sgd'): 84 | ''' 85 | M: Model 86 | O: Optimier 87 | S: Scheduler 88 | ''' 89 | # print(args.model) 90 | try: 91 | model = getattr(models, args.model)(args.dataset).cuda() 92 | except Exception as e: 93 | raise NotImplementedError("{} is not implemented".format(args.model)) 94 | 95 | if opt_type == 'sgd': 96 | optimizer = optim.SGD( params = model.parameters(), 97 | lr = args.learning_rate, 98 | momentum = args.momentum , 99 | weight_decay = args.weight_decay, 100 | # nesterov = True 101 | ) 102 | elif opt_type == 'adam': 103 | optimizer = optim.Adam( params = model.parameters() , 104 | lr = args.learning_rate , 105 | # betas = [0.9,0.999] , 106 | weight_decay = args.weight_decay 107 | ) 108 | elif opt_type == 'adamw': 109 | optimizer = optim.AdamW( params = model.parameters() , 110 | lr = args.learning_rate , 111 | # betas = [0.9,0.999] , 112 | weight_decay = args.weight_decay 113 | ) 114 | else: 115 | raise NotImplementedError 116 | scheduler = LrSchedulerSet( optimizer , args ) 117 | return model, optimizer, scheduler 118 | 119 | 120 | def create_model_opt() : 121 | model: torch.nn.Module = getattr(models, args.model)(args.dataset).cuda() 122 | model_para = f'{sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e6:.1f}M' 123 | print(f'[model={args.model}] [#para={model_para}] {model}\n') 124 | model.train() 125 | opt_cls = { 126 | 'sgd': SGD, 127 | 'adam': Adam, 'adamw': AdamW, 128 | 'lamb': partial(Lamb, max_grad_norm=1e7, always_adapt=True, bias_correction=False), 129 | } 130 | param_groups: List[dict] = get_param_groups(model, nowd_keys={'cls_token', 'pos_embed', 'mask_token', 'gamma'}, lr_scale=args.lr_scale) 131 | optimizer = opt_cls[args.opt_type](param_groups, lr=args.learning_rate, weight_decay=args.weight_decay) if args.opt_type !='sgd' else opt_cls[args.opt_type](param_groups, lr=args.learning_rate, weight_decay=args.weight_decay,momentum=args.momentum) 132 | print(f'[optimizer={type(optimizer)}]') 133 | 134 | if 'lamb' in args.opt_type: 135 | # label smoothing is solved in AdaptiveMixup with `label_smoothing`, so here smoothing=0 136 | criterion = BinaryCrossEntropy(smoothing=0, target_threshold=None) 137 | else: 138 | criterion = nn.CrossEntropyLoss() 139 | print(f'[loss_fn] {criterion}') 140 | return model, optimizer, criterion 141 | 142 | def lr_wd_annealing(optimizer, peak_lr, wd, cur_it, wp_it, max_it): 143 | wp_it = round(wp_it) 144 | if cur_it < wp_it: 145 | cur_lr = 0.005 * peak_lr + 0.995 * peak_lr * cur_it / wp_it 146 | else: 147 | ratio = (cur_it - wp_it) / (max_it - 1 - wp_it) 148 | cur_lr = 0.001 * peak_lr + 0.999 * peak_lr * (0.5 + 0.5 * math.cos(math.pi * ratio)) 149 | 150 | min_lr, max_lr = cur_lr, cur_lr 151 | min_wd, max_wd = wd, wd 152 | for param_group in optimizer.param_groups: 153 | scaled_lr = param_group['lr'] = cur_lr * param_group.get('lr_scale', 1) # 'lr_scale' could be assigned 154 | min_lr, max_lr = min(min_lr, scaled_lr), max(max_lr, scaled_lr) 155 | scaled_wd = param_group['weight_decay'] = wd * param_group.get('weight_decay_scale', 1) # 'weight_decay_scale' could be assigned 156 | min_wd, max_wd = min(min_wd, scaled_wd), max(max_wd, scaled_wd) 157 | return min_lr, max_lr, min_wd, max_wd 158 | 159 | def get_param_groups(model, nowd_keys=(), lr_scale=0.0): 160 | using_lr_scale = hasattr(model, 'get_layer_id_and_scale_exp') and 0.0 < lr_scale < 1.0 161 | print(f'[get_ft_param_groups][lr decay] using_lr_scale={using_lr_scale}, ft_lr_scale={lr_scale}') 162 | para_groups, para_groups_dbg = {}, {} 163 | 164 | for name, para in model.named_parameters(): 165 | if not para.requires_grad: 166 | continue # frozen weights 167 | if len(para.shape) == 1 or name.endswith('.bias') or any(k in name for k in nowd_keys): 168 | wd_scale, group_name = 0., 'no_decay' 169 | else: 170 | wd_scale, group_name = 1., 'decay' 171 | 172 | if using_lr_scale: 173 | layer_id, scale_exp = model.get_layer_id_and_scale_exp(name) 174 | group_name = f'layer{layer_id}_' + group_name 175 | this_lr_scale = lr_scale ** scale_exp 176 | dbg = f'[layer {layer_id}][sc = {lr_scale} ** {scale_exp}]' 177 | else: 178 | this_lr_scale = 1 179 | dbg = f'[no scale]' 180 | 181 | if group_name not in para_groups: 182 | para_groups[group_name] = {'params': [], 'weight_decay_scale': wd_scale, 'lr_scale': this_lr_scale} 183 | para_groups_dbg[group_name] = {'params': [], 'weight_decay_scale': wd_scale, 'lr_scale': dbg} 184 | para_groups[group_name]['params'].append(para) 185 | para_groups_dbg[group_name]['params'].append(name) 186 | 187 | for g in para_groups_dbg.values(): 188 | g['params'] = pformat(', '.join(g['params']), width=200) 189 | 190 | print(f'[get_ft_param_groups] param groups = \n{pformat(para_groups_dbg, indent=2, width=250)}\n') 191 | return list(para_groups.values()) 192 | 193 | 194 | def load_checkpoint(args,resume_from, model, optimizer): 195 | try: 196 | if args.ablationMode != '': 197 | resume_from = resume_from+ f'{args.ablationMode}Mode_{args.pre_epochs}Epochs_{args.dataset}_{args.Part}Part_{args.num_of_cv}NCV_{args.model}_{args.mask}_pretrained.pth' 198 | else: 199 | resume_from = resume_from+ f'{args.dataset}_{args.Part}Part_{args.num_of_cv}NCV_{args.model}_{args.mask}_pretrained.pth' 200 | except AttributeError: 201 | print('some variable is not exist') 202 | 203 | if os.path.exists(resume_from): 204 | print(f'==> ckpt `is {resume_from}`!') 205 | # return 0, '[no performance_desc]' 206 | print(f'[try to resume from file `{resume_from}`]') 207 | checkpoint = torch.load(resume_from, map_location='cpu') 208 | assert checkpoint.get('is_pretrain', False) == False, 'Please do not use `*_still_pretraining.pth`, which is ONLY for resuming the pretraining. Use `*_pretrained.pth` or `*_finetuned*.pth` instead.' 209 | 210 | ep_start, performance_desc = checkpoint.get('epoch', -1) + 1, checkpoint.get('performance_desc', '[no performance_desc]') 211 | missing, unexpected = model.load_state_dict(checkpoint.get('module', checkpoint), strict=False) 212 | print(f'[load_checkpoint] missing_keys={missing}') 213 | print(f'[load_checkpoint] unexpected_keys={unexpected}') 214 | print(f'[load_checkpoint] ep_start={ep_start}, performance_desc={performance_desc}') 215 | 216 | if 'optimizer' in checkpoint: 217 | optimizer.load_state_dict(checkpoint['optimizer']) 218 | else: 219 | print(f'==> ckpt `{resume_from}` not exist') 220 | if args.model in ['BaseCNN','BaseCNNSup','DeepConvLstm','Transformer_HAR','DeepConvLstmAttn','SA_HAR']: 221 | pass 222 | else: 223 | sys.exit(1) 224 | -------------------------------------------------------------------------------- /pretrain/main.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: 3 | Date: 2023-06-04 12:00:53 4 | LastEditTime: 2024-08-13 16:08:29 5 | FilePath: /chengdongzhou/action/MaskCAE/pretrain/main.py 6 | ''' 7 | import os 8 | 9 | os.environ["CUDA_VISIBLE_DEVICES"] = '1' 10 | 11 | 12 | import datetime 13 | import math 14 | import sys 15 | import time 16 | from functools import partial 17 | from typing import List 18 | 19 | import torch 20 | from torch.nn.parallel import DistributedDataParallel 21 | from torch.utils.data import DataLoader 22 | 23 | import dist 24 | import encoder 25 | from decoder import LightDecoder 26 | from models import build_sparse_encoder 27 | from sampler import DistInfiniteBatchSampler, worker_init_fn 28 | from MaskCAE import MaskCAE 29 | from utils import arg_util, misc, lamb 30 | from utils.lr_control import lr_wd_annealing, get_param_groups 31 | import numpy as np 32 | import random 33 | 34 | 35 | def get_data_size(data_name, is_sup = False ): 36 | Model_Seen_SSL_F_Size = { 37 | 'ucihar': ( 32, 9 ) , 38 | 'motion': ( 32, 12 ) , 39 | 'uschad': ( 32, 6 ) , 40 | } 41 | Model_Seen_Sup_F_Size = { 42 | 'ucihar': ( 128, 9 ) , 43 | 'motion': ( 128, 12 ) , 44 | 'uschad': ( 32, 6 ) , 45 | } 46 | if is_sup: 47 | size_dict = Model_Seen_Sup_F_Size 48 | else: 49 | size_dict = Model_Seen_SSL_F_Size 50 | if data_name in size_dict: 51 | pass 52 | else: 53 | raise Exception( 'please input correct data name') 54 | return size_dict[data_name] 55 | 56 | 57 | def main_pt(args:arg_util.Args,dataset_train): 58 | 59 | 60 | # build data 61 | print(f'[build data for pre-training] ...\n') 62 | data_loader_train = DataLoader( 63 | dataset = dataset_train, 64 | num_workers = args.dataloader_workers, 65 | pin_memory = True, 66 | batch_sampler = DistInfiniteBatchSampler( 67 | dataset_len = len(dataset_train), 68 | glb_batch_size = args.glb_batch_size, 69 | shuffle = True, 70 | filling = True, 71 | rank = dist.get_rank(), 72 | world_size = dist.get_world_size(), 73 | ), 74 | worker_init_fn = worker_init_fn 75 | ) 76 | itrt_train, iters_train = iter(data_loader_train), len(data_loader_train) 77 | print(f'[dataloader] gbs={args.glb_batch_size}, lbs={args.batch_size_per_gpu}, iters_train={iters_train}') 78 | 79 | # build encoder and decoder 80 | print(f'==> is sup? {args.is_sup}') 81 | enc: encoder.SparseEncoder = build_sparse_encoder(args.model,dataset_name =args.dataset_name ,input_size=get_data_size(args.dataset_name,args.is_sup), sbn=args.sbn, drop_path_rate=args.dp, verbose=False,mode=args.ablation_mode) 82 | dec = LightDecoder(enc.downsample_raito, sbn=args.sbn,mode=args.ablation_mode) 83 | model = MaskCAE( 84 | sparse_encoder = enc, 85 | dense_decoder = dec, 86 | mask_ratio = args.mask, 87 | densify_norm = args.densify_norm, 88 | sbn = args.sbn, 89 | mode = args.ablation_mode, 90 | sup_window_length = args.is_sup 91 | ).cuda() 92 | print(f'[PT model] model = {model}\n') 93 | 94 | 95 | # build optimizer and lr_scheduler 96 | param_groups: List[dict] = get_param_groups(model, nowd_keys={'cls_token', 'pos_embed', 'mask_token', 'gamma'}) 97 | opt_clz = { 98 | 'sgd': partial(torch.optim.SGD, momentum=0.9, nesterov=True ), 99 | 'adamw': partial(torch.optim.AdamW, betas=(0.9, args.ada) ), 100 | 'lamb': partial(lamb.TheSameAsTimmLAMB, betas=(0.9, args.ada), max_grad_norm=5.0 ), 101 | }[args.opt] 102 | optimizer = opt_clz(params=param_groups, lr=args.lr, weight_decay=0.0) 103 | print(f'[optimizer] optimizer({opt_clz}) ={optimizer}\n') 104 | 105 | # try to resume 106 | ep_start, performance_desc = misc.load_checkpoint(args.resume_from, model, optimizer) 107 | if ep_start >= args.ep: # load from a complete checkpoint file 108 | print(f' [*] [PT already done] Min/Last Recon Loss: {performance_desc}') 109 | else: # perform pre-training 110 | tb_lg = misc.TensorboardLogger(args.tb_lg_dir, is_master=dist.is_master(), prefix='pt') 111 | min_loss = 1e9 112 | print(f'[PT start] from ep{ep_start}') 113 | 114 | pt_start_time = time.time() 115 | for ep in range(ep_start, args.ep): 116 | ep_start_time = time.time() 117 | tb_lg.set_step(ep * iters_train) 118 | if hasattr(itrt_train, 'set_epoch'): 119 | itrt_train.set_epoch(ep) 120 | 121 | stats = pre_train_one_ep(ep, args, tb_lg, itrt_train, iters_train, model, optimizer) 122 | last_loss = stats['last_loss'] 123 | min_loss = min(min_loss, last_loss) 124 | performance_desc = f'{min_loss:.4f} {last_loss:.4f}' 125 | if args.ablation_mode != '': 126 | misc.save_checkpoint(f'{args.ablation_mode}Mode_{args.ep}Epochs_{args.dataset_name}_{args.Part}Part_{args.num_of_cv}NCV_{args.model}_{args.mask}_still_pretraining.pth', args, ep, performance_desc, model.state_dict(with_config=True), optimizer.state_dict()) 127 | misc.save_checkpoint_for_finetune(f'{args.ablation_mode}Mode_{args.ep}Epochs_{args.dataset_name}_{args.Part}Part_{args.num_of_cv}NCV_{args.model}_{args.mask}_pretrained.pth', args, model.sparse_encoder.sp_cnn.state_dict()) 128 | else: 129 | misc.save_checkpoint(f'{args.dataset_name}_{args.Part}Part_{args.num_of_cv}NCV_{args.model}_{args.mask}_still_pretraining.pth', args, ep, performance_desc, model.state_dict(with_config=True), optimizer.state_dict()) 130 | misc.save_checkpoint_for_finetune(f'{args.dataset_name}_{args.Part}Part_{args.num_of_cv}NCV_{args.model}_{args.mask}_pretrained.pth', args, model.sparse_encoder.sp_cnn.state_dict()) 131 | 132 | ep_cost = round(time.time() - ep_start_time, 2) + 1 # +1s: approximate the following logging cost 133 | remain_secs = (args.ep-1 - ep) * ep_cost 134 | remain_time = datetime.timedelta(seconds=round(remain_secs)) 135 | finish_time = time.strftime("%m-%d %H:%M", time.localtime(time.time() + remain_secs)) 136 | print(f' [*] [ep{ep}/{args.ep}] Min/Last Recon Loss: {performance_desc}, Cost: {ep_cost}s, Remain: {remain_time}, Finish @ {finish_time}') 137 | 138 | args.cur_ep = f'{ep + 1}/{args.ep}' 139 | args.remain_time, args.finish_time = str(remain_time), str(finish_time) 140 | args.last_loss = last_loss 141 | args.log_epoch() 142 | 143 | tb_lg.update(min_loss=min_loss, head='train', step=ep) 144 | tb_lg.update(rest_hours=round(remain_secs/60/60, 2), head='z_burnout', step=ep) 145 | tb_lg.flush() 146 | 147 | # finish pre-training 148 | tb_lg.update(min_loss=min_loss, head='result', step=ep_start) 149 | tb_lg.update(min_loss=min_loss, head='result', step=args.ep) 150 | tb_lg.flush() 151 | print(f'final args:\n{str(args)}') 152 | print('\n\n') 153 | print(f' [*] [PT finished] Min/Last Recon Loss: {performance_desc}, Total Cost: {(time.time() - pt_start_time) / 60 / 60:.1f}h\n') 154 | print('\n\n') 155 | tb_lg.close() 156 | time.sleep(10) 157 | 158 | args.remain_time, args.finish_time = '-', time.strftime("%m-%d %H:%M", time.localtime(time.time())) 159 | args.log_epoch() 160 | 161 | 162 | def pre_train_one_ep(ep, args: arg_util.Args, tb_lg: misc.TensorboardLogger, itrt_train, iters_train, model: DistributedDataParallel, optimizer): 163 | model.train() 164 | me = misc.MetricLogger(delimiter=' ') 165 | me.add_meter('max_lr', misc.SmoothedValue(window_size=1, fmt='{value:.5f}')) 166 | header = f'[PT] Epoch {ep}:' 167 | 168 | optimizer.zero_grad() 169 | early_clipping = args.clip > 0 and not hasattr(optimizer, 'global_grad_norm') 170 | late_clipping = hasattr(optimizer, 'global_grad_norm') 171 | if early_clipping: 172 | params_req_grad = [p for p in model.parameters() if p.requires_grad] 173 | 174 | for it, (inp, _) in enumerate(me.log_every(iters_train, itrt_train, 3, header)): 175 | # adjust lr and wd 176 | min_lr, max_lr, min_wd, max_wd = lr_wd_annealing(optimizer, args.lr, args.wd, args.wde, it + ep * iters_train, args.wp_ep * iters_train, args.ep * iters_train) 177 | 178 | # forward and backward 179 | inp = inp.cuda().float() 180 | # MaskCAE.forward 181 | loss = model(inp, active_b1fs=None, vis=False) 182 | optimizer.zero_grad() 183 | loss.backward() 184 | loss = loss.item() 185 | if not math.isfinite(loss): 186 | print(f'[rk{dist.get_rank():02d}] Loss is {loss}, stopping training!', force=True, flush=True) 187 | sys.exit(-1) 188 | 189 | # optimize 190 | grad_norm = None 191 | if early_clipping: grad_norm = torch.nn.utils.clip_grad_norm_(params_req_grad, args.clip).item() 192 | optimizer.step() 193 | if late_clipping: grad_norm = optimizer.global_grad_norm 194 | torch.cuda.synchronize() 195 | 196 | # log 197 | me.update(last_loss=loss) 198 | me.update(max_lr=max_lr) 199 | tb_lg.update(loss=me.meters['last_loss'].global_avg, head='train_loss') 200 | tb_lg.update(sche_lr=max_lr, head='train_hp/lr_max') 201 | tb_lg.update(sche_lr=min_lr, head='train_hp/lr_min') 202 | tb_lg.update(sche_wd=max_wd, head='train_hp/wd_max') 203 | tb_lg.update(sche_wd=min_wd, head='train_hp/wd_min') 204 | 205 | if grad_norm is not None: 206 | me.update(orig_norm=grad_norm) 207 | tb_lg.update(orig_norm=grad_norm, head='train_hp') 208 | tb_lg.set_step() 209 | 210 | me.synchronize_between_processes() 211 | return {k: meter.global_avg for k, meter in me.meters.items()} 212 | 213 | 214 | def set_seed(seed): 215 | # fix seed 216 | torch.manual_seed(seed) 217 | torch.cuda.manual_seed_all(seed) 218 | np.random.seed(seed) 219 | random.seed(seed) 220 | np.random.seed(seed) 221 | torch.backends.cudnn.deterministic = True 222 | 223 | if __name__ == '__main__': 224 | from create_dataset import GetHarDataset, dotdict 225 | from dataloaders import data_set 226 | import os 227 | args: arg_util.Args = arg_util.init_dist_and_get_args() 228 | os.environ["CUDA_VISIBLE_DEVICES"] = str(args.device) 229 | print(f'initial args:\n{str(args)}') 230 | args.log_epoch() 231 | set_seed(0) 232 | 233 | 234 | if args.DATASET_PRE == 'self' or args.DATASET_PRE == 'Sup': 235 | data_args = dotdict() 236 | if args.DATASET_PRE == 'Sup': 237 | data_args.yaml_path = 'configs/yaml/data_sup.yaml' 238 | else: 239 | data_args.yaml_path = 'configs/yaml/data.yaml' 240 | data_args, dataset\ 241 | = GetHarDataset(dataset_name=args.dataset_name,data_args=data_args) 242 | for currenct_cv in range(dataset.num_of_cv): 243 | print("================ {} Mode ====================".format(dataset.exp_mode)) 244 | print("================ {} CV ====================".format(dataset.num_of_cv)) 245 | dataset.update_train_val_test_keys() 246 | args.Part = dataset.index_of_cv 247 | args.num_of_cv = dataset.num_of_cv 248 | train_data = data_set(data_args,dataset,"train") 249 | main_pt(args,train_data) 250 | 251 | -------------------------------------------------------------------------------- /MaskCAE/pretrain/utils/misc.py: -------------------------------------------------------------------------------- 1 | 2 | import datetime 3 | import functools 4 | import os 5 | import subprocess 6 | import sys 7 | import time 8 | from collections import defaultdict, deque 9 | from typing import Iterator 10 | 11 | import numpy as np 12 | import pytz 13 | import torch 14 | from torch.utils.tensorboard import SummaryWriter 15 | 16 | import dist 17 | 18 | os_system = functools.partial(subprocess.call, shell=True) 19 | os_system_get_stdout = lambda cmd: subprocess.run(cmd, shell=True, stdout=subprocess.PIPE).stdout.decode('utf-8') 20 | def os_system_get_stdout_stderr(cmd): 21 | sp = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 22 | return sp.stdout.decode('utf-8'), sp.stderr.decode('utf-8') 23 | 24 | 25 | def is_pow2n(x): 26 | return x > 0 and (x & (x - 1) == 0) 27 | 28 | 29 | def time_str(for_dirname=False): 30 | return datetime.datetime.now(tz=pytz.timezone('Asia/Shanghai')).strftime('%m-%d_%H-%M-%S' if for_dirname else '[%m-%d %H:%M:%S]') 31 | 32 | 33 | def init_distributed_environ(exp_dir): 34 | dist.initialize() 35 | dist.barrier() 36 | 37 | import torch.backends.cudnn as cudnn 38 | cudnn.benchmark = True 39 | cudnn.deterministic = False 40 | 41 | _set_print_only_on_master_proc(is_master=dist.is_local_master()) 42 | if dist.is_local_master() and len(exp_dir): 43 | sys.stdout, sys.stderr = _SyncPrintToFile(exp_dir, stdout=True), _SyncPrintToFile(exp_dir, stdout=False) 44 | 45 | 46 | def _set_print_only_on_master_proc(is_master): 47 | import builtins as __builtin__ 48 | 49 | builtin_print = __builtin__.print 50 | 51 | def prt(msg, *args, **kwargs): 52 | force = kwargs.pop('force', False) 53 | clean = kwargs.pop('clean', False) 54 | deeper = kwargs.pop('deeper', False) 55 | if is_master or force: 56 | if not clean: 57 | f_back = sys._getframe().f_back 58 | if deeper and f_back.f_back is not None: 59 | f_back = f_back.f_back 60 | file_desc = f'{f_back.f_code.co_filename:24s}'[-24:] 61 | msg = f'{time_str()} ({file_desc}, line{f_back.f_lineno:-4d})=> {msg}' 62 | builtin_print(msg, *args, **kwargs) 63 | 64 | __builtin__.print = prt 65 | 66 | 67 | class _SyncPrintToFile(object): 68 | def __init__(self, exp_dir, stdout=True): 69 | self.terminal = sys.stdout if stdout else sys.stderr 70 | fname = os.path.join(exp_dir, 'stdout_backup.txt' if stdout else 'stderr_backup.txt') 71 | self.log = open(fname, 'w') 72 | self.log.flush() 73 | 74 | def write(self, message): 75 | self.terminal.write(message) 76 | self.log.write(message) 77 | self.log.flush() 78 | 79 | def flush(self): 80 | self.terminal.flush() 81 | self.log.flush() 82 | 83 | 84 | class TensorboardLogger(object): 85 | def __init__(self, log_dir, is_master, prefix='pt'): 86 | self.is_master = is_master 87 | self.writer = SummaryWriter(log_dir=log_dir) if self.is_master else None 88 | self.step = 0 89 | self.prefix = prefix 90 | self.log_freq = 300 91 | 92 | def set_step(self, step=None): 93 | if step is not None: 94 | self.step = step 95 | else: 96 | self.step += 1 97 | 98 | def get_loggable(self, step=None): 99 | if step is None: # iter wise 100 | step = self.step 101 | loggable = step % self.log_freq == 0 102 | else: # epoch wise 103 | loggable = True 104 | return step, (loggable and self.is_master) 105 | 106 | def update(self, head='scalar', step=None, **kwargs): 107 | step, loggable = self.get_loggable(step) 108 | if loggable: 109 | head = f'{self.prefix}_{head}' 110 | for k, v in kwargs.items(): 111 | if v is None: 112 | continue 113 | if isinstance(v, torch.Tensor): 114 | v = v.item() 115 | assert isinstance(v, (float, int)) 116 | self.writer.add_scalar(head + "/" + k, v, step) 117 | 118 | def log_distribution(self, tag, values, step=None): 119 | step, loggable = self.get_loggable(step) 120 | if loggable: 121 | if not isinstance(values, torch.Tensor): 122 | values = torch.tensor(values) 123 | self.writer.add_histogram(tag=tag, values=values, global_step=step) 124 | 125 | def log_image(self, tag, img, step=None, dataformats='NCHW'): 126 | step, loggable = self.get_loggable(step) 127 | if loggable: 128 | # img = img.cpu().numpy() 129 | self.writer.add_image(tag, img, step, dataformats=dataformats) 130 | 131 | def flush(self): 132 | if self.is_master: self.writer.flush() 133 | 134 | def close(self): 135 | if self.is_master: self.writer.close() 136 | 137 | 138 | def save_checkpoint(save_to, args, epoch, performance_desc, model_without_ddp_state, optimizer_state): 139 | checkpoint_path = os.path.join(args.exp_dir, save_to) 140 | if dist.is_local_master(): 141 | to_save = { 142 | 'args': str(args), 143 | 'arch': args.model, 144 | 'epoch': epoch, 145 | 'performance_desc': performance_desc, 146 | 'module': model_without_ddp_state, 147 | 'optimizer': optimizer_state, 148 | 'is_pretrain': True, 149 | } 150 | torch.save(to_save, checkpoint_path) 151 | 152 | 153 | def save_checkpoint_for_finetune(save_to, args, sp_cnn_state): 154 | checkpoint_path = os.path.join(args.exp_dir, save_to) 155 | if dist.is_local_master(): 156 | to_save = { 157 | 'arch': args.model, 158 | 'module': sp_cnn_state, 159 | 'is_pretrain': False, 160 | } 161 | torch.save(to_save, checkpoint_path) 162 | 163 | 164 | def load_checkpoint(resume_from, model_without_ddp, optimizer): 165 | if len(resume_from) == 0: 166 | return 0, '[no performance_desc]' 167 | print(f'[try to resume from file `{resume_from}`]') 168 | checkpoint = torch.load(resume_from, map_location='cpu') 169 | 170 | ep_start, performance_desc = checkpoint.get('epoch', -1) + 1, checkpoint.get('performance_desc', '[no performance_desc]') 171 | missing, unexpected = model_without_ddp.load_state_dict(checkpoint['module'], strict=False) 172 | print(f'[load_checkpoint] missing_keys={missing}') 173 | print(f'[load_checkpoint] unexpected_keys={unexpected}') 174 | print(f'[load_checkpoint] ep_start={ep_start}, performance_desc={performance_desc}') 175 | 176 | if 'optimizer' in checkpoint: 177 | optimizer.load_state_dict(checkpoint['optimizer']) 178 | return ep_start, performance_desc 179 | 180 | 181 | class SmoothedValue(object): 182 | """Track a series of values and provide access to smoothed values over a 183 | window or the global series average. 184 | """ 185 | 186 | def __init__(self, window_size=20, fmt=None): 187 | if fmt is None: 188 | fmt = "{median:.4f} ({global_avg:.4f})" 189 | self.deque = deque(maxlen=window_size) 190 | self.total = 0.0 191 | self.count = 0 192 | self.fmt = fmt 193 | 194 | def update(self, value, n=1): 195 | self.deque.append(value) 196 | self.count += n 197 | self.total += value * n 198 | 199 | def synchronize_between_processes(self): 200 | """ 201 | Warning: does not synchronize the deque! 202 | """ 203 | t = torch.tensor([self.count, self.total], dtype=torch.float64, device='cuda') 204 | dist.barrier() 205 | dist.allreduce(t) 206 | t = t.tolist() 207 | self.count = int(t[0]) 208 | self.total = t[1] 209 | 210 | @property 211 | def median(self): 212 | d = torch.tensor(list(self.deque)) 213 | return d.median().item() 214 | 215 | @property 216 | def avg(self): 217 | d = torch.tensor(list(self.deque), dtype=torch.float32) 218 | return d.mean().item() 219 | 220 | @property 221 | def global_avg(self): 222 | return self.total / self.count 223 | 224 | @property 225 | def max(self): 226 | return max(self.deque) 227 | 228 | @property 229 | def value(self): 230 | return self.deque[-1] 231 | 232 | def __str__(self): 233 | return self.fmt.format( 234 | median=self.median, 235 | avg=self.avg, 236 | global_avg=self.global_avg, 237 | max=self.max, 238 | value=self.value) 239 | 240 | 241 | class MetricLogger(object): 242 | def __init__(self, delimiter="\t"): 243 | self.meters = defaultdict(SmoothedValue) 244 | self.delimiter = delimiter 245 | 246 | def update(self, **kwargs): 247 | for k, v in kwargs.items(): 248 | if v is None: 249 | continue 250 | if isinstance(v, torch.Tensor): 251 | v = v.item() 252 | assert isinstance(v, (float, int)) 253 | self.meters[k].update(v) 254 | 255 | def __getattr__(self, attr): 256 | if attr in self.meters: 257 | return self.meters[attr] 258 | if attr in self.__dict__: 259 | return self.__dict__[attr] 260 | raise AttributeError("'{}' object has no attribute '{}'".format( 261 | type(self).__name__, attr)) 262 | 263 | def __str__(self): 264 | loss_str = [] 265 | for name, meter in self.meters.items(): 266 | loss_str.append( 267 | "{}: {}".format(name, str(meter)) 268 | ) 269 | return self.delimiter.join(loss_str) 270 | 271 | def synchronize_between_processes(self): 272 | for meter in self.meters.values(): 273 | meter.synchronize_between_processes() 274 | 275 | def add_meter(self, name, meter): 276 | self.meters[name] = meter 277 | 278 | def log_every(self, max_iters, itrt, print_freq, header=None): 279 | print_iters = set(np.linspace(0, max_iters - 1, print_freq, dtype=int).tolist()) 280 | if not header: 281 | header = '' 282 | start_time = time.time() 283 | end = time.time() 284 | self.iter_time = SmoothedValue(fmt='{avg:.4f}') 285 | self.data_time = SmoothedValue(fmt='{avg:.4f}') 286 | space_fmt = ':' + str(len(str(max_iters))) + 'd' 287 | log_msg = [ 288 | header, 289 | '[{0' + space_fmt + '}/{1}]', 290 | 'eta: {eta}', 291 | '{meters}', 292 | 'iter: {time}s', 293 | 'data: {data}s' 294 | ] 295 | log_msg = self.delimiter.join(log_msg) 296 | 297 | if isinstance(itrt, Iterator) and not hasattr(itrt, 'preload') and not hasattr(itrt, 'set_epoch'): 298 | for i in range(max_iters): 299 | obj = next(itrt) 300 | self.data_time.update(time.time() - end) 301 | yield obj 302 | self.iter_time.update(time.time() - end) 303 | if i in print_iters: 304 | eta_seconds = self.iter_time.global_avg * (max_iters - i) 305 | eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) 306 | print(log_msg.format( 307 | i, max_iters, eta=eta_string, 308 | meters=str(self), 309 | time=str(self.iter_time), data=str(self.data_time))) 310 | end = time.time() 311 | else: 312 | for i, obj in enumerate(itrt): 313 | self.data_time.update(time.time() - end) 314 | yield obj 315 | self.iter_time.update(time.time() - end) 316 | if i in print_iters: 317 | eta_seconds = self.iter_time.global_avg * (max_iters - i) 318 | eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) 319 | print(log_msg.format( 320 | i, max_iters, eta=eta_string, 321 | meters=str(self), 322 | time=str(self.iter_time), data=str(self.data_time))) 323 | end = time.time() 324 | 325 | total_time = time.time() - start_time 326 | total_time_str = str(datetime.timedelta(seconds=int(total_time))) 327 | print('{} Total time: {} ({:.3f} s / it)'.format( 328 | header, total_time_str, total_time / max_iters)) 329 | -------------------------------------------------------------------------------- /utils/augmentations.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Author: error: git config user.name && git config user.email & please set dead value or install git 3 | Date: 2022-10-12 21:38:44 4 | LastEditors: Please set LastEditors 5 | LastEditTime: 2023-05-30 13:42:15 6 | FilePath: /chengdongzhou/action/tRexHAR/utils/augmentations.py 7 | Description: 这是默认设置,请设置`customMade`, 打开koroFileHeader查看配置 进行设置: https://github.com/OBKoro1/koro1FileHeader/wiki/%E9%85%8D%E7%BD%AE 8 | ''' 9 | import numpy as np 10 | import torch 11 | import scipy 12 | from scipy.interpolate import CubicSpline 13 | import random 14 | from configs.base_configs import args 15 | 16 | def get_aug_data( data , RA = None , FA = None): 17 | if args.RandAug: 18 | aug_data = RA( data.squeeze(1) ).unsqueeze(1) 19 | inputs = torch.cat( [data,aug_data] , dim=0 ) 20 | elif args.FixAug: 21 | aug_data = FA( data.squeeze(1) , args.data_aug, args.data_aug2 ).unsqueeze(1) 22 | inputs = torch.cat( [data,aug_data] , dim=0 ) 23 | else: 24 | if args.data_aug and (not args.data_aug2) : 25 | aug_data = gen_aug( data.squeeze(1) , ssh_type= args.data_aug ).unsqueeze(1) 26 | inputs = torch.cat( [data,aug_data] , dim=0 ) 27 | elif args.data_aug and args.data_aug2 : 28 | aug1 = gen_aug( data.squeeze(1) , ssh_type= args.data_aug ).unsqueeze(1) 29 | aug2 = gen_aug( data.squeeze(1) , ssh_type= args.data_aug2 ).unsqueeze(1) 30 | inputs = torch.cat( [aug1,aug2] , dim=0 ) 31 | else: 32 | '''fully supervised without data augmentation''' 33 | inputs = data 34 | return inputs 35 | 36 | def gen_aug(sample, ssh_type = 'na'): 37 | if ssh_type == 'na': 38 | return sample 39 | elif ssh_type == 'shuffle': 40 | return shuffle(sample) 41 | elif ssh_type == 'jit_scal': 42 | scale_sample = scaling(sample, sigma=2) 43 | return torch.from_numpy(scale_sample) 44 | elif ssh_type == 'perm_jit': 45 | return jitter(permutation(sample, max_segments=10), sigma=0.8) 46 | elif ssh_type == 'resample': 47 | return torch.from_numpy(resample(sample)) 48 | elif ssh_type == 'noise': 49 | return jitter(sample) 50 | elif ssh_type == 'scale': 51 | return torch.from_numpy(scaling(sample)) 52 | elif ssh_type == 'negate': 53 | return negated(sample) 54 | elif ssh_type == 't_flip': 55 | return time_flipped(sample) 56 | elif ssh_type == 'rotation': 57 | if isinstance(multi_rotation(sample), np.ndarray): 58 | return torch.from_numpy(multi_rotation(sample)) 59 | else: 60 | return multi_rotation(sample) 61 | elif ssh_type == 'perm': 62 | return permutation(sample, max_segments=10) 63 | elif ssh_type == 't_warp': 64 | return torch.from_numpy(time_warp(sample)) 65 | elif ssh_type == 'hfc': 66 | fft, fd = generate_high(sample, r=(32,2), high=True) 67 | return fd 68 | elif ssh_type == 'lfc': 69 | fft, fd = generate_high(sample, r=(32,2), high=False) 70 | return fd 71 | elif ssh_type == 'p_shift': 72 | return ifft_phase_shift(sample) 73 | elif ssh_type == 'ap_p': 74 | return ifft_amp_phase_pert(sample) 75 | elif ssh_type == 'ap_f': 76 | return ifft_amp_phase_pert_fully(sample) 77 | else: 78 | print('The task is not available!\n') 79 | 80 | 81 | 82 | 83 | 84 | 85 | def shuffle(x): 86 | sample_ssh = [] 87 | for data in x: 88 | p = np.random.RandomState(seed=21).permutation(data.shape[1]) 89 | data = data[:, p] 90 | sample_ssh.append(data) 91 | return torch.stack(sample_ssh) 92 | 93 | 94 | def jitter(x, sigma=0.8): 95 | # https://arxiv.org/pdf/1706.00527.pdf 96 | return x + np.random.normal(loc=0., scale=sigma, size=x.shape) 97 | 98 | 99 | def scaling(x, sigma=1.1): # apply same distortion to the signals from each sensor 100 | # https://arxiv.org/pdf/1706.00527.pdf 101 | factor = np.random.normal(loc=2., scale=sigma, size=(x.shape[0], x.shape[1])) 102 | ai = [] 103 | for i in range(x.shape[2]): 104 | xi = x[:, :, i] 105 | ai.append(np.multiply(xi, factor[:, :])[:, :, np.newaxis]) 106 | return np.concatenate((ai), axis=2) 107 | 108 | 109 | def negated(X): 110 | return X * -1 111 | 112 | 113 | def time_flipped(X): 114 | inv_idx = torch.arange(X.size(1) - 1, -1, -1).long() 115 | return X[:, inv_idx, :] 116 | 117 | 118 | def permutation(x, max_segments=5, seg_mode="random"): 119 | orig_steps = np.arange(x.shape[1]) 120 | 121 | num_segs = np.random.randint(1, max_segments, size=(x.shape[0])) 122 | ret = np.zeros_like(x) 123 | for i, pat in enumerate(x): 124 | if num_segs[i] > 1: 125 | if seg_mode == "random": 126 | split_points = np.random.choice(x.shape[1] - 2, num_segs[i] - 1, replace=False) 127 | split_points.sort() 128 | splits = np.split(orig_steps, split_points) 129 | else: 130 | splits = np.array_split(orig_steps, num_segs[i]) 131 | np.random.shuffle(splits) 132 | warp = np.concatenate(splits).ravel() 133 | ret[i] = pat[warp, :] 134 | else: 135 | ret[i] = pat 136 | return torch.from_numpy(ret) 137 | 138 | 139 | def resample(x): 140 | from scipy.interpolate import interp1d 141 | orig_steps = np.arange(x.shape[1]) 142 | interp_steps = np.arange(0, orig_steps[-1]+0.001, 1/3) 143 | Interp = interp1d(orig_steps, x, axis=1) 144 | InterpVal = Interp(interp_steps) 145 | start = random.choice(orig_steps) 146 | resample_index = np.arange(start, 3 * x.shape[1], 2)[:x.shape[1]] 147 | return InterpVal[:, resample_index, :] 148 | 149 | 150 | def multi_rotation(x): 151 | n_channel = x.shape[2] 152 | n_rot = n_channel // 3 153 | x_rot = np.array([]) 154 | for i in range(n_rot): 155 | x_rot = np.concatenate((x_rot, rotation(x[:, :, i * 3:i * 3 + 3])), axis=2) if x_rot.size else rotation( 156 | x[:, :, i * 3:i * 3 + 3]) 157 | return x_rot 158 | 159 | def rotation(X): 160 | """ 161 | Applying a random 3D rotation 162 | """ 163 | axes = np.random.uniform(low=-1, high=1, size=(X.shape[0], X.shape[2])) 164 | angles = np.random.uniform(low=-np.pi, high=np.pi, size=(X.shape[0])) 165 | matrices = axis_angle_to_rotation_matrix_3d_vectorized(axes, angles) 166 | return np.matmul(X, matrices) 167 | 168 | def axis_angle_to_rotation_matrix_3d_vectorized(axes, angles): 169 | """ 170 | Get the rotational matrix corresponding to a rotation of (angle) radian around the axes 171 | Reference: the Transforms3d package - transforms3d.axangles.axangle2mat 172 | Formula: http://en.wikipedia.org/wiki/Rotation_matrix#Axis_and_angle 173 | """ 174 | axes = axes / np.linalg.norm(axes, ord=2, axis=1, keepdims=True) 175 | x = axes[:, 0]; y = axes[:, 1]; z = axes[:, 2] 176 | c = np.cos(angles) 177 | s = np.sin(angles) 178 | C = 1 - c 179 | 180 | xs = x*s; ys = y*s; zs = z*s 181 | xC = x*C; yC = y*C; zC = z*C 182 | xyC = x*yC; yzC = y*zC; zxC = z*xC 183 | 184 | m = np.array([ 185 | [ x*xC+c, xyC-zs, zxC+ys ], 186 | [ xyC+zs, y*yC+c, yzC-xs ], 187 | [ zxC-ys, yzC+xs, z*zC+c ]]) 188 | matrix_transposed = np.transpose(m, axes=(2,0,1)) 189 | return matrix_transposed 190 | 191 | def get_cubic_spline_interpolation(x_eval, x_data, y_data): 192 | """ 193 | Get values for the cubic spline interpolation 194 | """ 195 | cubic_spline = CubicSpline(x_data, y_data) 196 | return cubic_spline(x_eval) 197 | 198 | 199 | def time_warp(X, sigma=0.2, num_knots=4): 200 | """ 201 | Stretching and warping the time-series 202 | """ 203 | time_stamps = np.arange(X.shape[1]) 204 | knot_xs = np.arange(0, num_knots + 2, dtype=float) * (X.shape[1] - 1) / (num_knots + 1) 205 | spline_ys = np.random.normal(loc=1.0, scale=sigma, size=(X.shape[0] * X.shape[2], num_knots + 2)) 206 | 207 | spline_values = np.array([get_cubic_spline_interpolation(time_stamps, knot_xs, spline_ys_individual) for spline_ys_individual in spline_ys]) 208 | 209 | cumulative_sum = np.cumsum(spline_values, axis=1) 210 | distorted_time_stamps_all = cumulative_sum / cumulative_sum[:, -1][:, np.newaxis] * (X.shape[1] - 1) 211 | 212 | X_transformed = np.empty(shape=X.shape) 213 | for i, distorted_time_stamps in enumerate(distorted_time_stamps_all): 214 | X_transformed[i // X.shape[2], :, i % X.shape[2]] = np.interp(time_stamps, distorted_time_stamps, X[i // X.shape[2], :, i % X.shape[2]]) 215 | return X_transformed 216 | 217 | 218 | def distance(i, j, imageSize, r): 219 | dis_x = np.sqrt((i - imageSize[0] / 2) ** 2) 220 | dis_y = np.sqrt((j - imageSize[1] / 2) ** 2) 221 | if dis_x < r[0] and dis_y < r[1]: 222 | return 1.0 223 | else: 224 | return 0 225 | 226 | 227 | def mask_radial(img, r): 228 | rows, cols = img.shape 229 | mask = torch.zeros((rows, cols)) 230 | for i in range(rows): 231 | for j in range(cols): 232 | mask[i, j] = distance(i, j, imageSize=(rows, cols), r=r) 233 | return mask 234 | 235 | 236 | def generate_high(sample, r, high=True): 237 | # r: int, radius of the mask 238 | images = torch.unsqueeze(sample, 1) 239 | mask = mask_radial(torch.zeros([images.shape[2], images.shape[3]]), r) 240 | bs, c, h, w = images.shape 241 | x = images.reshape([bs * c, h, w]) 242 | fd = torch.fft.fftshift(torch.fft.fftn(x, dim=(-2, -1))) # shift: low f in the center 243 | mask = mask.unsqueeze(0).repeat([bs * c, 1, 1]) 244 | if high: 245 | fd = fd * (1.-mask) 246 | else: 247 | fd = fd * mask 248 | fft = torch.real(fd) 249 | fd = torch.fft.ifftn(torch.fft.ifftshift(fd), dim=(-2, -1)) 250 | fd = torch.real(fd) 251 | fd = torch.squeeze(fd.reshape([bs, c, h, w])) 252 | return fft, fd 253 | 254 | 255 | def ifft_phase_shift(sample): 256 | images = torch.unsqueeze(sample, 1) 257 | bs, c, h, w = images.shape 258 | x = images.reshape([bs * c, h, w]) 259 | fd = torch.fft.fftshift(torch.fft.fftn(x, dim=(-2, -1))) 260 | 261 | amp = fd.abs() 262 | phase = fd.angle() 263 | 264 | # phase shift 265 | angles = np.repeat(np.expand_dims(np.random.uniform(low=-np.pi, high=np.pi, size=(sample.shape[0], sample.shape[1])), axis=2), sample.shape[2], axis=2) 266 | phase = phase + angles 267 | 268 | cmp = amp * torch.exp(1j * phase) 269 | ifft = torch.squeeze(torch.real(torch.fft.ifftn(torch.fft.ifftshift(cmp), dim=(-2, -1))).reshape([bs, c, h, w])) 270 | 271 | return ifft 272 | 273 | 274 | def ifft_amp_phase_pert(sample): 275 | images = torch.unsqueeze(sample, 1) 276 | bs, c, h, w = images.shape 277 | x = images.reshape([bs * c, h, w]) 278 | fd = torch.fft.fftshift(torch.fft.fftn(x, dim=(-2, -1))) 279 | 280 | amp = fd.abs() 281 | phase = fd.angle() 282 | 283 | # select a segment to conduct perturbations 284 | start = np.random.randint(0, int(0.5 * sample.shape[1])) 285 | end = start + int(0.5 * sample.shape[1]) 286 | 287 | # phase shift 288 | angles = np.repeat(np.expand_dims(np.random.uniform(low=-np.pi, high=np.pi, size=(sample.shape[0], sample.shape[1])), axis=2), sample.shape[2], axis=2) 289 | phase[:, start:end, :] = phase[:, start:end, :] + angles[:, start:end, :] 290 | 291 | # amp shift 292 | amp[:, start:end, :] = amp[:, start:end, :] + np.random.normal(loc=0., scale=0.8, size=sample.shape)[:, start:end, :] 293 | 294 | cmp = amp * torch.exp(1j * phase) 295 | ifft = torch.squeeze(torch.real(torch.fft.ifftn(torch.fft.ifftshift(cmp), dim=(-2, -1))).reshape([bs, c, h, w])) 296 | 297 | return ifft 298 | 299 | 300 | def ifft_amp_phase_pert_fully(sample): 301 | images = torch.unsqueeze(sample, 1) 302 | bs, c, h, w = images.shape 303 | x = images.reshape([bs * c, h, w]) 304 | fd = torch.fft.fftshift(torch.fft.fftn(x, dim=(-2, -1))) 305 | 306 | amp = fd.abs() 307 | phase = fd.angle() 308 | 309 | # phase shift 310 | angles = np.repeat(np.expand_dims(np.random.uniform(low=-np.pi, high=np.pi, size=(sample.shape[0], sample.shape[1])), axis=2), sample.shape[2], axis=2) 311 | phase = phase + angles 312 | 313 | # amp shift 314 | amp = amp + np.random.normal(loc=0., scale=0.8, size=sample.shape) 315 | 316 | cmp = amp * torch.exp(1j * phase) 317 | ifft = torch.squeeze(torch.real(torch.fft.ifftn(torch.fft.ifftshift(cmp), dim=(-2, -1))).reshape([bs, c, h, w])) 318 | 319 | return ifft 320 | 321 | if __name__ =='__main__': 322 | x = torch.randn(2,171,10) 323 | aug_x = gen_aug(x,'rotation') 324 | print( aug_x.shape ) --------------------------------------------------------------------------------