├── dataset └── .gitkeep ├── model ├── __init__.py └── mobilenet_v2.py ├── utils ├── __init__.py ├── utils.py └── data_aug.py ├── .gitignore ├── dataloader ├── __init__.py ├── gtos_mobile.py ├── minc.py ├── dtd.py └── gtos.py ├── experiments ├── gtos_mobile.finetune.resnet │ ├── network.py │ ├── config.py │ └── train.py ├── gtos.dain.mobilenet │ ├── network.py │ ├── config.py │ └── train.py ├── dtd.dep.mobilenet │ ├── config.py │ ├── network.py │ └── train.py ├── gtos.dain.resnet │ ├── config.py │ ├── network.py │ └── train.py ├── gtos.tean.mobilenet │ ├── config.py │ ├── network.py │ └── train.py ├── gtos_mobile.dep.resnet │ ├── config.py │ ├── network.py │ └── train.py ├── gtos_mobile.dep.mobilenet │ ├── config.py │ ├── network.py │ └── train.py ├── minc.dep.mobilenet │ ├── config.py │ ├── network.py │ └── train.py ├── minc.dep.resnet │ ├── config.py │ ├── network.py │ └── train.py └── dtd.dep.resnet │ ├── config.py │ ├── network.py │ └── train.py ├── README.md └── LICENSE /dataset/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dataset/ 2 | -------------------------------------------------------------------------------- /dataloader/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | 4 | import torch 5 | 6 | 7 | # refer to https://github.com/xternalz/WideResNet-pytorch 8 | def save_checkpoint(state, config, is_best, filename='checkpoint.pth.tar'): 9 | """Saves checkpoint to disk""" 10 | if not os.path.exists(config.snapshot_dir): 11 | os.makedirs(config.snapshot_dir) 12 | filename = config.snapshot_dir + filename 13 | torch.save(state, filename) 14 | if is_best: 15 | shutil.copyfile(filename, config.snapshot_dir + 'model_best.pth.tar') -------------------------------------------------------------------------------- /utils/data_aug.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | _imagenet_pca = { 4 | 'eigval': torch.Tensor([0.2175, 0.0188, 0.0045]), 5 | 'eigvec': torch.Tensor([ 6 | [-0.5675, 0.7192, 0.4009], 7 | [-0.5808, -0.0045, -0.8140], 8 | [-0.5836, -0.6948, 0.4203], 9 | ]) 10 | } 11 | 12 | 13 | class Lighting(object): 14 | """Lighting noise(AlexNet - style PCA - based noise)""" 15 | 16 | def __init__(self, alphastd, eigval=_imagenet_pca['eigval'], 17 | eigvec=_imagenet_pca['eigvec']): 18 | self.alphastd = alphastd 19 | self.eigval = eigval 20 | self.eigvec = eigvec 21 | 22 | def __call__(self, img): 23 | if self.alphastd == 0: 24 | return img 25 | 26 | alpha = img.new().resize_(3).normal_(0, self.alphastd) 27 | rgb = self.eigvec.type_as(img).clone() \ 28 | .mul(alpha.view(1, 3).expand(3, 3)) \ 29 | .mul(self.eigval.view(1, 3).expand(3, 3)) \ 30 | .sum(1).squeeze() 31 | 32 | return img.add(rgb.view(3, 1, 1).expand_as(img)) -------------------------------------------------------------------------------- /experiments/gtos_mobile.finetune.resnet/network.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.autograd import Variable 4 | 5 | 6 | class FinetuneNet(nn.Module): 7 | def __init__(self, nclass, backbone): 8 | super(FinetuneNet, self).__init__() 9 | 10 | self.pretrained = backbone 11 | self.fc = nn.Linear(512, nclass) 12 | 13 | def forward(self, x): 14 | # pre-trained ResNet feature 15 | x = self.pretrained.conv1(x) 16 | x = self.pretrained.bn1(x) 17 | x = self.pretrained.relu(x) 18 | x = self.pretrained.maxpool(x) 19 | x = self.pretrained.layer1(x) 20 | x = self.pretrained.layer2(x) 21 | x = self.pretrained.layer3(x) 22 | x = self.pretrained.layer4(x) 23 | x = self.pretrained.avgpool(x) 24 | 25 | # finetune head 26 | x = torch.flatten(x, 1) 27 | x = self.fc(x) 28 | 29 | return x 30 | 31 | 32 | def test(): 33 | net = FinetuneNet(nclass=23).cuda() 34 | print(net) 35 | x = Variable(torch.randn(1,3,224,224)).cuda() 36 | y = net(x) 37 | print(y) 38 | params = net.parameters() 39 | sum = 0 40 | for param in params: 41 | sum += param.nelement() 42 | print('Total params:', sum) 43 | 44 | 45 | if __name__ == "__main__": 46 | test() 47 | -------------------------------------------------------------------------------- /experiments/gtos.dain.mobilenet/network.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.autograd import Variable 4 | 5 | 6 | class DAIN(nn.Module): 7 | def __init__(self, nclass, model1, model2): 8 | super(DAIN, self).__init__() 9 | self.model1 = model1 10 | self.model2 = model2 11 | self.fc = nn.Linear(1280*2, nclass) 12 | 13 | def forward(self, img, diff_img): 14 | # pre-trained ResNet feature 15 | img_f = self.model1.features(img) 16 | img_f = img_f.mean(3).mean(2) 17 | 18 | # differential angular feature 19 | diff_img_f = self.model2.features(diff_img) 20 | diff_img_f = diff_img_f.mean(3).mean(2) 21 | 22 | # DAIN head 23 | img_f = torch.flatten(img_f, 1) 24 | diff_img_f = torch.flatten(diff_img_f, 1) 25 | diff_img_f = diff_img_f + img_f 26 | out = torch.cat((img_f, diff_img_f), dim=1) 27 | out = self.fc(out) 28 | 29 | return out 30 | 31 | 32 | def test(): 33 | net = DAIN(nclass=23).cuda() 34 | print(net) 35 | x = Variable(torch.randn(1,3,224,224)).cuda() 36 | y = net(x) 37 | print(y) 38 | params = net.parameters() 39 | sum = 0 40 | for param in params: 41 | sum += param.nelement() 42 | print('Total params:', sum) 43 | 44 | 45 | if __name__ == "__main__": 46 | test() 47 | -------------------------------------------------------------------------------- /experiments/dtd.dep.mobilenet/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import sys 4 | import time 5 | 6 | from easydict import EasyDict as edict 7 | 8 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 9 | 10 | C = edict() 11 | config = C 12 | 13 | C.seed = 0 14 | 15 | """please config ROOT_dir and user when u first using""" 16 | C.repo_name = 'pytorch-material-classification' 17 | C.dataset = 'DTD' 18 | C.model = 'DEP' 19 | C.abs_dir = osp.realpath(".") 20 | C.this_dir = C.abs_dir.split(osp.sep)[-1] 21 | C.root_dir = C.abs_dir[:C.abs_dir.index(C.repo_name) + len(C.repo_name)] 22 | C.log_dir = osp.abspath(osp.join(C.root_dir, 'log', C.this_dir)) 23 | C.snapshot_dir = osp.abspath(osp.join(C.log_dir, "snapshot")) 24 | 25 | exp_time = time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime()) 26 | C.log_file = C.log_dir + '/log_' + exp_time + '.log' 27 | C.link_log_file = C.log_file + '/log_last.log' 28 | C.val_log_file = C.log_dir + '/val_' + exp_time + '.log' 29 | C.link_val_log_file = C.log_dir + '/val_last.log' 30 | 31 | """Data Dir and Weight Dir""" 32 | C.dataset_path = osp.join(C.root_dir, 'dataset/dtd/') 33 | C.img_root_folder = C.dataset_path 34 | C.gt_root_folder = C.dataset_path 35 | 36 | 37 | """Path Config""" 38 | 39 | def add_path(path): 40 | if path not in sys.path: 41 | sys.path.insert(0, path) 42 | 43 | 44 | add_path(C.root_dir) 45 | 46 | """Train Config""" 47 | C.lr = 1e-2 48 | C.lr_decay = 40 49 | C.momentum = 0.9 50 | C.weight_decay = 1e-4 51 | C.batch_size = 64 52 | C.start_epoch = 1 53 | C.nepochs = 100 54 | C.eval = False 55 | C.split = '1' 56 | 57 | C.cuda = True 58 | C.gpu = '0' 59 | C.resume = False 60 | C.momentum = 0.9 61 | C.weight_decay = 1e-4 62 | -------------------------------------------------------------------------------- /experiments/gtos.dain.mobilenet/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import sys 4 | import time 5 | 6 | from easydict import EasyDict as edict 7 | 8 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 9 | 10 | C = edict() 11 | config = C 12 | 13 | C.seed = 0 14 | 15 | """please config ROOT_dir and user when u first using""" 16 | C.repo_name = 'pytorch-material-classification' 17 | C.dataset = 'GTOS' 18 | C.model = 'DAIN' 19 | C.abs_dir = osp.realpath(".") 20 | C.this_dir = C.abs_dir.split(osp.sep)[-1] 21 | C.root_dir = C.abs_dir[:C.abs_dir.index(C.repo_name) + len(C.repo_name)] 22 | C.log_dir = osp.abspath(osp.join(C.root_dir, 'log', C.this_dir)) 23 | C.snapshot_dir = osp.abspath(osp.join(C.log_dir, "snapshot")) 24 | 25 | exp_time = time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime()) 26 | C.log_file = C.log_dir + '/log_' + exp_time + '.log' 27 | C.link_log_file = C.log_file + '/log_last.log' 28 | C.val_log_file = C.log_dir + '/val_' + exp_time + '.log' 29 | C.link_val_log_file = C.log_dir + '/val_last.log' 30 | 31 | """Data Dir and Weight Dir""" 32 | C.dataset_path = osp.join(C.root_dir, 'dataset/gtos/') 33 | C.img_root_folder = C.dataset_path 34 | C.gt_root_folder = C.dataset_path 35 | 36 | """Path Config""" 37 | 38 | 39 | def add_path(path): 40 | if path not in sys.path: 41 | sys.path.insert(0, path) 42 | 43 | 44 | add_path(C.root_dir) 45 | 46 | """Train Config""" 47 | C.lr = 1e-2 48 | C.lr_decay = 40 49 | C.momentum = 0.9 50 | C.weight_decay = 1e-4 51 | C.batch_size = 64 52 | C.start_epoch = 1 53 | C.nepochs = 100 54 | C.eval = False 55 | C.split = '1' 56 | 57 | C.cuda = True 58 | C.gpu = '0' 59 | C.resume = False 60 | C.momentum = 0.9 61 | C.weight_decay = 1e-4 62 | -------------------------------------------------------------------------------- /experiments/gtos.dain.resnet/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import sys 4 | import time 5 | 6 | from easydict import EasyDict as edict 7 | 8 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 9 | 10 | C = edict() 11 | config = C 12 | 13 | C.seed = 0 14 | 15 | """please config ROOT_dir and user when u first using""" 16 | C.repo_name = 'pytorch-material-classification' 17 | C.dataset = 'GTOS' 18 | C.model = 'DAIN' 19 | C.abs_dir = osp.realpath(".") 20 | C.this_dir = C.abs_dir.split(osp.sep)[-1] 21 | C.root_dir = C.abs_dir[:C.abs_dir.index(C.repo_name) + len(C.repo_name)] 22 | C.log_dir = osp.abspath(osp.join(C.root_dir, 'log', C.this_dir)) 23 | C.snapshot_dir = osp.abspath(osp.join(C.log_dir, "snapshot")) 24 | 25 | exp_time = time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime()) 26 | C.log_file = C.log_dir + '/log_' + exp_time + '.log' 27 | C.link_log_file = C.log_file + '/log_last.log' 28 | C.val_log_file = C.log_dir + '/val_' + exp_time + '.log' 29 | C.link_val_log_file = C.log_dir + '/val_last.log' 30 | 31 | """Data Dir and Weight Dir""" 32 | C.dataset_path = osp.join(C.root_dir, 'dataset/gtos/') 33 | C.img_root_folder = C.dataset_path 34 | C.gt_root_folder = C.dataset_path 35 | 36 | 37 | """Path Config""" 38 | 39 | def add_path(path): 40 | if path not in sys.path: 41 | sys.path.insert(0, path) 42 | 43 | 44 | add_path(C.root_dir) 45 | 46 | """Train Config""" 47 | C.lr = 1e-2 48 | C.lr_decay = 40 49 | C.momentum = 0.9 50 | C.weight_decay = 1e-4 51 | C.batch_size = 128 52 | C.start_epoch = 1 53 | C.nepochs = 100 54 | C.eval = False 55 | C.split = '1' 56 | 57 | C.cuda = True 58 | C.gpu = '0' 59 | C.resume = False 60 | C.momentum = 0.9 61 | C.weight_decay = 1e-4 62 | -------------------------------------------------------------------------------- /experiments/gtos.tean.mobilenet/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import sys 4 | import time 5 | 6 | from easydict import EasyDict as edict 7 | 8 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 9 | 10 | C = edict() 11 | config = C 12 | 13 | C.seed = 0 14 | 15 | """please config ROOT_dir and user when u first using""" 16 | C.repo_name = 'pytorch-material-classification' 17 | C.dataset = 'GTOS' 18 | C.model = 'TEAN' 19 | C.abs_dir = osp.realpath(".") 20 | C.this_dir = C.abs_dir.split(osp.sep)[-1] 21 | C.root_dir = C.abs_dir[:C.abs_dir.index(C.repo_name) + len(C.repo_name)] 22 | C.log_dir = osp.abspath(osp.join(C.root_dir, 'log', C.this_dir)) 23 | C.snapshot_dir = osp.abspath(osp.join(C.log_dir, "snapshot")) 24 | 25 | exp_time = time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime()) 26 | C.log_file = C.log_dir + '/log_' + exp_time + '.log' 27 | C.link_log_file = C.log_file + '/log_last.log' 28 | C.val_log_file = C.log_dir + '/val_' + exp_time + '.log' 29 | C.link_val_log_file = C.log_dir + '/val_last.log' 30 | 31 | """Data Dir and Weight Dir""" 32 | C.dataset_path = osp.join(C.root_dir, 'dataset/gtos/') 33 | C.img_root_folder = C.dataset_path 34 | C.gt_root_folder = C.dataset_path 35 | 36 | """Path Config""" 37 | 38 | 39 | def add_path(path): 40 | if path not in sys.path: 41 | sys.path.insert(0, path) 42 | 43 | 44 | add_path(C.root_dir) 45 | 46 | """Train Config""" 47 | C.lr = 1e-2 48 | C.lr_decay = 40 49 | C.momentum = 0.9 50 | C.weight_decay = 1e-4 51 | C.batch_size = 64 52 | C.start_epoch = 1 53 | C.nepochs = 100 54 | C.eval = False 55 | C.split = '1' 56 | 57 | C.cuda = True 58 | C.gpu = '0' 59 | C.resume = False 60 | C.momentum = 0.9 61 | C.weight_decay = 1e-4 62 | -------------------------------------------------------------------------------- /experiments/gtos_mobile.dep.resnet/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import sys 4 | import time 5 | 6 | from easydict import EasyDict as edict 7 | 8 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 9 | 10 | C = edict() 11 | config = C 12 | 13 | C.seed = 0 14 | 15 | """please config ROOT_dir and user when u first using""" 16 | C.repo_name = 'pytorch-material-classification' 17 | C.dataset = 'GTOS_MOBILE' 18 | C.model = 'DEP' 19 | C.abs_dir = osp.realpath(".") 20 | C.this_dir = C.abs_dir.split(osp.sep)[-1] 21 | C.root_dir = C.abs_dir[:C.abs_dir.index(C.repo_name) + len(C.repo_name)] 22 | C.log_dir = osp.abspath(osp.join(C.root_dir, 'log', C.this_dir)) 23 | C.snapshot_dir = osp.abspath(osp.join(C.log_dir, "snapshot")) 24 | 25 | exp_time = time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime()) 26 | C.log_file = C.log_dir + '/log_' + exp_time + '.log' 27 | C.link_log_file = C.log_file + '/log_last.log' 28 | C.val_log_file = C.log_dir + '/val_' + exp_time + '.log' 29 | C.link_val_log_file = C.log_dir + '/val_last.log' 30 | 31 | """Data Dir and Weight Dir""" 32 | C.dataset_path = osp.join(C.root_dir, 'dataset/gtos-mobile/') 33 | C.img_root_folder = C.dataset_path 34 | C.gt_root_folder = C.dataset_path 35 | 36 | 37 | """Path Config""" 38 | 39 | def add_path(path): 40 | if path not in sys.path: 41 | sys.path.insert(0, path) 42 | 43 | 44 | add_path(C.root_dir) 45 | 46 | """Train Config""" 47 | C.lr = 1e-2 48 | C.lr_decay = 40 49 | C.momentum = 0.9 50 | C.weight_decay = 1e-4 51 | C.batch_size = 128 52 | C.start_epoch = 1 53 | C.nepochs = 100 54 | C.eval = False 55 | 56 | C.cuda = True 57 | C.gpu = '0' 58 | C.resume = False 59 | C.momentum = 0.9 60 | C.weight_decay = 1e-4 61 | -------------------------------------------------------------------------------- /experiments/gtos_mobile.dep.mobilenet/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import sys 4 | import time 5 | 6 | from easydict import EasyDict as edict 7 | 8 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 9 | 10 | C = edict() 11 | config = C 12 | 13 | C.seed = 0 14 | 15 | """please config ROOT_dir and user when u first using""" 16 | C.repo_name = 'pytorch-material-classification' 17 | C.dataset = 'GTOS_MOBILE' 18 | C.model = 'DEP' 19 | C.abs_dir = osp.realpath(".") 20 | C.this_dir = C.abs_dir.split(osp.sep)[-1] 21 | C.root_dir = C.abs_dir[:C.abs_dir.index(C.repo_name) + len(C.repo_name)] 22 | C.log_dir = osp.abspath(osp.join(C.root_dir, 'log', C.this_dir)) 23 | C.snapshot_dir = osp.abspath(osp.join(C.log_dir, "snapshot")) 24 | 25 | exp_time = time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime()) 26 | C.log_file = C.log_dir + '/log_' + exp_time + '.log' 27 | C.link_log_file = C.log_file + '/log_last.log' 28 | C.val_log_file = C.log_dir + '/val_' + exp_time + '.log' 29 | C.link_val_log_file = C.log_dir + '/val_last.log' 30 | 31 | """Data Dir and Weight Dir""" 32 | C.dataset_path = osp.join(C.root_dir, 'dataset/gtos-mobile/') 33 | C.img_root_folder = C.dataset_path 34 | C.gt_root_folder = C.dataset_path 35 | 36 | 37 | """Path Config""" 38 | 39 | def add_path(path): 40 | if path not in sys.path: 41 | sys.path.insert(0, path) 42 | 43 | 44 | add_path(C.root_dir) 45 | 46 | """Train Config""" 47 | C.lr = 1e-2 48 | C.lr_decay = 40 49 | C.momentum = 0.9 50 | C.weight_decay = 1e-4 51 | C.batch_size = 128 52 | C.start_epoch = 1 53 | C.nepochs = 100 54 | C.eval = False 55 | 56 | C.cuda = True 57 | C.gpu = '0' 58 | C.resume = False 59 | C.momentum = 0.9 60 | C.weight_decay = 1e-4 61 | -------------------------------------------------------------------------------- /experiments/gtos_mobile.finetune.resnet/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import sys 4 | import time 5 | 6 | from easydict import EasyDict as edict 7 | 8 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 9 | 10 | C = edict() 11 | config = C 12 | 13 | C.seed = 0 14 | 15 | """please config ROOT_dir and user when u first using""" 16 | C.repo_name = 'pytorch-material-classification' 17 | C.dataset = 'GTOS_MOBILE' 18 | C.model = 'DEP' 19 | C.abs_dir = osp.realpath(".") 20 | C.this_dir = C.abs_dir.split(osp.sep)[-1] 21 | C.root_dir = C.abs_dir[:C.abs_dir.index(C.repo_name) + len(C.repo_name)] 22 | C.log_dir = osp.abspath(osp.join(C.root_dir, 'log', C.this_dir)) 23 | C.snapshot_dir = osp.abspath(osp.join(C.log_dir, "snapshot")) 24 | 25 | exp_time = time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime()) 26 | C.log_file = C.log_dir + '/log_' + exp_time + '.log' 27 | C.link_log_file = C.log_file + '/log_last.log' 28 | C.val_log_file = C.log_dir + '/val_' + exp_time + '.log' 29 | C.link_val_log_file = C.log_dir + '/val_last.log' 30 | 31 | """Data Dir and Weight Dir""" 32 | C.dataset_path = osp.join(C.root_dir, 'dataset/gtos-mobile/') 33 | C.img_root_folder = C.dataset_path 34 | C.gt_root_folder = C.dataset_path 35 | 36 | 37 | """Path Config""" 38 | 39 | def add_path(path): 40 | if path not in sys.path: 41 | sys.path.insert(0, path) 42 | 43 | 44 | add_path(C.root_dir) 45 | 46 | """Train Config""" 47 | C.lr = 1e-2 48 | C.lr_decay = 40 49 | C.momentum = 0.9 50 | C.weight_decay = 1e-4 51 | C.batch_size = 128 52 | C.start_epoch = 1 53 | C.nepochs = 100 54 | C.eval = False 55 | 56 | C.cuda = True 57 | C.gpu = '0' 58 | C.resume = False 59 | C.momentum = 0.9 60 | C.weight_decay = 1e-4 61 | -------------------------------------------------------------------------------- /experiments/minc.dep.mobilenet/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import sys 4 | import time 5 | 6 | from easydict import EasyDict as edict 7 | 8 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 9 | 10 | C = edict() 11 | config = C 12 | 13 | C.seed = 0 14 | 15 | """please config ROOT_dir and user when u first using""" 16 | C.repo_name = 'pytorch-material-classification' 17 | C.dataset = 'MINC' 18 | C.model = 'DEP' 19 | C.abs_dir = osp.realpath(".") 20 | C.this_dir = C.abs_dir.split(osp.sep)[-1] 21 | C.root_dir = C.abs_dir[:C.abs_dir.index(C.repo_name) + len(C.repo_name)] 22 | C.log_dir = osp.abspath(osp.join(C.root_dir, 'log', C.this_dir)) 23 | C.snapshot_dir = osp.abspath(osp.join(C.log_dir, "snapshot")) 24 | 25 | exp_time = time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime()) 26 | C.log_file = C.log_dir + '/log_' + exp_time + '.log' 27 | C.link_log_file = C.log_file + '/log_last.log' 28 | C.val_log_file = C.log_dir + '/val_' + exp_time + '.log' 29 | C.link_val_log_file = C.log_dir + '/val_last.log' 30 | 31 | """Data Dir and Weight Dir""" 32 | C.dataset_path = osp.join(C.root_dir, 'dataset/minc-2500/') 33 | C.img_root_folder = C.dataset_path 34 | C.gt_root_folder = C.dataset_path 35 | 36 | 37 | """Path Config""" 38 | 39 | def add_path(path): 40 | if path not in sys.path: 41 | sys.path.insert(0, path) 42 | 43 | 44 | add_path(C.root_dir) 45 | 46 | """Train Config""" 47 | C.lr = 1e-2 48 | C.lr_decay = 40 49 | C.momentum = 0.9 50 | C.weight_decay = 1e-4 51 | C.batch_size = 128 52 | C.start_epoch = 1 53 | C.nepochs = 100 54 | C.eval = False 55 | C.split = '1' 56 | 57 | C.cuda = True 58 | C.gpu = '0' 59 | C.resume = False 60 | C.momentum = 0.9 61 | C.weight_decay = 1e-4 62 | -------------------------------------------------------------------------------- /experiments/minc.dep.resnet/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import sys 4 | import time 5 | 6 | from easydict import EasyDict as edict 7 | 8 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 9 | 10 | C = edict() 11 | config = C 12 | 13 | C.seed = 0 14 | 15 | """please config ROOT_dir and user when u first using""" 16 | C.repo_name = 'pytorch-material-classification' 17 | C.dataset = 'MINC' 18 | C.model = 'DEP' 19 | C.abs_dir = osp.realpath(".") 20 | C.this_dir = C.abs_dir.split(osp.sep)[-1] 21 | C.root_dir = C.abs_dir[:C.abs_dir.index(C.repo_name) + len(C.repo_name)] 22 | C.log_dir = osp.abspath(osp.join(C.root_dir, 'log', C.this_dir)) 23 | C.snapshot_dir = osp.abspath(osp.join(C.log_dir, "snapshot")) 24 | 25 | exp_time = time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime()) 26 | C.log_file = C.log_dir + '/log_' + exp_time + '.log' 27 | C.link_log_file = C.log_file + '/log_last.log' 28 | C.val_log_file = C.log_dir + '/val_' + exp_time + '.log' 29 | C.link_val_log_file = C.log_dir + '/val_last.log' 30 | 31 | """Data Dir and Weight Dir""" 32 | C.dataset_path = osp.join(C.root_dir, 'dataset/minc-2500/') 33 | C.img_root_folder = C.dataset_path 34 | C.gt_root_folder = C.dataset_path 35 | 36 | """Path Config""" 37 | 38 | 39 | def add_path(path): 40 | if path not in sys.path: 41 | sys.path.insert(0, path) 42 | 43 | 44 | add_path(C.root_dir) 45 | 46 | """Train Config""" 47 | C.lr = 1e-2 48 | C.lr_decay = 40 49 | C.momentum = 0.9 50 | C.weight_decay = 1e-4 51 | C.batch_size = 128 52 | C.start_epoch = 1 53 | C.nepochs = 100 54 | C.eval = False 55 | C.split = '1' 56 | 57 | C.cuda = True 58 | C.gpu = '0' 59 | C.resume = False 60 | C.momentum = 0.9 61 | C.weight_decay = 1e-4 62 | -------------------------------------------------------------------------------- /experiments/dtd.dep.resnet/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import sys 4 | import time 5 | import numpy as np 6 | from easydict import EasyDict as edict 7 | import argparse 8 | 9 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 10 | 11 | C = edict() 12 | config = C 13 | 14 | C.seed = 0 15 | 16 | """please config ROOT_dir and user when u first using""" 17 | C.repo_name = 'pytorch-material-classification' 18 | C.dataset = 'DTD' 19 | C.model = 'DEP' 20 | C.abs_dir = osp.realpath(".") 21 | C.this_dir = C.abs_dir.split(osp.sep)[-1] 22 | C.root_dir = C.abs_dir[:C.abs_dir.index(C.repo_name) + len(C.repo_name)] 23 | C.log_dir = osp.abspath(osp.join(C.root_dir, 'log', C.this_dir)) 24 | C.snapshot_dir = osp.abspath(osp.join(C.log_dir, "snapshot")) 25 | 26 | exp_time = time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime()) 27 | C.log_file = C.log_dir + '/log_' + exp_time + '.log' 28 | C.link_log_file = C.log_file + '/log_last.log' 29 | C.val_log_file = C.log_dir + '/val_' + exp_time + '.log' 30 | C.link_val_log_file = C.log_dir + '/val_last.log' 31 | 32 | """Data Dir and Weight Dir""" 33 | C.dataset_path = osp.join(C.root_dir, 'dataset/dtd/') 34 | C.img_root_folder = C.dataset_path 35 | C.gt_root_folder = C.dataset_path 36 | 37 | 38 | """Path Config""" 39 | 40 | def add_path(path): 41 | if path not in sys.path: 42 | sys.path.insert(0, path) 43 | 44 | 45 | add_path(C.root_dir) 46 | 47 | """Train Config""" 48 | C.lr = 1e-2 49 | C.lr_decay = 40 50 | C.momentum = 0.9 51 | C.weight_decay = 1e-4 52 | C.batch_size = 64 53 | C.start_epoch = 1 54 | C.nepochs = 100 55 | C.eval = False 56 | C.split = '1' 57 | 58 | C.cuda = True 59 | C.gpu = '0' 60 | C.resume = False 61 | C.momentum = 0.9 62 | C.weight_decay = 1e-4 63 | -------------------------------------------------------------------------------- /dataloader/gtos_mobile.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | 4 | import torch 5 | from utils.data_aug import Lighting 6 | from torchvision import datasets, transforms 7 | 8 | class Dataloder(): 9 | def __init__(self, config): 10 | normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], 11 | std=[0.229, 0.224, 0.225]) 12 | transform_train = transforms.Compose([ 13 | transforms.Resize(256), 14 | transforms.RandomResizedCrop(224), 15 | transforms.RandomHorizontalFlip(), 16 | transforms.ColorJitter(0.4,0.4,0.4), 17 | transforms.ToTensor(), 18 | Lighting(0.1), 19 | normalize, 20 | ]) 21 | transform_test = transforms.Compose([ 22 | transforms.Resize(256), 23 | transforms.CenterCrop(224), 24 | transforms.ToTensor(), 25 | normalize, 26 | ]) 27 | 28 | trainset = datasets.ImageFolder(os.path.join(config.dataset_path, 'train'), transform_train) 29 | testset = datasets.ImageFolder(os.path.join(config.dataset_path, 'test'), transform_test) 30 | 31 | 32 | kwargs = {'num_workers': 8, 'pin_memory': True} if config.cuda else {} 33 | trainloader = torch.utils.data.DataLoader(trainset, batch_size= 34 | config.batch_size, shuffle=True, **kwargs) 35 | testloader = torch.utils.data.DataLoader(testset, batch_size= 36 | config.batch_size, shuffle=False, **kwargs) 37 | 38 | 39 | self.trainloader = trainloader 40 | self.testloader = testloader 41 | self.classes = trainset.classes 42 | 43 | def getloader(self): 44 | return self.classes, self.trainloader, self.testloader 45 | 46 | 47 | if __name__ == "__main__": 48 | data_dir = 'dataset/gtos-mobile' 49 | trainset = datasets.ImageFolder(os.path.join(data_dir, 'train')) 50 | testset = datasets.ImageFolder(os.path.join(data_dir, 'test')) 51 | print(trainset.classes) 52 | print(len(testset)) 53 | -------------------------------------------------------------------------------- /experiments/gtos.dain.resnet/network.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.autograd import Variable 4 | 5 | 6 | class DAIN(nn.Module): 7 | def __init__(self, nclass, model1, model2): 8 | super(DAIN, self).__init__() 9 | self.model1 = model1 10 | self.model2 = model2 11 | self.fc = nn.Linear(512*2, nclass) 12 | 13 | def forward(self, img, diff_img): 14 | # pre-trained ResNet feature 15 | img_f = self.model1.conv1(img) 16 | img_f = self.model1.bn1(img_f) 17 | img_f = self.model1.relu(img_f) 18 | img_f = self.model1.maxpool(img_f) 19 | img_f = self.model1.layer1(img_f) 20 | img_f = self.model1.layer2(img_f) 21 | img_f = self.model1.layer3(img_f) 22 | img_f = self.model1.layer4(img_f) 23 | img_f = self.model1.avgpool(img_f) 24 | 25 | # differential angular feature 26 | diff_img_f = self.model2.conv1(diff_img) 27 | diff_img_f = self.model2.bn1(diff_img_f) 28 | diff_img_f = self.model2.relu(diff_img_f) 29 | diff_img_f = self.model2.maxpool(diff_img_f) 30 | diff_img_f = self.model2.layer1(diff_img_f) 31 | diff_img_f = self.model2.layer2(diff_img_f) 32 | diff_img_f = self.model2.layer3(diff_img_f) 33 | diff_img_f = self.model2.layer4(diff_img_f) 34 | diff_img_f = self.model2.avgpool(diff_img_f) 35 | 36 | # DAIN head 37 | img_f = torch.flatten(img_f, 1) 38 | diff_img_f = torch.flatten(diff_img_f, 1) 39 | diff_img_f = diff_img_f + img_f 40 | out = torch.cat((img_f, diff_img_f), dim=1) 41 | out = self.fc(out) 42 | 43 | return out 44 | 45 | 46 | def test(): 47 | net = DAIN(nclass=23).cuda() 48 | print(net) 49 | x = Variable(torch.randn(1,3,224,224)).cuda() 50 | y = net(x) 51 | print(y) 52 | params = net.parameters() 53 | sum = 0 54 | for param in params: 55 | sum += param.nelement() 56 | print('Total params:', sum) 57 | 58 | 59 | if __name__ == "__main__": 60 | test() 61 | -------------------------------------------------------------------------------- /experiments/dtd.dep.mobilenet/network.py: -------------------------------------------------------------------------------- 1 | import encoding 2 | import torch 3 | import torch.nn as nn 4 | from torch.autograd import Variable 5 | 6 | 7 | class DEPNet(nn.Module): 8 | def __init__(self, nclass, backbone): 9 | super(DEPNet, self).__init__() 10 | n_codes = 8 11 | self.pretrained = backbone.features 12 | self.encode = nn.Sequential( 13 | nn.BatchNorm2d(1280), 14 | encoding.nn.Encoding(D=1280,K=n_codes), 15 | encoding.nn.View(-1, 1280*n_codes), 16 | encoding.nn.Normalize(), 17 | nn.Linear(1280*n_codes, 64) 18 | ) 19 | self.pool = nn.Sequential( 20 | nn.AvgPool2d(7), 21 | encoding.nn.View(-1, 1280), 22 | nn.Linear(1280, 64), 23 | nn.BatchNorm1d(64), 24 | ) 25 | self.fc = nn.Sequential( 26 | encoding.nn.Normalize(), 27 | nn.Linear(64*64, 128), 28 | encoding.nn.Normalize(), 29 | nn.Linear(128, nclass)) 30 | 31 | def forward(self, x): 32 | if isinstance(x, Variable): 33 | _, _, h, w = x.size() 34 | elif isinstance(x, tuple) or isinstance(x, list): 35 | var_input = x 36 | while not isinstance(var_input, Variable): 37 | var_input = var_input[0] 38 | _, _, h, w = var_input.size() 39 | else: 40 | raise RuntimeError('unknown input type: ', type(x)) 41 | 42 | 43 | # pre-trained ResNet feature 44 | x = self.pretrained(x) 45 | 46 | # DEP head 47 | x1 = self.encode(x) 48 | x2 = self.pool(x) 49 | x1 = x1.unsqueeze(1).expand(x1.size(0),x2.size(1),x1.size(-1)) 50 | x = x1*x2.unsqueeze(-1) 51 | x=x.view(-1,x1.size(-1)*x2.size(1)) 52 | x = self.fc(x) 53 | 54 | return x 55 | 56 | 57 | def test(): 58 | net = Net(nclass=23).cuda() 59 | print(net) 60 | x = Variable(torch.randn(1,3,224,224)).cuda() 61 | y = net(x) 62 | print(y) 63 | params = net.parameters() 64 | sum = 0 65 | for param in params: 66 | sum += param.nelement() 67 | print('Total params:', sum) 68 | 69 | 70 | if __name__ == "__main__": 71 | test() 72 | -------------------------------------------------------------------------------- /experiments/gtos_mobile.dep.mobilenet/network.py: -------------------------------------------------------------------------------- 1 | import encoding 2 | import torch 3 | import torch.nn as nn 4 | from torch.autograd import Variable 5 | 6 | 7 | class DEPNet(nn.Module): 8 | def __init__(self, nclass, backbone): 9 | super(DEPNet, self).__init__() 10 | n_codes = 8 11 | self.pretrained = backbone.features 12 | self.encode = nn.Sequential( 13 | nn.BatchNorm2d(1280), 14 | encoding.nn.Encoding(D=1280,K=n_codes), 15 | encoding.nn.View(-1, 1280*n_codes), 16 | encoding.nn.Normalize(), 17 | nn.Linear(1280*n_codes, 64) 18 | ) 19 | self.pool = nn.Sequential( 20 | nn.AvgPool2d(7), 21 | encoding.nn.View(-1, 1280), 22 | nn.Linear(1280, 64), 23 | nn.BatchNorm1d(64), 24 | ) 25 | self.fc = nn.Sequential( 26 | encoding.nn.Normalize(), 27 | nn.Linear(64*64, 128), 28 | encoding.nn.Normalize(), 29 | nn.Linear(128, nclass)) 30 | 31 | def forward(self, x): 32 | if isinstance(x, Variable): 33 | _, _, h, w = x.size() 34 | elif isinstance(x, tuple) or isinstance(x, list): 35 | var_input = x 36 | while not isinstance(var_input, Variable): 37 | var_input = var_input[0] 38 | _, _, h, w = var_input.size() 39 | else: 40 | raise RuntimeError('unknown input type: ', type(x)) 41 | 42 | 43 | # pre-trained ResNet feature 44 | x = self.pretrained(x) 45 | 46 | # DEP head 47 | x1 = self.encode(x) 48 | x2 = self.pool(x) 49 | x1 = x1.unsqueeze(1).expand(x1.size(0),x2.size(1),x1.size(-1)) 50 | x = x1*x2.unsqueeze(-1) 51 | x=x.view(-1,x1.size(-1)*x2.size(1)) 52 | x = self.fc(x) 53 | 54 | return x 55 | 56 | 57 | def test(): 58 | net = DEPNet(nclass=23).cuda() 59 | print(net) 60 | x = Variable(torch.randn(1,3,224,224)).cuda() 61 | y = net(x) 62 | print(y) 63 | params = net.parameters() 64 | sum = 0 65 | for param in params: 66 | sum += param.nelement() 67 | print('Total params:', sum) 68 | 69 | 70 | if __name__ == "__main__": 71 | test() 72 | -------------------------------------------------------------------------------- /experiments/minc.dep.mobilenet/network.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.autograd import Variable 3 | import torch.nn as nn 4 | 5 | import encoding 6 | 7 | 8 | class DEPNet(nn.Module): 9 | def __init__(self, nclass, backbone): 10 | super(DEPNet, self).__init__() 11 | 12 | n_codes = 8 13 | self.pretrained = backbone.features 14 | self.encode = nn.Sequential( 15 | nn.BatchNorm2d(1280), 16 | encoding.nn.Encoding(D=1280, K=n_codes), 17 | encoding.nn.View(-1, 1280 * n_codes), 18 | encoding.nn.Normalize(), 19 | nn.Linear(1280 * n_codes, 64) 20 | ) 21 | self.pool = nn.Sequential( 22 | nn.AvgPool2d(7), 23 | encoding.nn.View(-1, 1280), 24 | nn.Linear(1280, 64), 25 | nn.BatchNorm1d(64), 26 | ) 27 | self.fc = nn.Sequential( 28 | encoding.nn.Normalize(), 29 | nn.Linear(64 * 64, 128), 30 | encoding.nn.Normalize(), 31 | nn.Linear(128, nclass)) 32 | 33 | def forward(self, x): 34 | if isinstance(x, Variable): 35 | _, _, h, w = x.size() 36 | elif isinstance(x, tuple) or isinstance(x, list): 37 | var_input = x 38 | while not isinstance(var_input, Variable): 39 | var_input = var_input[0] 40 | _, _, h, w = var_input.size() 41 | else: 42 | raise RuntimeError('unknown input type: ', type(x)) 43 | 44 | # pre-trained ResNet feature 45 | x = self.pretrained(x) 46 | 47 | # DEP head 48 | x1 = self.encode(x) 49 | x2 = self.pool(x) 50 | x1 = x1.unsqueeze(1).expand(x1.size(0), x2.size(1), x1.size(-1)) 51 | x = x1 * x2.unsqueeze(-1) 52 | x = x.view(-1, x1.size(-1) * x2.size(1)) 53 | x = self.fc(x) 54 | 55 | return x 56 | 57 | 58 | def test(): 59 | net = Net(nclass=23).cuda() 60 | print(net) 61 | x = Variable(torch.randn(1, 3, 224, 224)).cuda() 62 | y = net(x) 63 | print(y) 64 | params = net.parameters() 65 | sum = 0 66 | for param in params: 67 | sum += param.nelement() 68 | print('Total params:', sum) 69 | 70 | 71 | if __name__ == "__main__": 72 | test() 73 | -------------------------------------------------------------------------------- /experiments/gtos.tean.mobilenet/network.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.autograd import Variable 4 | 5 | 6 | class TEAN(nn.Module): 7 | def __init__(self, nclass, model1, model2): 8 | super(TEAN, self).__init__() 9 | self.model1 = model1 10 | self.model2 = model2 11 | self.model1.classifier[1] = nn.Linear(128+128, num_classes) 12 | 13 | self.head = nn.Sequential( 14 | encoding.nn.Encoding(D=1280,K=n_codes), 15 | encoding.nn.View(-1, 1280*n_codes), 16 | encoding.nn.Normalize(), 17 | #nn.ReLU(inplace=True), 18 | nn.Linear(1280*n_codes, 64), 19 | nn.BatchNorm1d(64), 20 | ) 21 | self.pool = nn.Sequential( 22 | nn.AvgPool2d(7), 23 | encoding.nn.View(-1, 1280), 24 | nn.Linear(1280, 64), 25 | nn.BatchNorm1d(64), 26 | ) 27 | self.fc = nn.Sequential( 28 | encoding.nn.Normalize(), 29 | #nn.ReLU(inplace=True), 30 | nn.Linear(64*64, 128), 31 | encoding.nn.Normalize(), 32 | ) 33 | self.pool2 = nn.Sequential( 34 | nn.AvgPool2d(7), 35 | encoding.nn.View(-1, 1280), 36 | nn.Linear(1280, 128), 37 | nn.BatchNorm1d(128), 38 | ) 39 | 40 | def forward(self, img, diff_img): 41 | # pre-trained ResNet feature 42 | img_f = self.model1.features(img) 43 | 44 | # differential angular feature 45 | diff_img_f = self.model2.features(diff_img) 46 | diff_img_f = img_f + diff_img_f 47 | diff_img_f = self.pool2(diff_img_f) 48 | 49 | # dep feature 50 | x1 = self.head(img_fea) 51 | x2 = self.pool(img_fea) 52 | x1 = x1.unsqueeze(1).expand(x1.size(0),x2.size(1),x1.size(-1)) 53 | x = x1*x2.unsqueeze(-1) 54 | enc_fea = x.view(-1,x1.size(-1)*x2.size(1)) 55 | enc_fea = self.fc(enc_fea) 56 | 57 | # TEAN head 58 | out = torch.cat((enc_fea, ang_fea), dim=1) 59 | out = self.model1.classifier(out) 60 | 61 | return out 62 | 63 | 64 | def test(): 65 | net = TEAN(nclass=23).cuda() 66 | print(net) 67 | x = Variable(torch.randn(1,3,224,224)).cuda() 68 | y = net(x) 69 | print(y) 70 | params = net.parameters() 71 | sum = 0 72 | for param in params: 73 | sum += param.nelement() 74 | print('Total params:', sum) 75 | 76 | 77 | if __name__ == "__main__": 78 | test() 79 | -------------------------------------------------------------------------------- /experiments/dtd.dep.resnet/network.py: -------------------------------------------------------------------------------- 1 | import encoding 2 | import torch 3 | import torch.nn as nn 4 | from torch.autograd import Variable 5 | 6 | 7 | class DEPNet(nn.Module): 8 | def __init__(self, nclass, backbone): 9 | super(DEPNet, self).__init__() 10 | 11 | n_codes = 8 12 | self.pretrained = backbone 13 | self.encode = nn.Sequential( 14 | nn.BatchNorm2d(512), 15 | encoding.nn.Encoding(D=512, K=n_codes), 16 | encoding.nn.View(-1, 512 * n_codes), 17 | encoding.nn.Normalize(), 18 | nn.Linear(512 * n_codes, 64) 19 | ) 20 | self.pool = nn.Sequential( 21 | nn.AvgPool2d(7), 22 | encoding.nn.View(-1, 512), 23 | nn.Linear(512, 64), 24 | nn.BatchNorm1d(64), 25 | ) 26 | self.fc = nn.Sequential( 27 | encoding.nn.Normalize(), 28 | nn.Linear(64 * 64, 128), 29 | encoding.nn.Normalize(), 30 | nn.Linear(128, nclass)) 31 | 32 | def forward(self, x): 33 | if isinstance(x, Variable): 34 | _, _, h, w = x.size() 35 | elif isinstance(x, tuple) or isinstance(x, list): 36 | var_input = x 37 | while not isinstance(var_input, Variable): 38 | var_input = var_input[0] 39 | _, _, h, w = var_input.size() 40 | else: 41 | raise RuntimeError('unknown input type: ', type(x)) 42 | 43 | # pre-trained ResNet feature 44 | x = self.pretrained.conv1(x) 45 | x = self.pretrained.bn1(x) 46 | x = self.pretrained.relu(x) 47 | x = self.pretrained.maxpool(x) 48 | x = self.pretrained.layer1(x) 49 | x = self.pretrained.layer2(x) 50 | x = self.pretrained.layer3(x) 51 | x = self.pretrained.layer4(x) 52 | 53 | # DEP head 54 | x1 = self.encode(x) 55 | x2 = self.pool(x) 56 | x1 = x1.unsqueeze(1).expand(x1.size(0), x2.size(1), x1.size(-1)) 57 | x = x1 * x2.unsqueeze(-1) 58 | x = x.view(-1, x1.size(-1) * x2.size(1)) 59 | x = self.fc(x) 60 | 61 | return x 62 | 63 | 64 | def test(): 65 | net = Net(nclass=23).cuda() 66 | print(net) 67 | x = Variable(torch.randn(1, 3, 224, 224)).cuda() 68 | y = net(x) 69 | print(y) 70 | params = net.parameters() 71 | sum = 0 72 | for param in params: 73 | sum += param.nelement() 74 | print('Total params:', sum) 75 | 76 | 77 | if __name__ == "__main__": 78 | test() 79 | -------------------------------------------------------------------------------- /experiments/minc.dep.resnet/network.py: -------------------------------------------------------------------------------- 1 | import encoding 2 | import torch 3 | import torch.nn as nn 4 | from torch.autograd import Variable 5 | 6 | 7 | class DEPNet(nn.Module): 8 | def __init__(self, nclass, backbone): 9 | super(DEPNet, self).__init__() 10 | 11 | n_codes = 8 12 | self.pretrained = backbone 13 | self.encode = nn.Sequential( 14 | nn.BatchNorm2d(512), 15 | encoding.nn.Encoding(D=512,K=n_codes), 16 | encoding.nn.View(-1, 512*n_codes), 17 | encoding.nn.Normalize(), 18 | nn.Linear(512*n_codes, 64) 19 | ) 20 | self.pool = nn.Sequential( 21 | nn.AvgPool2d(7), 22 | encoding.nn.View(-1, 512), 23 | nn.Linear(512, 64), 24 | nn.BatchNorm1d(64), 25 | ) 26 | self.fc = nn.Sequential( 27 | encoding.nn.Normalize(), 28 | nn.Linear(64*64, 128), 29 | encoding.nn.Normalize(), 30 | nn.Linear(128, nclass)) 31 | 32 | def forward(self, x): 33 | if isinstance(x, Variable): 34 | _, _, h, w = x.size() 35 | elif isinstance(x, tuple) or isinstance(x, list): 36 | var_input = x 37 | while not isinstance(var_input, Variable): 38 | var_input = var_input[0] 39 | _, _, h, w = var_input.size() 40 | else: 41 | raise RuntimeError('unknown input type: ', type(x)) 42 | 43 | 44 | # pre-trained ResNet feature 45 | x = self.pretrained.conv1(x) 46 | x = self.pretrained.bn1(x) 47 | x = self.pretrained.relu(x) 48 | x = self.pretrained.maxpool(x) 49 | x = self.pretrained.layer1(x) 50 | x = self.pretrained.layer2(x) 51 | x = self.pretrained.layer3(x) 52 | x = self.pretrained.layer4(x) 53 | 54 | # DEP head 55 | x1 = self.encode(x) 56 | x2 = self.pool(x) 57 | x1 = x1.unsqueeze(1).expand(x1.size(0),x2.size(1),x1.size(-1)) 58 | x = x1*x2.unsqueeze(-1) 59 | x=x.view(-1,x1.size(-1)*x2.size(1)) 60 | x = self.fc(x) 61 | 62 | return x 63 | 64 | 65 | def test(): 66 | net = DEPNet(nclass=23).cuda() 67 | print(net) 68 | x = Variable(torch.randn(1,3,224,224)).cuda() 69 | y = net(x) 70 | print(y) 71 | params = net.parameters() 72 | sum = 0 73 | for param in params: 74 | sum += param.nelement() 75 | print('Total params:', sum) 76 | 77 | 78 | if __name__ == "__main__": 79 | test() 80 | -------------------------------------------------------------------------------- /experiments/gtos_mobile.dep.resnet/network.py: -------------------------------------------------------------------------------- 1 | import encoding 2 | import torch 3 | import torch.nn as nn 4 | from torch.autograd import Variable 5 | 6 | 7 | class DEPNet(nn.Module): 8 | def __init__(self, nclass, backbone): 9 | super(DEPNet, self).__init__() 10 | 11 | n_codes = 8 12 | self.pretrained = backbone 13 | self.encode = nn.Sequential( 14 | nn.BatchNorm2d(512), 15 | encoding.nn.Encoding(D=512,K=n_codes), 16 | encoding.nn.View(-1, 512*n_codes), 17 | encoding.nn.Normalize(), 18 | nn.Linear(512*n_codes, 64) 19 | ) 20 | self.pool = nn.Sequential( 21 | nn.AvgPool2d(7), 22 | encoding.nn.View(-1, 512), 23 | nn.Linear(512, 64), 24 | nn.BatchNorm1d(64), 25 | ) 26 | self.fc = nn.Sequential( 27 | encoding.nn.Normalize(), 28 | nn.Linear(64*64, 128), 29 | encoding.nn.Normalize(), 30 | nn.Linear(128, nclass)) 31 | 32 | def forward(self, x): 33 | if isinstance(x, Variable): 34 | _, _, h, w = x.size() 35 | elif isinstance(x, tuple) or isinstance(x, list): 36 | var_input = x 37 | while not isinstance(var_input, Variable): 38 | var_input = var_input[0] 39 | _, _, h, w = var_input.size() 40 | else: 41 | raise RuntimeError('unknown input type: ', type(x)) 42 | 43 | 44 | # pre-trained ResNet feature 45 | x = self.pretrained.conv1(x) 46 | x = self.pretrained.bn1(x) 47 | x = self.pretrained.relu(x) 48 | x = self.pretrained.maxpool(x) 49 | x = self.pretrained.layer1(x) 50 | x = self.pretrained.layer2(x) 51 | x = self.pretrained.layer3(x) 52 | x = self.pretrained.layer4(x) 53 | 54 | # DEP head 55 | x1 = self.encode(x) 56 | x2 = self.pool(x) 57 | x1 = x1.unsqueeze(1).expand(x1.size(0),x2.size(1),x1.size(-1)) 58 | x = x1*x2.unsqueeze(-1) 59 | x=x.view(-1,x1.size(-1)*x2.size(1)) 60 | x = self.fc(x) 61 | 62 | return x 63 | 64 | 65 | def test(): 66 | net = DEPNet(nclass=23).cuda() 67 | print(net) 68 | x = Variable(torch.randn(1,3,224,224)).cuda() 69 | y = net(x) 70 | print(y) 71 | params = net.parameters() 72 | sum = 0 73 | for param in params: 74 | sum += param.nelement() 75 | print('Total params:', sum) 76 | 77 | 78 | if __name__ == "__main__": 79 | test() 80 | -------------------------------------------------------------------------------- /dataloader/minc.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | 4 | import torch 5 | import torch.utils.data as data 6 | from PIL import Image 7 | from torchvision import transforms 8 | 9 | def find_classes(dir): 10 | classes = [d for d in os.listdir(dir) if os.path.isdir(os.path.join(dir, d))] 11 | classes.sort() 12 | class_to_idx = {classes[i]: i for i in range(len(classes))} 13 | return classes, class_to_idx 14 | 15 | 16 | def make_dataset(txtnames, datadir, class_to_idx): 17 | images = [] 18 | labels = [] 19 | for txtname in txtnames: 20 | with open(txtname, 'r') as lines: 21 | for line in lines: 22 | classname = line.split('/')[1] 23 | _img = os.path.join(datadir, line.strip()) 24 | assert os.path.isfile(_img) 25 | images.append(_img) 26 | labels.append(class_to_idx[classname]) 27 | 28 | return images, labels 29 | 30 | 31 | class MINCDataloader(data.Dataset): 32 | def __init__(self, config, transform=None, train=True): 33 | classes, class_to_idx = find_classes(os.path.join(config.dataset_path, 'images')) 34 | self.classes = classes 35 | self.class_to_idx = class_to_idx 36 | self.train = train 37 | self.transform = transform 38 | 39 | if train: 40 | filename = [os.path.join(config.dataset_path, 'labels/train' + config.split + '.txt'), 41 | os.path.join(config.dataset_path, 'labels/validate' + config.split + '.txt')] 42 | else: 43 | filename = [os.path.join(config.dataset_path, 'labels/test' + config.split + '.txt')] 44 | 45 | self.images, self.labels = make_dataset(filename, config.dataset_path, class_to_idx) 46 | assert (len(self.images) == len(self.labels)) 47 | 48 | def __getitem__(self, index): 49 | _img = Image.open(self.images[index]).convert('RGB') 50 | _label = self.labels[index] 51 | if self.transform is not None: 52 | _img = self.transform(_img) 53 | 54 | return _img, _label 55 | 56 | def __len__(self): 57 | return len(self.images) 58 | 59 | 60 | class Dataloder(): 61 | def __init__(self, config): 62 | normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], 63 | std=[0.229, 0.224, 0.225]) 64 | transform_train = transforms.Compose([ 65 | transforms.Resize(256), 66 | transforms.RandomResizedCrop(224), 67 | transforms.RandomHorizontalFlip(), 68 | transforms.RandomVerticalFlip(), 69 | transforms.ToTensor(), 70 | normalize, 71 | ]) 72 | transform_test = transforms.Compose([ 73 | transforms.Resize(256), 74 | transforms.CenterCrop(224), 75 | transforms.ToTensor(), 76 | normalize, 77 | ]) 78 | 79 | trainset = MINCDataloader(config, transform_train, train=True) 80 | testset = MINCDataloader(config, transform_test, train=False) 81 | 82 | kwargs = {'num_workers': 8, 'pin_memory': True} 83 | trainloader = torch.utils.data.DataLoader(trainset, batch_size= 84 | config.batch_size, shuffle=True, **kwargs) 85 | testloader = torch.utils.data.DataLoader(testset, batch_size= 86 | config.batch_size, shuffle=False, **kwargs) 87 | self.classes = trainset.classes 88 | self.trainloader = trainloader 89 | self.testloader = testloader 90 | 91 | def getloader(self): 92 | return self.classes, self.trainloader, self.testloader 93 | -------------------------------------------------------------------------------- /dataloader/dtd.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | 4 | import torch 5 | import torch.utils.data as data 6 | from PIL import Image 7 | from torchvision import transforms 8 | 9 | 10 | def find_classes(dir): 11 | classes = [d for d in os.listdir(dir) if os.path.isdir(os.path.join(dir, d))] 12 | classes.sort() 13 | class_to_idx = {classes[i]: i for i in range(len(classes))} 14 | return classes, class_to_idx 15 | 16 | 17 | def make_dataset(txtnames, datadir, class_to_idx): 18 | images = [] 19 | labels = [] 20 | for txtname in txtnames: 21 | with open(txtname, 'r') as lines: 22 | for line in lines: 23 | classname = line.split('/')[0] 24 | _img = os.path.join(datadir, 'images', line.strip()) 25 | assert os.path.isfile(_img) 26 | images.append(_img) 27 | labels.append(class_to_idx[classname]) 28 | 29 | return images, labels 30 | 31 | 32 | class DTDDataloader(data.Dataset): 33 | def __init__(self, config, transform=None, train=True): 34 | classes, class_to_idx = find_classes(os.path.join(config.dataset_path, 'images')) 35 | self.classes = classes 36 | self.class_to_idx = class_to_idx 37 | self.train = train 38 | self.transform = transform 39 | 40 | if train: 41 | filename = [os.path.join(config.dataset_path, 'labels/train' + config.split + '.txt'), 42 | os.path.join(config.dataset_path, 'labels/val' + config.split + '.txt')] 43 | else: 44 | filename = [os.path.join(config.dataset_path, 'labels/test' + config.split + '.txt')] 45 | 46 | self.images, self.labels = make_dataset(filename, config.dataset_path, class_to_idx) 47 | assert (len(self.images) == len(self.labels)) 48 | 49 | def __getitem__(self, index): 50 | _img = Image.open(self.images[index]).convert('RGB') 51 | _label = self.labels[index] 52 | if self.transform is not None: 53 | _img = self.transform(_img) 54 | 55 | return _img, _label 56 | 57 | def __len__(self): 58 | return len(self.images) 59 | 60 | 61 | class Dataloder(): 62 | def __init__(self, config): 63 | normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], 64 | std=[0.229, 0.224, 0.225]) 65 | transform_train = transforms.Compose([ 66 | transforms.Resize(256), 67 | transforms.RandomResizedCrop(224), 68 | transforms.RandomHorizontalFlip(), 69 | transforms.RandomVerticalFlip(), 70 | transforms.ToTensor(), 71 | normalize, 72 | ]) 73 | transform_test = transforms.Compose([ 74 | transforms.Resize(256), 75 | transforms.CenterCrop(224), 76 | transforms.ToTensor(), 77 | normalize, 78 | ]) 79 | 80 | trainset = DTDDataloader(config, transform_train, train=True) 81 | testset = DTDDataloader(config, transform_test, train=False) 82 | 83 | kwargs = {'num_workers': 8, 'pin_memory': True} 84 | trainloader = torch.utils.data.DataLoader(trainset, batch_size= 85 | config.batch_size, shuffle=True, **kwargs) 86 | testloader = torch.utils.data.DataLoader(testset, batch_size= 87 | config.batch_size, shuffle=False, **kwargs) 88 | self.classes = trainset.classes 89 | self.trainloader = trainloader 90 | self.testloader = testloader 91 | 92 | def getloader(self): 93 | return self.classes, self.trainloader, self.testloader 94 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pytorch Material Classification 2 | 3 | This repo provides examples for material classification in GTOS, GTOS-MOBILE, DTD and MINC dataset using PyTorch. 4 | 5 | ## Setup 6 | 7 | ### Prerequisites 8 | 9 | - Ubuntu 10 | - Pytorch 11 | - `pip3 install torch torchvision` 12 | - Easydict 13 | - `pip3 install easydict` 14 | - tqdm 15 | - `pip3 install tqdm` 16 | - [Pytorch-Encoding](https://github.com/zhanghang1989/PyTorch-Encoding) 17 | - `pip3 install torch-encoding` 18 | - Note: You need to install Pytorch 1.0 for torch-encoding, or you can modify the encoding layer based on [this](https://github.com/zhanghang1989/PyTorch-Encoding/issues/161) for latest Pytorch. 19 | 20 | ### Getting Started 21 | 22 | - Clone this repo: 23 | ```bash 24 | git clone https://github.com/jiaxue1993/pytorch-material-classification.git 25 | cd pytorch-material-classification/ 26 | ``` 27 | 28 | - Download [GTOS](https://1drv.ms/u/s!AmTf4gl42ObncLmEnEv4R5LyxT4?e=ekkFfX), [GTOS_MOBILE](https://1drv.ms/u/s!AmTf4gl42ObnblEtikrw4HfD9fc?e=LjJir4), [DTD](https://www.robots.ox.ac.uk/~vgg/data/dtd/download/dtd-r1.0.1.tar.gz), [MINC](http://opensurfaces.cs.cornell.edu/static/minc/minc-2500.tar.gz) to the dataset folder 29 | 30 | - Navigate to different experiment folder and train network. For example, you can finetune ResNet on GTOS-MOBILE dataset with followint command 31 | ```bash 32 | cd experiments/gtos_mobile.finetune.resnet/ 33 | python train.py 34 | ``` 35 | 36 | ## Accuracy & Statistics 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 |
Base ModelDatasetMethodAccuracyPretrained Model
MobileNetGTOSFinetune80.4One Drive
DAIN82.5
DEP83.3
TEAN84.7
ResNet-50DTDDEP73.2
MINCDEP82.0
82 | 83 | ## Citation 84 | 85 | Please consider citing following projects in your publications if it helps your research. 86 | 87 | **Differential Angular Imaging for Material Recognition** [[pdf]](http://openaccess.thecvf.com/content_cvpr_2017/papers/Xue_Differential_Angular_Imaging_CVPR_2017_paper.pdf) 88 | ``` 89 | @inproceedings{xue2017differential, 90 | title={Differential angular imaging for material recognition}, 91 | author={Xue, Jia and Zhang, Hang and Dana, Kristin and Nishino, Ko}, 92 | booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition}, 93 | pages={764--773}, 94 | year={2017} 95 | } 96 | ``` 97 | 98 | **Deep Texture Manifold for Ground Terrain Recognition** [[pdf]](http://openaccess.thecvf.com/content_cvpr_2018/papers/Xue_Deep_Texture_Manifold_CVPR_2018_paper.pdf) 99 | ``` 100 | @inproceedings{xue2018deep, 101 | title={Deep texture manifold for ground terrain recognition}, 102 | author={Xue, Jia and Zhang, Hang and Dana, Kristin}, 103 | booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition}, 104 | pages={558--567}, 105 | year={2018} 106 | } 107 | ``` 108 | 109 | **Deep TEN: Texture Encoding Network** [[pdf]](http://openaccess.thecvf.com/content_cvpr_2017/papers/Zhang_Deep_TEN_Texture_CVPR_2017_paper.pdf) 110 | ``` 111 | @inproceedings{zhang2017deep, 112 | title={Deep ten: Texture encoding network}, 113 | author={Zhang, Hang and Xue, Jia and Dana, Kristin}, 114 | booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, 115 | pages={708--717}, 116 | year={2017} 117 | } 118 | ``` 119 | 120 | **Differential Viewpoints for Ground Terrain Material Recognition** [[pdf]](https://ieeexplore.ieee.org/abstract/document/9200748)[[arxiv]](https://arxiv.org/pdf/2009.11072.pdf) 121 | ``` 122 | @article{xue2020differential, 123 | title={Differential Viewpoints for Ground Terrain Material Recognition}, 124 | author={Xue, Jia and Zhang, Hang and Nishino, Ko and Dana, Kristin}, 125 | journal={IEEE Transactions on Pattern Analysis and Machine Intelligence}, 126 | year={2020}, 127 | publisher={IEEE} 128 | } 129 | ``` 130 | 131 | ## Acknowledgement 132 | Part of the code comes from [PyTorch-Encoding](https://github.com/zhanghang1989/PyTorch-Encoding), [TorchSeg](https://github.com/ycszen/TorchSeg), [pytorch-mobilenet-v2](https://github.com/tonylins/pytorch-mobilenet-v2) 133 | -------------------------------------------------------------------------------- /model/mobilenet_v2.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import math 3 | 4 | 5 | def conv_bn(inp, oup, stride): 6 | return nn.Sequential( 7 | nn.Conv2d(inp, oup, 3, stride, 1, bias=False), 8 | nn.BatchNorm2d(oup), 9 | nn.ReLU6(inplace=True) 10 | ) 11 | 12 | 13 | def conv_1x1_bn(inp, oup): 14 | return nn.Sequential( 15 | nn.Conv2d(inp, oup, 1, 1, 0, bias=False), 16 | nn.BatchNorm2d(oup), 17 | nn.ReLU6(inplace=True) 18 | ) 19 | 20 | 21 | def make_divisible(x, divisible_by=8): 22 | import numpy as np 23 | return int(np.ceil(x * 1. / divisible_by) * divisible_by) 24 | 25 | 26 | class InvertedResidual(nn.Module): 27 | def __init__(self, inp, oup, stride, expand_ratio): 28 | super(InvertedResidual, self).__init__() 29 | self.stride = stride 30 | assert stride in [1, 2] 31 | 32 | hidden_dim = int(inp * expand_ratio) 33 | self.use_res_connect = self.stride == 1 and inp == oup 34 | 35 | if expand_ratio == 1: 36 | self.conv = nn.Sequential( 37 | # dw 38 | nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False), 39 | nn.BatchNorm2d(hidden_dim), 40 | nn.ReLU6(inplace=True), 41 | # pw-linear 42 | nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), 43 | nn.BatchNorm2d(oup), 44 | ) 45 | else: 46 | self.conv = nn.Sequential( 47 | # pw 48 | nn.Conv2d(inp, hidden_dim, 1, 1, 0, bias=False), 49 | nn.BatchNorm2d(hidden_dim), 50 | nn.ReLU6(inplace=True), 51 | # dw 52 | nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False), 53 | nn.BatchNorm2d(hidden_dim), 54 | nn.ReLU6(inplace=True), 55 | # pw-linear 56 | nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), 57 | nn.BatchNorm2d(oup), 58 | ) 59 | 60 | def forward(self, x): 61 | if self.use_res_connect: 62 | return x + self.conv(x) 63 | else: 64 | return self.conv(x) 65 | 66 | 67 | class MobileNetV2(nn.Module): 68 | def __init__(self, n_class=1000, input_size=224, width_mult=1.): 69 | super(MobileNetV2, self).__init__() 70 | block = InvertedResidual 71 | input_channel = 32 72 | last_channel = 1280 73 | interverted_residual_setting = [ 74 | # t, c, n, s 75 | [1, 16, 1, 1], 76 | [6, 24, 2, 2], 77 | [6, 32, 3, 2], 78 | [6, 64, 4, 2], 79 | [6, 96, 3, 1], 80 | [6, 160, 3, 2], 81 | [6, 320, 1, 1], 82 | ] 83 | 84 | # building first layer 85 | assert input_size % 32 == 0 86 | # input_channel = make_divisible(input_channel * width_mult) # first channel is always 32! 87 | self.last_channel = make_divisible(last_channel * width_mult) if width_mult > 1.0 else last_channel 88 | self.features = [conv_bn(3, input_channel, 2)] 89 | # building inverted residual blocks 90 | for t, c, n, s in interverted_residual_setting: 91 | output_channel = make_divisible(c * width_mult) if t > 1 else c 92 | for i in range(n): 93 | if i == 0: 94 | self.features.append(block(input_channel, output_channel, s, expand_ratio=t)) 95 | else: 96 | self.features.append(block(input_channel, output_channel, 1, expand_ratio=t)) 97 | input_channel = output_channel 98 | # building last several layers 99 | self.features.append(conv_1x1_bn(input_channel, self.last_channel)) 100 | # make it nn.Sequential 101 | self.features = nn.Sequential(*self.features) 102 | 103 | # building classifier 104 | self.classifier = nn.Linear(self.last_channel, n_class) 105 | 106 | self._initialize_weights() 107 | 108 | def forward(self, x): 109 | x = self.features(x) 110 | x = x.mean(3).mean(2) 111 | x = self.classifier(x) 112 | return x 113 | 114 | def _initialize_weights(self): 115 | for m in self.modules(): 116 | if isinstance(m, nn.Conv2d): 117 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 118 | m.weight.data.normal_(0, math.sqrt(2. / n)) 119 | if m.bias is not None: 120 | m.bias.data.zero_() 121 | elif isinstance(m, nn.BatchNorm2d): 122 | m.weight.data.fill_(1) 123 | m.bias.data.zero_() 124 | elif isinstance(m, nn.Linear): 125 | n = m.weight.size(1) 126 | m.weight.data.normal_(0, 0.01) 127 | m.bias.data.zero_() 128 | 129 | 130 | def mobilenet_v2(pretrained=True): 131 | model = MobileNetV2(width_mult=1) 132 | 133 | if pretrained: 134 | try: 135 | from torch.hub import load_state_dict_from_url 136 | except ImportError: 137 | from torch.utils.model_zoo import load_url as load_state_dict_from_url 138 | state_dict = load_state_dict_from_url( 139 | 'https://www.dropbox.com/s/47tyzpofuuyyv1b/mobilenetv2_1.0-f2a8633.pth.tar?dl=1', progress=True) 140 | model.load_state_dict(state_dict) 141 | return model 142 | 143 | 144 | if __name__ == '__main__': 145 | net = mobilenet_v2(True) 146 | 147 | 148 | 149 | 150 | 151 | -------------------------------------------------------------------------------- /dataloader/gtos.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | import random 4 | 5 | import torch 6 | import torch.utils.data as data 7 | import torchvision.transforms.functional as TF 8 | from PIL import Image 9 | from torchvision import transforms 10 | 11 | 12 | def find_classes(classdir): 13 | classes = [] 14 | class_to_idx = {} 15 | with open(classdir, 'r') as f: 16 | for line in f: 17 | label, name = line.split(' ') 18 | classes.append(name) 19 | class_to_idx[name] = int(label) - 1 20 | return classes, class_to_idx 21 | 22 | 23 | def make_dataset(txtname, datadir): 24 | rgbimages = [] 25 | diffimages = [] 26 | labels = [] 27 | with open(txtname, "r") as lines: 28 | for line in lines: 29 | name, label = line.split(' ') 30 | name = name.split('/')[-1] 31 | for filename in os.listdir(os.path.join(datadir, 'diff_imgs', name)): 32 | _rgbimg = os.path.join(datadir, 'color_imgs', name, filename) 33 | _diffimg = os.path.join(datadir, 'diff_imgs', name, filename) 34 | assert os.path.isfile(_rgbimg) 35 | rgbimages.append(_rgbimg) 36 | diffimages.append(_diffimg) 37 | labels.append(int(label)-1) 38 | 39 | return rgbimages, diffimages, labels 40 | 41 | 42 | class GTOSDataloder(data.Dataset): 43 | def __init__(self, config, train=True, transform=None): 44 | classes, class_to_idx = find_classes(os.path.join(config.dataset_path, 'labels/classInd.txt')) 45 | self.classes = classes 46 | self.class_to_idx = class_to_idx 47 | self.train = train 48 | self.transform = transform 49 | self.rgbnormalize = transforms.Normalize(mean=[0.447, 0.388, 0.340], 50 | std=[0.216, 0.204, 0.197]) 51 | self.diffnormalize = transforms.Normalize(mean=[0.495, 0.495, 0.495], 52 | std=[0.07, 0.07, 0.07]) 53 | 54 | if train: 55 | filename = os.path.join(config.dataset_path, 'labels/train'+ config.split +'.txt') 56 | else: 57 | filename = os.path.join(config.dataset_path, 'labels/test'+ config.split +'.txt') 58 | 59 | self.rgbimages, self.diffimages, self.labels = make_dataset(filename, config.dataset_path) 60 | assert (len(self.rgbimages) == len(self.labels)) 61 | 62 | def train_transform(self, _rgbimg, _diffimg): 63 | # sizes = [(224,224),(246,246),(268,268)] 64 | 65 | resize = transforms.Resize(size=(256, 256)) 66 | # resize = transforms.Resize(size=sizes[random.randint(0,2)]) 67 | _rgbimg, _diffimg = resize(_rgbimg), resize(_diffimg) 68 | 69 | # Random crop 70 | i, j, h, w = transforms.RandomCrop.get_params( 71 | _rgbimg, output_size=(224, 224)) 72 | _rgbimg = TF.crop(_rgbimg, i, j, h, w) 73 | _diffimg = TF.crop(_diffimg, i, j, h, w) 74 | 75 | # Random horizontal and vertical flip 76 | if random.random() > 0.5: 77 | _rgbimg = TF.hflip(_rgbimg) 78 | _diffimg = TF.hflip(_diffimg) 79 | if random.random() > 0.5: 80 | _rgbimg = TF.vflip(_rgbimg) 81 | _diffimg = TF.vflip(_diffimg) 82 | 83 | # To tensor 84 | _rgbimg = TF.to_tensor(_rgbimg) 85 | _diffimg = TF.to_tensor(_diffimg) 86 | 87 | # Normalize 88 | _rgbimg = self.rgbnormalize(_rgbimg) 89 | _diffimg = self.diffnormalize(_diffimg) 90 | 91 | return _rgbimg, _diffimg 92 | 93 | def test_transform(self, _rgbimg, _diffimg): 94 | 95 | rgbtransform_test = transforms.Compose([ 96 | transforms.Resize(256), 97 | transforms.CenterCrop(224), 98 | transforms.ToTensor(), 99 | self.rgbnormalize, 100 | ]) 101 | 102 | difftransform_test = transforms.Compose([ 103 | transforms.Resize(256), 104 | transforms.CenterCrop(224), 105 | transforms.ToTensor(), 106 | self.diffnormalize, 107 | ]) 108 | 109 | _rgbimg = rgbtransform_test(_rgbimg) 110 | _diffimg = difftransform_test(_diffimg) 111 | 112 | return _rgbimg, _diffimg 113 | 114 | 115 | def __getitem__(self, index): 116 | _rgbimg = Image.open(self.rgbimages[index]).convert('RGB') 117 | _diffimg = Image.open(self.diffimages[index]).convert('RGB') 118 | _label = self.labels[index] 119 | if self.transform is not None: 120 | if self.train: 121 | _rgbimg, _diffimg = self.train_transform(_rgbimg, _diffimg) 122 | else: 123 | _rgbimg, _diffimg = self.test_transform(_rgbimg, _diffimg) 124 | 125 | 126 | return _rgbimg, _diffimg, _label 127 | 128 | def __len__(self): 129 | return len(self.rgbimages) 130 | # return 10000 131 | 132 | 133 | class Dataloder(): 134 | def __init__(self, config): 135 | 136 | trainset = GTOSDataloder(config, train=True, transform=True) 137 | testset = GTOSDataloder(config, train=False, transform=True) 138 | 139 | kwargs = {'num_workers': 0, 'pin_memory': True} if config.cuda else {} 140 | trainloader = torch.utils.data.DataLoader(trainset, batch_size= 141 | config.batch_size, shuffle=True, **kwargs) 142 | testloader = torch.utils.data.DataLoader(testset, batch_size= 143 | config.batch_size, shuffle=False, **kwargs) 144 | self.classes = trainset.classes 145 | self.trainloader = trainloader 146 | self.testloader = testloader 147 | 148 | def getloader(self): 149 | return self.classes, self.trainloader, self.testloader 150 | 151 | 152 | 153 | if __name__ == "__main__": 154 | trainset = GTOSDataloder(None, train=True) 155 | testset = GTOSDataloder(None, train=False) 156 | print(len(trainset.classes)) 157 | print(len(testset)) 158 | -------------------------------------------------------------------------------- /experiments/dtd.dep.mobilenet/train.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import os 4 | import sys 5 | 6 | import torch 7 | import torch.nn as nn 8 | import torch.optim as optim 9 | from torch.autograd import Variable 10 | from tqdm import tqdm 11 | from config import config 12 | from utils.utils import save_checkpoint 13 | 14 | from dataloader.dtd import Dataloder 15 | from model.mobilenet_v2 import mobilenet_v2 16 | from network import DEPNet 17 | 18 | # global variable 19 | best_pred = 100.0 20 | errlist_train = [] 21 | errlist_val = [] 22 | 23 | 24 | def adjust_learning_rate(optimizer, config, epoch): 25 | lr = config.lr * (0.1 ** ((epoch - 1) // config.lr_decay)) 26 | if (epoch - 1) % config.lr_decay == 0: 27 | print('LR is set to {}'.format(lr)) 28 | 29 | for param_group in optimizer.param_groups: 30 | param_group['lr'] = lr 31 | 32 | 33 | def main(): 34 | # init the config 35 | global best_pred, errlist_train, errlist_val 36 | config.cuda = config.cuda and torch.cuda.is_available() 37 | torch.manual_seed(config.seed) 38 | if config.cuda: 39 | os.environ["CUDA_VISIBLE_DEVICES"] = config.gpu 40 | torch.cuda.manual_seed(config.seed) 41 | # init dataloader 42 | classes, train_loader, test_loader = Dataloder(config).getloader() 43 | 44 | # init the model 45 | backbone = mobilenet_v2(True) 46 | model = DEPNet(len(classes), backbone) 47 | print(model) 48 | # criterion and optimizer 49 | criterion = nn.CrossEntropyLoss() 50 | optimizer = optim.SGD(model.parameters(), lr=config.lr, momentum=config.momentum, 51 | weight_decay=config.weight_decay) 52 | if config.cuda: 53 | model.cuda() 54 | # Please use CUDA_VISIBLE_DEVICES to control the number of gpus 55 | model = torch.nn.DataParallel(model) 56 | 57 | # check point 58 | if config.resume is not None: 59 | if os.path.isfile(config.resume): 60 | print("=> loading checkpoint '{}'".format(config.resume)) 61 | checkpoint = torch.load(config.resume) 62 | config.start_epoch = checkpoint['epoch'] + 1 63 | best_pred = checkpoint['best_pred'] 64 | errlist_train = checkpoint['errlist_train'] 65 | errlist_val = checkpoint['errlist_val'] 66 | model.load_state_dict(checkpoint['state_dict']) 67 | optimizer.load_state_dict(checkpoint['optimizer']) 68 | print("=> loaded checkpoint '{}' (epoch {})" 69 | .format(config.resume, checkpoint['epoch'])) 70 | else: 71 | print("=> no resume checkpoint found at '{}'". \ 72 | format(config.resume)) 73 | 74 | def train(epoch): 75 | model.train() 76 | global best_pred, errlist_train 77 | train_loss, correct, total = 0, 0, 0 78 | adjust_learning_rate(optimizer, config, epoch) 79 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 80 | pbar = tqdm(range(len(train_loader)), file=sys.stdout, 81 | bar_format=bar_format) 82 | data_loader = iter(train_loader) 83 | 84 | for batch_idx in pbar: 85 | data, target = data_loader.next() 86 | if config.cuda: 87 | data, target = data.cuda(), target.cuda() 88 | data, target = Variable(data), Variable(target) 89 | optimizer.zero_grad() 90 | output = model(data) 91 | loss = criterion(output, target) 92 | loss.backward() 93 | optimizer.step() 94 | 95 | train_loss += loss.item() 96 | pred = output.data.max(1)[1] 97 | correct += pred.eq(target.data).cpu().sum().numpy() 98 | total += target.size(0) 99 | err = 100 - 100. * correct / total 100 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 101 | + 'Iter: {}/{} '.format(batch_idx, len(train_loader)) \ 102 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 103 | (train_loss / (batch_idx + 1), 104 | err, total - correct, total) 105 | pbar.set_description(print_str) 106 | errlist_train += [err] 107 | 108 | def test(epoch): 109 | model.eval() 110 | global best_pred, errlist_train, errlist_val 111 | test_loss, correct, total = 0, 0, 0 112 | is_best = False 113 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 114 | pbar = tqdm(range(len(test_loader)), file=sys.stdout, 115 | bar_format=bar_format) 116 | data_loader = iter(test_loader) 117 | with torch.no_grad(): 118 | for batch_idx in pbar: 119 | data, target = data_loader.next() 120 | if config.cuda: 121 | data, target = data.cuda(), target.cuda() 122 | output = model(data) 123 | test_loss += criterion(output, target).item() 124 | # get the index of the max log-probability 125 | pred = output.data.max(1)[1] 126 | correct += pred.eq(target.data).cpu().sum().numpy() 127 | total += target.size(0) 128 | 129 | err = 100 - 100. * correct / total 130 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 131 | + 'Iter: {}/{} '.format(batch_idx, len(test_loader)) \ 132 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 133 | (test_loss / (batch_idx + 1), 134 | err, total - correct, total) 135 | pbar.set_description(print_str) 136 | 137 | if config.eval: 138 | print('Error rate is %.3f' % err) 139 | return 140 | # save checkpoint 141 | errlist_val += [err] 142 | if err < best_pred: 143 | best_pred = err 144 | is_best = True 145 | print('Best Accuracy: %.3f' % (100 - best_pred)) 146 | save_checkpoint({ 147 | 'epoch': epoch, 148 | 'state_dict': model.state_dict(), 149 | 'optimizer': optimizer.state_dict(), 150 | 'best_pred': best_pred, 151 | 'errlist_train': errlist_train, 152 | 'errlist_val': errlist_val, 153 | }, config, is_best=is_best) 154 | 155 | if config.eval: 156 | test(config.start_epoch) 157 | return 158 | 159 | for epoch in range(config.start_epoch, config.nepochs + 1): 160 | print('Epoch:', epoch) 161 | train(epoch) 162 | test(epoch) 163 | 164 | 165 | if __name__ == "__main__": 166 | main() 167 | -------------------------------------------------------------------------------- /experiments/dtd.dep.resnet/train.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import os 4 | import sys 5 | 6 | import torch 7 | import torch.nn as nn 8 | import torch.optim as optim 9 | from torch.autograd import Variable 10 | from torchvision.models import resnet 11 | from tqdm import tqdm 12 | from config import config 13 | from utils.utils import save_checkpoint 14 | 15 | from dataloader.dtd import Dataloder 16 | from network import DEPNet 17 | 18 | # global variable 19 | best_pred = 100.0 20 | errlist_train = [] 21 | errlist_val = [] 22 | 23 | 24 | def adjust_learning_rate(optimizer, config, epoch): 25 | lr = config.lr * (0.1 ** ((epoch - 1) // config.lr_decay)) 26 | if (epoch - 1) % config.lr_decay == 0: 27 | print('LR is set to {}'.format(lr)) 28 | 29 | for param_group in optimizer.param_groups: 30 | param_group['lr'] = lr 31 | 32 | 33 | def main(): 34 | # init the config 35 | global best_pred, errlist_train, errlist_val 36 | config.cuda = config.cuda and torch.cuda.is_available() 37 | torch.manual_seed(config.seed) 38 | if config.cuda: 39 | os.environ["CUDA_VISIBLE_DEVICES"] = config.gpu 40 | torch.cuda.manual_seed(config.seed) 41 | # init dataloader 42 | classes, train_loader, test_loader = Dataloder(config).getloader() 43 | 44 | # init the model 45 | backbone = resnet.resnet18(pretrained=True) 46 | model = DEPNet(len(classes), backbone) 47 | print(model) 48 | # criterion and optimizer 49 | criterion = nn.CrossEntropyLoss() 50 | optimizer = optim.SGD(model.parameters(), lr=config.lr, momentum=config.momentum, 51 | weight_decay=config.weight_decay) 52 | if config.cuda: 53 | model.cuda() 54 | # Please use CUDA_VISIBLE_DEVICES to control the number of gpus 55 | model = torch.nn.DataParallel(model) 56 | 57 | # check point 58 | if config.resume is not None: 59 | if os.path.isfile(config.resume): 60 | print("=> loading checkpoint '{}'".format(config.resume)) 61 | checkpoint = torch.load(config.resume) 62 | config.start_epoch = checkpoint['epoch'] + 1 63 | best_pred = checkpoint['best_pred'] 64 | errlist_train = checkpoint['errlist_train'] 65 | errlist_val = checkpoint['errlist_val'] 66 | model.load_state_dict(checkpoint['state_dict']) 67 | optimizer.load_state_dict(checkpoint['optimizer']) 68 | print("=> loaded checkpoint '{}' (epoch {})" 69 | .format(config.resume, checkpoint['epoch'])) 70 | else: 71 | print("=> no resume checkpoint found at '{}'". \ 72 | format(config.resume)) 73 | 74 | def train(epoch): 75 | model.train() 76 | global best_pred, errlist_train 77 | train_loss, correct, total = 0, 0, 0 78 | adjust_learning_rate(optimizer, config, epoch) 79 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 80 | pbar = tqdm(range(len(train_loader)), file=sys.stdout, 81 | bar_format=bar_format) 82 | data_loader = iter(train_loader) 83 | 84 | for batch_idx in pbar: 85 | data, target = data_loader.next() 86 | if config.cuda: 87 | data, target = data.cuda(), target.cuda() 88 | data, target = Variable(data), Variable(target) 89 | optimizer.zero_grad() 90 | output = model(data) 91 | loss = criterion(output, target) 92 | loss.backward() 93 | optimizer.step() 94 | 95 | train_loss += loss.item() 96 | pred = output.data.max(1)[1] 97 | correct += pred.eq(target.data).cpu().sum().numpy() 98 | total += target.size(0) 99 | err = 100 - 100. * correct / total 100 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 101 | + 'Iter: {}/{} '.format(batch_idx, len(train_loader)) \ 102 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 103 | (train_loss / (batch_idx + 1), 104 | err, total - correct, total) 105 | pbar.set_description(print_str) 106 | errlist_train += [err] 107 | 108 | def test(epoch): 109 | model.eval() 110 | global best_pred, errlist_train, errlist_val 111 | test_loss, correct, total = 0, 0, 0 112 | is_best = False 113 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 114 | pbar = tqdm(range(len(test_loader)), file=sys.stdout, 115 | bar_format=bar_format) 116 | data_loader = iter(test_loader) 117 | with torch.no_grad(): 118 | for batch_idx in pbar: 119 | data, target = data_loader.next() 120 | if config.cuda: 121 | data, target = data.cuda(), target.cuda() 122 | output = model(data) 123 | test_loss += criterion(output, target).item() 124 | # get the index of the max log-probability 125 | pred = output.data.max(1)[1] 126 | correct += pred.eq(target.data).cpu().sum().numpy() 127 | total += target.size(0) 128 | 129 | err = 100 - 100. * correct / total 130 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 131 | + 'Iter: {}/{} '.format(batch_idx, len(test_loader)) \ 132 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 133 | (test_loss / (batch_idx + 1), 134 | err, total - correct, total) 135 | pbar.set_description(print_str) 136 | 137 | if config.eval: 138 | print('Error rate is %.3f' % err) 139 | return 140 | # save checkpoint 141 | errlist_val += [err] 142 | if err < best_pred: 143 | best_pred = err 144 | is_best = True 145 | print('Best Accuracy: %.3f' % (100 - best_pred)) 146 | save_checkpoint({ 147 | 'epoch': epoch, 148 | 'state_dict': model.state_dict(), 149 | 'optimizer': optimizer.state_dict(), 150 | 'best_pred': best_pred, 151 | 'errlist_train': errlist_train, 152 | 'errlist_val': errlist_val, 153 | }, config, is_best=is_best) 154 | 155 | if config.eval: 156 | test(config.start_epoch) 157 | return 158 | 159 | for epoch in range(config.start_epoch, config.nepochs + 1): 160 | print('Epoch:', epoch) 161 | train(epoch) 162 | test(epoch) 163 | 164 | 165 | if __name__ == "__main__": 166 | main() 167 | -------------------------------------------------------------------------------- /experiments/minc.dep.mobilenet/train.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import os 4 | import sys 5 | 6 | import torch 7 | import torch.nn as nn 8 | import torch.optim as optim 9 | from torch.autograd import Variable 10 | from tqdm import tqdm 11 | from config import config 12 | from utils.utils import save_checkpoint 13 | 14 | from dataloader.minc import Dataloder 15 | from model.mobilenet_v2 import mobilenet_v2 16 | from network import DEPNet 17 | 18 | # global variable 19 | best_pred = 100.0 20 | errlist_train = [] 21 | errlist_val = [] 22 | 23 | 24 | def adjust_learning_rate(optimizer, config, epoch): 25 | lr = config.lr * (0.1 ** ((epoch - 1) // config.lr_decay)) 26 | if (epoch - 1) % config.lr_decay == 0: 27 | print('LR is set to {}'.format(lr)) 28 | 29 | for param_group in optimizer.param_groups: 30 | param_group['lr'] = lr 31 | 32 | 33 | def main(): 34 | # init the config 35 | global best_pred, errlist_train, errlist_val 36 | config.cuda = config.cuda and torch.cuda.is_available() 37 | torch.manual_seed(config.seed) 38 | if config.cuda: 39 | os.environ["CUDA_VISIBLE_DEVICES"] = config.gpu 40 | torch.cuda.manual_seed(config.seed) 41 | # init dataloader 42 | classes, train_loader, test_loader = Dataloder(config).getloader() 43 | 44 | # init the model 45 | backbone = mobilenet_v2(True) 46 | model = DEPNet(len(classes), backbone) 47 | print(model) 48 | # criterion and optimizer 49 | criterion = nn.CrossEntropyLoss() 50 | optimizer = optim.SGD(model.parameters(), lr=config.lr, momentum=config.momentum, 51 | weight_decay=config.weight_decay) 52 | if config.cuda: 53 | model.cuda() 54 | # Please use CUDA_VISIBLE_DEVICES to control the number of gpus 55 | model = torch.nn.DataParallel(model) 56 | 57 | # check point 58 | if config.resume is not None: 59 | if os.path.isfile(config.resume): 60 | print("=> loading checkpoint '{}'".format(config.resume)) 61 | checkpoint = torch.load(config.resume) 62 | config.start_epoch = checkpoint['epoch'] + 1 63 | best_pred = checkpoint['best_pred'] 64 | errlist_train = checkpoint['errlist_train'] 65 | errlist_val = checkpoint['errlist_val'] 66 | model.load_state_dict(checkpoint['state_dict']) 67 | optimizer.load_state_dict(checkpoint['optimizer']) 68 | print("=> loaded checkpoint '{}' (epoch {})" 69 | .format(config.resume, checkpoint['epoch'])) 70 | else: 71 | print("=> no resume checkpoint found at '{}'". \ 72 | format(config.resume)) 73 | 74 | def train(epoch): 75 | model.train() 76 | global best_pred, errlist_train 77 | train_loss, correct, total = 0, 0, 0 78 | adjust_learning_rate(optimizer, config, epoch) 79 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 80 | pbar = tqdm(range(len(train_loader)), file=sys.stdout, 81 | bar_format=bar_format) 82 | data_loader = iter(train_loader) 83 | 84 | for batch_idx in pbar: 85 | data, target = data_loader.next() 86 | if config.cuda: 87 | data, target = data.cuda(), target.cuda() 88 | data, target = Variable(data), Variable(target) 89 | optimizer.zero_grad() 90 | output = model(data) 91 | loss = criterion(output, target) 92 | loss.backward() 93 | optimizer.step() 94 | 95 | train_loss += loss.item() 96 | pred = output.data.max(1)[1] 97 | correct += pred.eq(target.data).cpu().sum().numpy() 98 | total += target.size(0) 99 | err = 100 - 100. * correct / total 100 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 101 | + 'Iter: {}/{} '.format(batch_idx, len(train_loader)) \ 102 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 103 | (train_loss / (batch_idx + 1), 104 | err, total - correct, total) 105 | pbar.set_description(print_str) 106 | errlist_train += [err] 107 | 108 | def test(epoch): 109 | model.eval() 110 | global best_pred, errlist_train, errlist_val 111 | test_loss, correct, total = 0, 0, 0 112 | is_best = False 113 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 114 | pbar = tqdm(range(len(test_loader)), file=sys.stdout, 115 | bar_format=bar_format) 116 | data_loader = iter(test_loader) 117 | with torch.no_grad(): 118 | for batch_idx in pbar: 119 | data, target = data_loader.next() 120 | if config.cuda: 121 | data, target = data.cuda(), target.cuda() 122 | output = model(data) 123 | test_loss += criterion(output, target).item() 124 | # get the index of the max log-probability 125 | pred = output.data.max(1)[1] 126 | correct += pred.eq(target.data).cpu().sum().numpy() 127 | total += target.size(0) 128 | 129 | err = 100 - 100. * correct / total 130 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 131 | + 'Iter: {}/{} '.format(batch_idx, len(test_loader)) \ 132 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 133 | (test_loss / (batch_idx + 1), 134 | err, total - correct, total) 135 | pbar.set_description(print_str) 136 | 137 | if config.eval: 138 | print('Error rate is %.3f' % err) 139 | return 140 | # save checkpoint 141 | errlist_val += [err] 142 | if err < best_pred: 143 | best_pred = err 144 | is_best = True 145 | print('Best Accuracy: %.3f' % (100 - best_pred)) 146 | save_checkpoint({ 147 | 'epoch': epoch, 148 | 'state_dict': model.state_dict(), 149 | 'optimizer': optimizer.state_dict(), 150 | 'best_pred': best_pred, 151 | 'errlist_train': errlist_train, 152 | 'errlist_val': errlist_val, 153 | }, config, is_best=is_best) 154 | 155 | if config.eval: 156 | test(config.start_epoch) 157 | return 158 | 159 | for epoch in range(config.start_epoch, config.nepochs + 1): 160 | print('Epoch:', epoch) 161 | train(epoch) 162 | test(epoch) 163 | 164 | 165 | if __name__ == "__main__": 166 | main() 167 | -------------------------------------------------------------------------------- /experiments/minc.dep.resnet/train.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import os 4 | import sys 5 | 6 | import torch 7 | import torch.nn as nn 8 | import torch.optim as optim 9 | from torch.autograd import Variable 10 | from torchvision.models import resnet 11 | from tqdm import tqdm 12 | from config import config 13 | from utils.utils import save_checkpoint 14 | 15 | from dataloader.minc import Dataloder 16 | from network import DEPNet 17 | 18 | # global variable 19 | best_pred = 100.0 20 | errlist_train = [] 21 | errlist_val = [] 22 | 23 | 24 | def adjust_learning_rate(optimizer, config, epoch): 25 | lr = config.lr * (0.1 ** ((epoch - 1) // config.lr_decay)) 26 | if (epoch - 1) % config.lr_decay == 0: 27 | print('LR is set to {}'.format(lr)) 28 | 29 | for param_group in optimizer.param_groups: 30 | param_group['lr'] = lr 31 | 32 | 33 | def main(): 34 | # init the config 35 | global best_pred, errlist_train, errlist_val 36 | config.cuda = config.cuda and torch.cuda.is_available() 37 | torch.manual_seed(config.seed) 38 | if config.cuda: 39 | os.environ["CUDA_VISIBLE_DEVICES"] = config.gpu 40 | torch.cuda.manual_seed(config.seed) 41 | # init dataloader 42 | classes, train_loader, test_loader = Dataloder(config).getloader() 43 | 44 | # init the model 45 | backbone = resnet.resnet18(pretrained=True) 46 | model = DEPNet(len(classes), backbone) 47 | print(model) 48 | # criterion and optimizer 49 | criterion = nn.CrossEntropyLoss() 50 | optimizer = optim.SGD(model.parameters(), lr=config.lr, momentum=config.momentum, 51 | weight_decay=config.weight_decay) 52 | if config.cuda: 53 | model.cuda() 54 | # Please use CUDA_VISIBLE_DEVICES to control the number of gpus 55 | model = torch.nn.DataParallel(model) 56 | 57 | # check point 58 | if config.resume is not None: 59 | if os.path.isfile(config.resume): 60 | print("=> loading checkpoint '{}'".format(config.resume)) 61 | checkpoint = torch.load(config.resume) 62 | config.start_epoch = checkpoint['epoch'] + 1 63 | best_pred = checkpoint['best_pred'] 64 | errlist_train = checkpoint['errlist_train'] 65 | errlist_val = checkpoint['errlist_val'] 66 | model.load_state_dict(checkpoint['state_dict']) 67 | optimizer.load_state_dict(checkpoint['optimizer']) 68 | print("=> loaded checkpoint '{}' (epoch {})" 69 | .format(config.resume, checkpoint['epoch'])) 70 | else: 71 | print("=> no resume checkpoint found at '{}'". \ 72 | format(config.resume)) 73 | 74 | def train(epoch): 75 | model.train() 76 | global best_pred, errlist_train 77 | train_loss, correct, total = 0, 0, 0 78 | adjust_learning_rate(optimizer, config, epoch) 79 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 80 | pbar = tqdm(range(len(train_loader)), file=sys.stdout, 81 | bar_format=bar_format) 82 | data_loader = iter(train_loader) 83 | 84 | for batch_idx in pbar: 85 | data, target = data_loader.next() 86 | if config.cuda: 87 | data, target = data.cuda(), target.cuda() 88 | data, target = Variable(data), Variable(target) 89 | optimizer.zero_grad() 90 | output = model(data) 91 | loss = criterion(output, target) 92 | loss.backward() 93 | optimizer.step() 94 | 95 | train_loss += loss.item() 96 | pred = output.data.max(1)[1] 97 | correct += pred.eq(target.data).cpu().sum().numpy() 98 | total += target.size(0) 99 | err = 100 - 100. * correct / total 100 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 101 | + 'Iter: {}/{} '.format(batch_idx, len(train_loader)) \ 102 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 103 | (train_loss / (batch_idx + 1), 104 | err, total - correct, total) 105 | pbar.set_description(print_str) 106 | errlist_train += [err] 107 | 108 | def test(epoch): 109 | model.eval() 110 | global best_pred, errlist_train, errlist_val 111 | test_loss, correct, total = 0, 0, 0 112 | is_best = False 113 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 114 | pbar = tqdm(range(len(test_loader)), file=sys.stdout, 115 | bar_format=bar_format) 116 | data_loader = iter(test_loader) 117 | with torch.no_grad(): 118 | for batch_idx in pbar: 119 | data, target = data_loader.next() 120 | if config.cuda: 121 | data, target = data.cuda(), target.cuda() 122 | output = model(data) 123 | test_loss += criterion(output, target).item() 124 | # get the index of the max log-probability 125 | pred = output.data.max(1)[1] 126 | correct += pred.eq(target.data).cpu().sum().numpy() 127 | total += target.size(0) 128 | 129 | err = 100 - 100. * correct / total 130 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 131 | + 'Iter: {}/{} '.format(batch_idx, len(test_loader)) \ 132 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 133 | (test_loss / (batch_idx + 1), 134 | err, total - correct, total) 135 | pbar.set_description(print_str) 136 | 137 | if config.eval: 138 | print('Error rate is %.3f' % err) 139 | return 140 | # save checkpoint 141 | errlist_val += [err] 142 | if err < best_pred: 143 | best_pred = err 144 | is_best = True 145 | print('Best Accuracy: %.3f' % (100 - best_pred)) 146 | save_checkpoint({ 147 | 'epoch': epoch, 148 | 'state_dict': model.state_dict(), 149 | 'optimizer': optimizer.state_dict(), 150 | 'best_pred': best_pred, 151 | 'errlist_train': errlist_train, 152 | 'errlist_val': errlist_val, 153 | }, config, is_best=is_best) 154 | 155 | if config.eval: 156 | test(config.start_epoch) 157 | return 158 | 159 | for epoch in range(config.start_epoch, config.nepochs + 1): 160 | print('Epoch:', epoch) 161 | train(epoch) 162 | test(epoch) 163 | 164 | 165 | if __name__ == "__main__": 166 | main() 167 | -------------------------------------------------------------------------------- /experiments/gtos_mobile.dep.mobilenet/train.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import os 4 | import sys 5 | 6 | import torch 7 | import torch.nn as nn 8 | import torch.optim as optim 9 | from torch.autograd import Variable 10 | from tqdm import tqdm 11 | from config import config 12 | from utils.utils import save_checkpoint 13 | 14 | from dataloader.gtos_mobile import Dataloder 15 | from model.mobilenet_v2 import mobilenet_v2 16 | from network import DEPNet 17 | 18 | # global variable 19 | best_pred = 100.0 20 | errlist_train = [] 21 | errlist_val = [] 22 | 23 | 24 | def adjust_learning_rate(optimizer, config, epoch): 25 | lr = config.lr * (0.1 ** ((epoch - 1) // config.lr_decay)) 26 | if (epoch - 1) % config.lr_decay == 0: 27 | print('LR is set to {}'.format(lr)) 28 | 29 | for param_group in optimizer.param_groups: 30 | param_group['lr'] = lr 31 | 32 | 33 | def main(): 34 | # init the config 35 | global best_pred, errlist_train, errlist_val 36 | config.cuda = config.cuda and torch.cuda.is_available() 37 | torch.manual_seed(config.seed) 38 | if config.cuda: 39 | os.environ["CUDA_VISIBLE_DEVICES"] = config.gpu 40 | torch.cuda.manual_seed(config.seed) 41 | # init dataloader 42 | classes, train_loader, test_loader = Dataloder(config).getloader() 43 | 44 | # init the model 45 | backbone = mobilenet_v2(True) 46 | model = DEPNet(len(classes), backbone) 47 | print(model) 48 | # criterion and optimizer 49 | criterion = nn.CrossEntropyLoss() 50 | optimizer = optim.SGD(model.parameters(), lr=config.lr, momentum=config.momentum, 51 | weight_decay=config.weight_decay) 52 | if config.cuda: 53 | model.cuda() 54 | # Please use CUDA_VISIBLE_DEVICES to control the number of gpus 55 | model = torch.nn.DataParallel(model) 56 | 57 | # check point 58 | if config.resume is not None: 59 | if os.path.isfile(config.resume): 60 | print("=> loading checkpoint '{}'".format(config.resume)) 61 | checkpoint = torch.load(config.resume) 62 | config.start_epoch = checkpoint['epoch'] + 1 63 | best_pred = checkpoint['best_pred'] 64 | errlist_train = checkpoint['errlist_train'] 65 | errlist_val = checkpoint['errlist_val'] 66 | model.load_state_dict(checkpoint['state_dict']) 67 | optimizer.load_state_dict(checkpoint['optimizer']) 68 | print("=> loaded checkpoint '{}' (epoch {})" 69 | .format(config.resume, checkpoint['epoch'])) 70 | else: 71 | print("=> no resume checkpoint found at '{}'". \ 72 | format(config.resume)) 73 | 74 | def train(epoch): 75 | model.train() 76 | global best_pred, errlist_train 77 | train_loss, correct, total = 0, 0, 0 78 | adjust_learning_rate(optimizer, config, epoch) 79 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 80 | pbar = tqdm(range(len(train_loader)), file=sys.stdout, 81 | bar_format=bar_format) 82 | data_loader = iter(train_loader) 83 | 84 | for batch_idx in pbar: 85 | data, target = data_loader.next() 86 | if config.cuda: 87 | data, target = data.cuda(), target.cuda() 88 | data, target = Variable(data), Variable(target) 89 | optimizer.zero_grad() 90 | output = model(data) 91 | loss = criterion(output, target) 92 | loss.backward() 93 | optimizer.step() 94 | 95 | train_loss += loss.item() 96 | pred = output.data.max(1)[1] 97 | correct += pred.eq(target.data).cpu().sum().numpy() 98 | total += target.size(0) 99 | err = 100 - 100. * correct / total 100 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 101 | + 'Iter: {}/{} '.format(batch_idx, len(train_loader)) \ 102 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 103 | (train_loss / (batch_idx + 1), 104 | err, total - correct, total) 105 | pbar.set_description(print_str) 106 | errlist_train += [err] 107 | 108 | def test(epoch): 109 | model.eval() 110 | global best_pred, errlist_train, errlist_val 111 | test_loss, correct, total = 0, 0, 0 112 | is_best = False 113 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 114 | pbar = tqdm(range(len(test_loader)), file=sys.stdout, 115 | bar_format=bar_format) 116 | data_loader = iter(test_loader) 117 | with torch.no_grad(): 118 | for batch_idx in pbar: 119 | data, target = data_loader.next() 120 | if config.cuda: 121 | data, target = data.cuda(), target.cuda() 122 | output = model(data) 123 | test_loss += criterion(output, target).item() 124 | # get the index of the max log-probability 125 | pred = output.data.max(1)[1] 126 | correct += pred.eq(target.data).cpu().sum().numpy() 127 | total += target.size(0) 128 | 129 | err = 100 - 100. * correct / total 130 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 131 | + 'Iter: {}/{} '.format(batch_idx, len(test_loader)) \ 132 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 133 | (test_loss / (batch_idx + 1), 134 | err, total - correct, total) 135 | pbar.set_description(print_str) 136 | 137 | if config.eval: 138 | print('Error rate is %.3f' % err) 139 | return 140 | # save checkpoint 141 | errlist_val += [err] 142 | if err < best_pred: 143 | best_pred = err 144 | is_best = True 145 | print('Best Accuracy: %.3f' % (100 - best_pred)) 146 | save_checkpoint({ 147 | 'epoch': epoch, 148 | 'state_dict': model.state_dict(), 149 | 'optimizer': optimizer.state_dict(), 150 | 'best_pred': best_pred, 151 | 'errlist_train': errlist_train, 152 | 'errlist_val': errlist_val, 153 | }, config, is_best=is_best) 154 | 155 | if config.eval: 156 | test(config.start_epoch) 157 | return 158 | 159 | for epoch in range(config.start_epoch, config.nepochs + 1): 160 | print('Epoch:', epoch) 161 | train(epoch) 162 | test(epoch) 163 | 164 | 165 | if __name__ == "__main__": 166 | main() 167 | -------------------------------------------------------------------------------- /experiments/gtos_mobile.dep.resnet/train.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import os 4 | import sys 5 | 6 | import torch 7 | import torch.nn as nn 8 | import torch.optim as optim 9 | from torch.autograd import Variable 10 | from torchvision.models import resnet 11 | from tqdm import tqdm 12 | from config import config 13 | from utils.utils import save_checkpoint 14 | 15 | from dataloader.gtos_mobile import Dataloder 16 | from network import DEPNet 17 | 18 | # global variable 19 | best_pred = 100.0 20 | errlist_train = [] 21 | errlist_val = [] 22 | 23 | 24 | def adjust_learning_rate(optimizer, config, epoch): 25 | lr = config.lr * (0.1 ** ((epoch - 1) // config.lr_decay)) 26 | if (epoch - 1) % config.lr_decay == 0: 27 | print('LR is set to {}'.format(lr)) 28 | 29 | for param_group in optimizer.param_groups: 30 | param_group['lr'] = lr 31 | 32 | 33 | def main(): 34 | # init the config 35 | global best_pred, errlist_train, errlist_val 36 | config.cuda = config.cuda and torch.cuda.is_available() 37 | torch.manual_seed(config.seed) 38 | if config.cuda: 39 | os.environ["CUDA_VISIBLE_DEVICES"] = config.gpu 40 | torch.cuda.manual_seed(config.seed) 41 | # init dataloader 42 | classes, train_loader, test_loader = Dataloder(config).getloader() 43 | 44 | # init the model 45 | backbone = resnet.resnet18(pretrained=True) 46 | model = DEPNet(len(classes), backbone) 47 | print(model) 48 | # criterion and optimizer 49 | criterion = nn.CrossEntropyLoss() 50 | optimizer = optim.SGD(model.parameters(), lr=config.lr, momentum=config.momentum, 51 | weight_decay=config.weight_decay) 52 | if config.cuda: 53 | model.cuda() 54 | # Please use CUDA_VISIBLE_DEVICES to control the number of gpus 55 | model = torch.nn.DataParallel(model) 56 | 57 | # check point 58 | if config.resume is not None: 59 | if os.path.isfile(config.resume): 60 | print("=> loading checkpoint '{}'".format(config.resume)) 61 | checkpoint = torch.load(config.resume) 62 | config.start_epoch = checkpoint['epoch'] + 1 63 | best_pred = checkpoint['best_pred'] 64 | errlist_train = checkpoint['errlist_train'] 65 | errlist_val = checkpoint['errlist_val'] 66 | model.load_state_dict(checkpoint['state_dict']) 67 | optimizer.load_state_dict(checkpoint['optimizer']) 68 | print("=> loaded checkpoint '{}' (epoch {})" 69 | .format(config.resume, checkpoint['epoch'])) 70 | else: 71 | print("=> no resume checkpoint found at '{}'". \ 72 | format(config.resume)) 73 | 74 | def train(epoch): 75 | model.train() 76 | global best_pred, errlist_train 77 | train_loss, correct, total = 0, 0, 0 78 | adjust_learning_rate(optimizer, config, epoch) 79 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 80 | pbar = tqdm(range(len(train_loader)), file=sys.stdout, 81 | bar_format=bar_format) 82 | data_loader = iter(train_loader) 83 | 84 | for batch_idx in pbar: 85 | data, target = data_loader.next() 86 | if config.cuda: 87 | data, target = data.cuda(), target.cuda() 88 | data, target = Variable(data), Variable(target) 89 | optimizer.zero_grad() 90 | output = model(data) 91 | loss = criterion(output, target) 92 | loss.backward() 93 | optimizer.step() 94 | 95 | train_loss += loss.item() 96 | pred = output.data.max(1)[1] 97 | correct += pred.eq(target.data).cpu().sum().numpy() 98 | total += target.size(0) 99 | err = 100 - 100. * correct / total 100 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 101 | + 'Iter: {}/{} '.format(batch_idx, len(train_loader)) \ 102 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 103 | (train_loss / (batch_idx + 1), 104 | err, total - correct, total) 105 | pbar.set_description(print_str) 106 | errlist_train += [err] 107 | 108 | def test(epoch): 109 | model.eval() 110 | global best_pred, errlist_train, errlist_val 111 | test_loss, correct, total = 0, 0, 0 112 | is_best = False 113 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 114 | pbar = tqdm(range(len(test_loader)), file=sys.stdout, 115 | bar_format=bar_format) 116 | data_loader = iter(test_loader) 117 | with torch.no_grad(): 118 | for batch_idx in pbar: 119 | data, target = data_loader.next() 120 | if config.cuda: 121 | data, target = data.cuda(), target.cuda() 122 | output = model(data) 123 | test_loss += criterion(output, target).item() 124 | # get the index of the max log-probability 125 | pred = output.data.max(1)[1] 126 | correct += pred.eq(target.data).cpu().sum().numpy() 127 | total += target.size(0) 128 | 129 | err = 100 - 100. * correct / total 130 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 131 | + 'Iter: {}/{} '.format(batch_idx, len(test_loader)) \ 132 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 133 | (test_loss / (batch_idx + 1), 134 | err, total - correct, total) 135 | pbar.set_description(print_str) 136 | 137 | if config.eval: 138 | print('Error rate is %.3f' % err) 139 | return 140 | # save checkpoint 141 | errlist_val += [err] 142 | if err < best_pred: 143 | best_pred = err 144 | is_best = True 145 | print('Best Accuracy: %.3f' % (100 - best_pred)) 146 | save_checkpoint({ 147 | 'epoch': epoch, 148 | 'state_dict': model.state_dict(), 149 | 'optimizer': optimizer.state_dict(), 150 | 'best_pred': best_pred, 151 | 'errlist_train': errlist_train, 152 | 'errlist_val': errlist_val, 153 | }, config, is_best=is_best) 154 | 155 | if config.eval: 156 | test(config.start_epoch) 157 | return 158 | 159 | for epoch in range(config.start_epoch, config.nepochs + 1): 160 | print('Epoch:', epoch) 161 | train(epoch) 162 | test(epoch) 163 | 164 | 165 | if __name__ == "__main__": 166 | main() 167 | -------------------------------------------------------------------------------- /experiments/gtos_mobile.finetune.resnet/train.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import os 4 | import sys 5 | 6 | import torch 7 | import torch.nn as nn 8 | import torch.optim as optim 9 | from torch.autograd import Variable 10 | from torchvision.models import resnet 11 | from tqdm import tqdm 12 | from config import config 13 | from utils.utils import save_checkpoint 14 | 15 | from dataloader.gtos_mobile import Dataloder 16 | from network import FinetuneNet 17 | 18 | # global variable 19 | best_pred = 100.0 20 | errlist_train = [] 21 | errlist_val = [] 22 | 23 | 24 | def adjust_learning_rate(optimizer, config, epoch): 25 | lr = config.lr * (0.1 ** ((epoch - 1) // config.lr_decay)) 26 | if (epoch - 1) % config.lr_decay == 0: 27 | print('LR is set to {}'.format(lr)) 28 | 29 | for param_group in optimizer.param_groups: 30 | param_group['lr'] = lr 31 | 32 | 33 | def main(): 34 | # init the config 35 | global best_pred, errlist_train, errlist_val 36 | config.cuda = config.cuda and torch.cuda.is_available() 37 | torch.manual_seed(config.seed) 38 | if config.cuda: 39 | os.environ["CUDA_VISIBLE_DEVICES"] = config.gpu 40 | torch.cuda.manual_seed(config.seed) 41 | # init dataloader 42 | classes, train_loader, test_loader = Dataloder(config).getloader() 43 | 44 | # init the model 45 | backbone = resnet.resnet18(pretrained=True) 46 | model = FinetuneNet(len(classes), backbone) 47 | print(model) 48 | # criterion and optimizer 49 | criterion = nn.CrossEntropyLoss() 50 | optimizer = optim.SGD(model.parameters(), lr=config.lr, momentum=config.momentum, 51 | weight_decay=config.weight_decay) 52 | if config.cuda: 53 | model.cuda() 54 | # Please use CUDA_VISIBLE_DEVICES to control the number of gpus 55 | model = torch.nn.DataParallel(model) 56 | 57 | # check point 58 | if config.resume is not None: 59 | if os.path.isfile(config.resume): 60 | print("=> loading checkpoint '{}'".format(config.resume)) 61 | checkpoint = torch.load(config.resume) 62 | config.start_epoch = checkpoint['epoch'] + 1 63 | best_pred = checkpoint['best_pred'] 64 | errlist_train = checkpoint['errlist_train'] 65 | errlist_val = checkpoint['errlist_val'] 66 | model.load_state_dict(checkpoint['state_dict']) 67 | optimizer.load_state_dict(checkpoint['optimizer']) 68 | print("=> loaded checkpoint '{}' (epoch {})" 69 | .format(config.resume, checkpoint['epoch'])) 70 | else: 71 | print("=> no resume checkpoint found at '{}'". \ 72 | format(config.resume)) 73 | 74 | def train(epoch): 75 | model.train() 76 | global best_pred, errlist_train 77 | train_loss, correct, total = 0, 0, 0 78 | adjust_learning_rate(optimizer, config, epoch) 79 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 80 | pbar = tqdm(range(len(train_loader)), file=sys.stdout, 81 | bar_format=bar_format) 82 | data_loader = iter(train_loader) 83 | 84 | for batch_idx in pbar: 85 | data, target = data_loader.next() 86 | if config.cuda: 87 | data, target = data.cuda(), target.cuda() 88 | data, target = Variable(data), Variable(target) 89 | optimizer.zero_grad() 90 | output = model(data) 91 | loss = criterion(output, target) 92 | loss.backward() 93 | optimizer.step() 94 | 95 | train_loss += loss.item() 96 | pred = output.data.max(1)[1] 97 | correct += pred.eq(target.data).cpu().sum().numpy() 98 | total += target.size(0) 99 | err = 100 - 100. * correct / total 100 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 101 | + 'Iter: {}/{} '.format(batch_idx, len(train_loader)) \ 102 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 103 | (train_loss / (batch_idx + 1), 104 | err, total - correct, total) 105 | pbar.set_description(print_str) 106 | errlist_train += [err] 107 | 108 | def test(epoch): 109 | model.eval() 110 | global best_pred, errlist_train, errlist_val 111 | test_loss, correct, total = 0, 0, 0 112 | is_best = False 113 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 114 | pbar = tqdm(range(len(test_loader)), file=sys.stdout, 115 | bar_format=bar_format) 116 | data_loader = iter(test_loader) 117 | with torch.no_grad(): 118 | for batch_idx in pbar: 119 | data, target = data_loader.next() 120 | if config.cuda: 121 | data, target = data.cuda(), target.cuda() 122 | output = model(data) 123 | test_loss += criterion(output, target).item() 124 | # get the index of the max log-probability 125 | pred = output.data.max(1)[1] 126 | correct += pred.eq(target.data).cpu().sum().numpy() 127 | total += target.size(0) 128 | 129 | err = 100 - 100. * correct / total 130 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 131 | + 'Iter: {}/{} '.format(batch_idx, len(test_loader)) \ 132 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 133 | (test_loss / (batch_idx + 1), 134 | err, total - correct, total) 135 | pbar.set_description(print_str) 136 | 137 | if config.eval: 138 | print('Error rate is %.3f' % err) 139 | return 140 | # save checkpoint 141 | errlist_val += [err] 142 | if err < best_pred: 143 | best_pred = err 144 | is_best = True 145 | print('Best Accuracy: %.3f' % (100 - best_pred)) 146 | save_checkpoint({ 147 | 'epoch': epoch, 148 | 'state_dict': model.state_dict(), 149 | 'optimizer': optimizer.state_dict(), 150 | 'best_pred': best_pred, 151 | 'errlist_train': errlist_train, 152 | 'errlist_val': errlist_val, 153 | }, config, is_best=is_best) 154 | 155 | if config.eval: 156 | test(config.start_epoch) 157 | return 158 | 159 | for epoch in range(config.start_epoch, config.nepochs + 1): 160 | print('Epoch:', epoch) 161 | train(epoch) 162 | test(epoch) 163 | 164 | 165 | if __name__ == "__main__": 166 | main() 167 | -------------------------------------------------------------------------------- /experiments/gtos.dain.mobilenet/train.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import os 4 | import sys 5 | 6 | import torch 7 | import torch.nn as nn 8 | import torch.optim as optim 9 | from torch.autograd import Variable 10 | from tqdm import tqdm 11 | from config import config 12 | from utils.utils import save_checkpoint 13 | 14 | from dataloader.gtos import Dataloder 15 | from model.mobilenet_v2 import mobilenet_v2 16 | from network import DAIN 17 | 18 | # global variable 19 | best_pred = 100.0 20 | errlist_train = [] 21 | errlist_val = [] 22 | 23 | 24 | def adjust_learning_rate(optimizer, config, epoch): 25 | lr = config.lr * (0.1 ** ((epoch - 1) // config.lr_decay)) 26 | if (epoch - 1) % config.lr_decay == 0: 27 | print('LR is set to {}'.format(lr)) 28 | 29 | for param_group in optimizer.param_groups: 30 | param_group['lr'] = lr 31 | 32 | 33 | def main(): 34 | # init the config 35 | global best_pred, errlist_train, errlist_val 36 | config.cuda = config.cuda and torch.cuda.is_available() 37 | torch.manual_seed(config.seed) 38 | if config.cuda: 39 | os.environ["CUDA_VISIBLE_DEVICES"] = config.gpu 40 | torch.cuda.manual_seed(config.seed) 41 | # init dataloader 42 | classes, train_loader, test_loader = Dataloder(config).getloader() 43 | 44 | # init the model 45 | backbone1 = mobilenet_v2(True) 46 | backbone2 = mobilenet_v2(True) 47 | model = DAIN(len(classes), backbone1, backbone2) 48 | print(model) 49 | # criterion and optimizer 50 | criterion = nn.CrossEntropyLoss() 51 | optimizer = optim.SGD(model.parameters(), lr=config.lr, momentum=config.momentum, 52 | weight_decay=config.weight_decay) 53 | if config.cuda: 54 | model.cuda() 55 | # Please use CUDA_VISIBLE_DEVICES to control the number of gpus 56 | model = torch.nn.DataParallel(model) 57 | 58 | # check point 59 | if config.resume is not None: 60 | if os.path.isfile(config.resume): 61 | print("=> loading checkpoint '{}'".format(config.resume)) 62 | checkpoint = torch.load(config.resume) 63 | config.start_epoch = checkpoint['epoch'] + 1 64 | best_pred = checkpoint['best_pred'] 65 | errlist_train = checkpoint['errlist_train'] 66 | errlist_val = checkpoint['errlist_val'] 67 | model.load_state_dict(checkpoint['state_dict']) 68 | optimizer.load_state_dict(checkpoint['optimizer']) 69 | print("=> loaded checkpoint '{}' (epoch {})" 70 | .format(config.resume, checkpoint['epoch'])) 71 | else: 72 | print("=> no resume checkpoint found at '{}'". \ 73 | format(config.resume)) 74 | 75 | def train(epoch): 76 | model.train() 77 | global best_pred, errlist_train 78 | train_loss, correct, total = 0, 0, 0 79 | adjust_learning_rate(optimizer, config, epoch) 80 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 81 | pbar = tqdm(range(len(train_loader)), file=sys.stdout, 82 | bar_format=bar_format) 83 | data_loader = iter(train_loader) 84 | 85 | for batch_idx in pbar: 86 | img, diff_img, target = data_loader.next() 87 | if config.cuda: 88 | img, diff_img, target = img.cuda(), diff_img.cuda(), target.cuda() 89 | img, diff_img, target = Variable(img), Variable(diff_img), Variable(target) 90 | optimizer.zero_grad() 91 | output = model(img, diff_img) 92 | loss = criterion(output, target) 93 | loss.backward() 94 | optimizer.step() 95 | 96 | train_loss += loss.item() 97 | pred = output.data.max(1)[1] 98 | correct += pred.eq(target.data).cpu().sum().numpy() 99 | total += target.size(0) 100 | err = 100 - 100. * correct / total 101 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 102 | + 'Iter: {}/{} '.format(batch_idx, len(train_loader)) \ 103 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 104 | (train_loss / (batch_idx + 1), 105 | err, total - correct, total) 106 | pbar.set_description(print_str) 107 | errlist_train += [err] 108 | 109 | def test(epoch): 110 | model.eval() 111 | global best_pred, errlist_train, errlist_val 112 | test_loss, correct, total = 0, 0, 0 113 | is_best = False 114 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 115 | pbar = tqdm(range(len(test_loader)), file=sys.stdout, 116 | bar_format=bar_format) 117 | data_loader = iter(test_loader) 118 | with torch.no_grad(): 119 | for batch_idx in pbar: 120 | img, diff_img, target = data_loader.next() 121 | if config.cuda: 122 | img, diff_img, target = img.cuda(), diff_img.cuda(), target.cuda() 123 | output = model(img, diff_img) 124 | test_loss += criterion(output, target).item() 125 | # get the index of the max log-probability 126 | pred = output.data.max(1)[1] 127 | correct += pred.eq(target.data).cpu().sum().numpy() 128 | total += target.size(0) 129 | 130 | err = 100 - 100. * correct / total 131 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 132 | + 'Iter: {}/{} '.format(batch_idx, len(test_loader)) \ 133 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 134 | (test_loss / (batch_idx + 1), 135 | err, total - correct, total) 136 | pbar.set_description(print_str) 137 | 138 | if config.eval: 139 | print('Error rate is %.3f' % err) 140 | return 141 | # save checkpoint 142 | errlist_val += [err] 143 | if err < best_pred: 144 | best_pred = err 145 | is_best = True 146 | print('Best Accuracy: %.3f' % (100 - best_pred)) 147 | save_checkpoint({ 148 | 'epoch': epoch, 149 | 'state_dict': model.state_dict(), 150 | 'optimizer': optimizer.state_dict(), 151 | 'best_pred': best_pred, 152 | 'errlist_train': errlist_train, 153 | 'errlist_val': errlist_val, 154 | }, config, is_best=is_best) 155 | 156 | if config.eval: 157 | test(config.start_epoch) 158 | return 159 | 160 | for epoch in range(config.start_epoch, config.nepochs + 1): 161 | print('Epoch:', epoch) 162 | train(epoch) 163 | test(epoch) 164 | 165 | 166 | if __name__ == "__main__": 167 | main() 168 | -------------------------------------------------------------------------------- /experiments/gtos.tean.mobilenet/train.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import os 4 | import sys 5 | 6 | import torch 7 | import torch.nn as nn 8 | import torch.optim as optim 9 | from torch.autograd import Variable 10 | from tqdm import tqdm 11 | from config import config 12 | from utils.utils import save_checkpoint 13 | 14 | from dataloader.gtos import Dataloder 15 | from model.mobilenet_v2 import mobilenet_v2 16 | from network import TEAN 17 | 18 | # global variable 19 | best_pred = 100.0 20 | errlist_train = [] 21 | errlist_val = [] 22 | 23 | 24 | def adjust_learning_rate(optimizer, config, epoch): 25 | lr = config.lr * (0.1 ** ((epoch - 1) // config.lr_decay)) 26 | if (epoch - 1) % config.lr_decay == 0: 27 | print('LR is set to {}'.format(lr)) 28 | 29 | for param_group in optimizer.param_groups: 30 | param_group['lr'] = lr 31 | 32 | 33 | def main(): 34 | # init the config 35 | global best_pred, errlist_train, errlist_val 36 | config.cuda = config.cuda and torch.cuda.is_available() 37 | torch.manual_seed(config.seed) 38 | if config.cuda: 39 | os.environ["CUDA_VISIBLE_DEVICES"] = config.gpu 40 | torch.cuda.manual_seed(config.seed) 41 | # init dataloader 42 | classes, train_loader, test_loader = Dataloder(config).getloader() 43 | 44 | # init the model 45 | backbone1 = mobilenet_v2(True) 46 | backbone2 = mobilenet_v2(True) 47 | model = TEAN(len(classes), backbone1, backbone2) 48 | print(model) 49 | # criterion and optimizer 50 | criterion = nn.CrossEntropyLoss() 51 | optimizer = optim.SGD(model.parameters(), lr=config.lr, momentum=config.momentum, 52 | weight_decay=config.weight_decay) 53 | if config.cuda: 54 | model.cuda() 55 | # Please use CUDA_VISIBLE_DEVICES to control the number of gpus 56 | model = torch.nn.DataParallel(model) 57 | 58 | # check point 59 | if config.resume is not None: 60 | if os.path.isfile(config.resume): 61 | print("=> loading checkpoint '{}'".format(config.resume)) 62 | checkpoint = torch.load(config.resume) 63 | config.start_epoch = checkpoint['epoch'] + 1 64 | best_pred = checkpoint['best_pred'] 65 | errlist_train = checkpoint['errlist_train'] 66 | errlist_val = checkpoint['errlist_val'] 67 | model.load_state_dict(checkpoint['state_dict']) 68 | optimizer.load_state_dict(checkpoint['optimizer']) 69 | print("=> loaded checkpoint '{}' (epoch {})" 70 | .format(config.resume, checkpoint['epoch'])) 71 | else: 72 | print("=> no resume checkpoint found at '{}'". \ 73 | format(config.resume)) 74 | 75 | def train(epoch): 76 | model.train() 77 | global best_pred, errlist_train 78 | train_loss, correct, total = 0, 0, 0 79 | adjust_learning_rate(optimizer, config, epoch) 80 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 81 | pbar = tqdm(range(len(train_loader)), file=sys.stdout, 82 | bar_format=bar_format) 83 | data_loader = iter(train_loader) 84 | 85 | for batch_idx in pbar: 86 | img, diff_img, target = data_loader.next() 87 | if config.cuda: 88 | img, diff_img, target = img.cuda(), diff_img.cuda(), target.cuda() 89 | img, diff_img, target = Variable(img), Variable(diff_img), Variable(target) 90 | optimizer.zero_grad() 91 | output = model(img, diff_img) 92 | loss = criterion(output, target) 93 | loss.backward() 94 | optimizer.step() 95 | 96 | train_loss += loss.item() 97 | pred = output.data.max(1)[1] 98 | correct += pred.eq(target.data).cpu().sum().numpy() 99 | total += target.size(0) 100 | err = 100 - 100. * correct / total 101 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 102 | + 'Iter: {}/{} '.format(batch_idx, len(train_loader)) \ 103 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 104 | (train_loss / (batch_idx + 1), 105 | err, total - correct, total) 106 | pbar.set_description(print_str) 107 | errlist_train += [err] 108 | 109 | def test(epoch): 110 | model.eval() 111 | global best_pred, errlist_train, errlist_val 112 | test_loss, correct, total = 0, 0, 0 113 | is_best = False 114 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 115 | pbar = tqdm(range(len(test_loader)), file=sys.stdout, 116 | bar_format=bar_format) 117 | data_loader = iter(test_loader) 118 | with torch.no_grad(): 119 | for batch_idx in pbar: 120 | img, diff_img, target = data_loader.next() 121 | if config.cuda: 122 | img, diff_img, target = img.cuda(), diff_img.cuda(), target.cuda() 123 | output = model(img, diff_img) 124 | test_loss += criterion(output, target).item() 125 | # get the index of the max log-probability 126 | pred = output.data.max(1)[1] 127 | correct += pred.eq(target.data).cpu().sum().numpy() 128 | total += target.size(0) 129 | 130 | err = 100 - 100. * correct / total 131 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 132 | + 'Iter: {}/{} '.format(batch_idx, len(test_loader)) \ 133 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 134 | (test_loss / (batch_idx + 1), 135 | err, total - correct, total) 136 | pbar.set_description(print_str) 137 | 138 | if config.eval: 139 | print('Error rate is %.3f' % err) 140 | return 141 | # save checkpoint 142 | errlist_val += [err] 143 | if err < best_pred: 144 | best_pred = err 145 | is_best = True 146 | print('Best Accuracy: %.3f' % (100 - best_pred)) 147 | save_checkpoint({ 148 | 'epoch': epoch, 149 | 'state_dict': model.state_dict(), 150 | 'optimizer': optimizer.state_dict(), 151 | 'best_pred': best_pred, 152 | 'errlist_train': errlist_train, 153 | 'errlist_val': errlist_val, 154 | }, config, is_best=is_best) 155 | 156 | if config.eval: 157 | test(config.start_epoch) 158 | return 159 | 160 | for epoch in range(config.start_epoch, config.nepochs + 1): 161 | print('Epoch:', epoch) 162 | train(epoch) 163 | test(epoch) 164 | 165 | 166 | if __name__ == "__main__": 167 | main() 168 | -------------------------------------------------------------------------------- /experiments/gtos.dain.resnet/train.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import os 4 | import sys 5 | 6 | import torch 7 | import torch.nn as nn 8 | import torch.optim as optim 9 | from torch.autograd import Variable 10 | from torchvision.models import resnet 11 | from tqdm import tqdm 12 | from config import config 13 | from utils.utils import save_checkpoint 14 | 15 | from dataloader.gtos import Dataloder 16 | from network import DAIN 17 | 18 | # global variable 19 | best_pred = 100.0 20 | errlist_train = [] 21 | errlist_val = [] 22 | 23 | 24 | def adjust_learning_rate(optimizer, config, epoch): 25 | lr = config.lr * (0.1 ** ((epoch - 1) // config.lr_decay)) 26 | if (epoch - 1) % config.lr_decay == 0: 27 | print('LR is set to {}'.format(lr)) 28 | 29 | for param_group in optimizer.param_groups: 30 | param_group['lr'] = lr 31 | 32 | 33 | def main(): 34 | # init the config 35 | global best_pred, errlist_train, errlist_val 36 | config.cuda = config.cuda and torch.cuda.is_available() 37 | torch.manual_seed(config.seed) 38 | if config.cuda: 39 | os.environ["CUDA_VISIBLE_DEVICES"] = config.gpu 40 | torch.cuda.manual_seed(config.seed) 41 | # init dataloader 42 | classes, train_loader, test_loader = Dataloder(config).getloader() 43 | 44 | # init the model 45 | backbone1 = resnet.resnet18(pretrained=True) 46 | backbone2 = resnet.resnet18(pretrained=True) 47 | model = DAIN(len(classes), backbone1, backbone2) 48 | print(model) 49 | # criterion and optimizer 50 | criterion = nn.CrossEntropyLoss() 51 | optimizer = optim.SGD(model.parameters(), lr=config.lr, momentum=config.momentum, 52 | weight_decay=config.weight_decay) 53 | if config.cuda: 54 | model.cuda() 55 | # Please use CUDA_VISIBLE_DEVICES to control the number of gpus 56 | model = torch.nn.DataParallel(model) 57 | 58 | # check point 59 | if config.resume is not None: 60 | if os.path.isfile(config.resume): 61 | print("=> loading checkpoint '{}'".format(config.resume)) 62 | checkpoint = torch.load(config.resume) 63 | config.start_epoch = checkpoint['epoch'] + 1 64 | best_pred = checkpoint['best_pred'] 65 | errlist_train = checkpoint['errlist_train'] 66 | errlist_val = checkpoint['errlist_val'] 67 | model.load_state_dict(checkpoint['state_dict']) 68 | optimizer.load_state_dict(checkpoint['optimizer']) 69 | print("=> loaded checkpoint '{}' (epoch {})" 70 | .format(config.resume, checkpoint['epoch'])) 71 | else: 72 | print("=> no resume checkpoint found at '{}'". \ 73 | format(config.resume)) 74 | 75 | def train(epoch): 76 | model.train() 77 | global best_pred, errlist_train 78 | train_loss, correct, total = 0, 0, 0 79 | adjust_learning_rate(optimizer, config, epoch) 80 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 81 | pbar = tqdm(range(len(train_loader)), file=sys.stdout, 82 | bar_format=bar_format) 83 | data_loader = iter(train_loader) 84 | 85 | for batch_idx in pbar: 86 | img, diff_img, target = data_loader.next() 87 | if config.cuda: 88 | img, diff_img, target = img.cuda(), diff_img.cuda(), target.cuda() 89 | img, diff_img, target = Variable(img), Variable(diff_img), Variable(target) 90 | optimizer.zero_grad() 91 | output = model(img, diff_img) 92 | loss = criterion(output, target) 93 | loss.backward() 94 | optimizer.step() 95 | 96 | train_loss += loss.item() 97 | pred = output.data.max(1)[1] 98 | correct += pred.eq(target.data).cpu().sum().numpy() 99 | total += target.size(0) 100 | err = 100 - 100. * correct / total 101 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 102 | + 'Iter: {}/{} '.format(batch_idx, len(train_loader)) \ 103 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 104 | (train_loss / (batch_idx + 1), 105 | err, total - correct, total) 106 | pbar.set_description(print_str) 107 | errlist_train += [err] 108 | 109 | def test(epoch): 110 | model.eval() 111 | global best_pred, errlist_train, errlist_val 112 | test_loss, correct, total = 0, 0, 0 113 | is_best = False 114 | bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' 115 | pbar = tqdm(range(len(test_loader)), file=sys.stdout, 116 | bar_format=bar_format) 117 | data_loader = iter(test_loader) 118 | with torch.no_grad(): 119 | for batch_idx in pbar: 120 | img, diff_img, target = data_loader.next() 121 | if config.cuda: 122 | img, diff_img, target = img.cuda(), diff_img.cuda(), target.cuda() 123 | output = model(img, diff_img) 124 | test_loss += criterion(output, target).item() 125 | # get the index of the max log-probability 126 | pred = output.data.max(1)[1] 127 | correct += pred.eq(target.data).cpu().sum().numpy() 128 | total += target.size(0) 129 | 130 | err = 100 - 100. * correct / total 131 | print_str = 'Epoch: {}/{} '.format(epoch, config.nepochs) \ 132 | + 'Iter: {}/{} '.format(batch_idx, len(test_loader)) \ 133 | + 'Loss: %.3f | Err: %.3f%% (%d/%d)' % \ 134 | (test_loss / (batch_idx + 1), 135 | err, total - correct, total) 136 | pbar.set_description(print_str) 137 | 138 | if config.eval: 139 | print('Error rate is %.3f' % err) 140 | return 141 | # save checkpoint 142 | errlist_val += [err] 143 | if err < best_pred: 144 | best_pred = err 145 | is_best = True 146 | print('Best Accuracy: %.3f' % (100 - best_pred)) 147 | save_checkpoint({ 148 | 'epoch': epoch, 149 | 'state_dict': model.state_dict(), 150 | 'optimizer': optimizer.state_dict(), 151 | 'best_pred': best_pred, 152 | 'errlist_train': errlist_train, 153 | 'errlist_val': errlist_val, 154 | }, config, is_best=is_best) 155 | 156 | if config.eval: 157 | test(config.start_epoch) 158 | return 159 | 160 | for epoch in range(config.start_epoch, config.nepochs + 1): 161 | print('Epoch:', epoch) 162 | train(epoch) 163 | test(epoch) 164 | 165 | 166 | if __name__ == "__main__": 167 | main() 168 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------