├── mhbn.png
├── __pycache__
    ├── mhbn.cpython-36.pyc
    ├── mvcnn.cpython-36.pyc
    ├── util.cpython-36.pyc
    └── custom_dataset.cpython-36.pyc
├── util.py
├── README.md
├── custom_dataset.py
├── mhbn.py
├── main.py
├── resnet.py
└── main.ipynb


/mhbn.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/liyuan24/MHBNN-PyTorch/HEAD/mhbn.png


--------------------------------------------------------------------------------
/__pycache__/mhbn.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/liyuan24/MHBNN-PyTorch/HEAD/__pycache__/mhbn.cpython-36.pyc


--------------------------------------------------------------------------------
/__pycache__/mvcnn.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/liyuan24/MHBNN-PyTorch/HEAD/__pycache__/mvcnn.cpython-36.pyc


--------------------------------------------------------------------------------
/__pycache__/util.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/liyuan24/MHBNN-PyTorch/HEAD/__pycache__/util.cpython-36.pyc


--------------------------------------------------------------------------------
/__pycache__/custom_dataset.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/liyuan24/MHBNN-PyTorch/HEAD/__pycache__/custom_dataset.cpython-36.pyc


--------------------------------------------------------------------------------
/util.py:
--------------------------------------------------------------------------------
 1 | import torch
 2 | import os
 3 | 
 4 | def logEpoch(logger, model, epoch, loss, accuracy):
 5 |     # 1. Log scalar values (scalar summary)
 6 |     info = {'loss': loss.item(), 'accuracy': accuracy.item()}
 7 | 
 8 |     for tag, value in info.items():
 9 |         logger.scalar_summary(tag, value, epoch)
10 | 
11 |     # 2. Log values and gradients of the parameters (histogram summary)
12 |     for tag, value in model.named_parameters():
13 |         tag = tag.replace('.', '/')
14 |         logger.histo_summary(tag, value.data.cpu().numpy(), epoch)
15 |         logger.histo_summary(tag + '/grad', value.grad.data.cpu().numpy(), epoch)
16 | 
17 |     # 3. Log training images (image summary)
18 |     #info = {'images': images.view(-1, 28, 28)[:10].cpu().numpy()}
19 | 
20 |     #for tag, images in info.items():
21 |         #logger.image_summary(tag, images, epoch)
22 | 
23 | def save_checkpoint(state, model, resnet=None, checkpoint='checkpoint', filename='checkpoint.pth.tar'):
24 |     if resnet:
25 |         filepath = os.path.join(checkpoint, model + str(resnet) + '_' + filename)
26 |     else:
27 |         filepath = os.path.join(checkpoint, model + '_' + filename)
28 |     torch.save(state, filepath)
29 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # MHBNN-PyTorch
 2 | 
 3 | A Pytorch implementation of Multi-view Harmonized Bilinear Netowrk for 3D Object Detection(MHBN) inpsired by [Tan Yu et al](http://openaccess.thecvf.com/content_cvpr_2018/html/Yu_Multi-View_Harmonized_Bilinear_CVPR_2018_paper.html). 
 4 | 
 5 | In this paper, the 3D object recognition problem is converted to multi-view 2D image classification problem. For each 3D object, there are multiple images taken from different views. 
 6 | 
 7 | ![](https://github.com/LiyuanLacfo/MHBNN-PyTorch/blob/master/mhbn.png)
 8 | 
 9 | ### Dependecies
10 | 
11 | * torch 0.4.1
12 | * torchvision
13 | * numpy
14 | 
15 | ### Dataset
16 | 
17 | * ModelNet CAD data can be found at [Princeton](http://modelnet.cs.princeton.edu/)
18 | * ModelNet40 12-view png images can be downloaded at [google drive](https://drive.google.com/file/d/0B4v2jR3WsindMUE3N2xiLVpyLW8/view?usp=sharing)
19 | * You can also create your own png dataset with [Blend](https://github.com/WeiTang114/BlenderPhong)
20 | 
21 | ### Train the model
22 | 
23 | ```
24 | python main.py --data <path to your png data>
25 | ```
26 | 
27 | ### Special Thanks
28 | 
29 | I refered to [RBirkeland](https://github.com/RBirkeland/MVCNN-PyTorch) for some code.
30 | 
31 | 


--------------------------------------------------------------------------------
/custom_dataset.py:
--------------------------------------------------------------------------------
 1 | from torch.utils.data.dataset import Dataset
 2 | import os
 3 | from PIL import Image
 4 | 
 5 | class MultiViewDataSet(Dataset):
 6 | 
 7 |     def find_classes(self, dir):
 8 |         classes = [d for d in os.listdir(dir) if os.path.isdir(os.path.join(dir, d))]
 9 |         classes.sort()
10 |         class_to_idx = {classes[i]: i for i in range(len(classes))}
11 | 
12 |         return classes, class_to_idx
13 | 
14 |     def __init__(self, root, data_type, transform=None, target_transform=None):
15 |         self.x = []
16 |         self.y = []
17 |         self.root = root
18 | 
19 |         self.classes, self.class_to_idx = self.find_classes(root)
20 | 
21 |         self.transform = transform
22 |         self.target_transform = target_transform
23 | 
24 |         # root / <label>  / <train/test> / <item> / <view>.png
25 |         for label in os.listdir(root): # Label
26 |             for item in os.listdir(root + '/' + label + '/' + data_type):
27 |                 views = []
28 |                 for view in os.listdir(root + '/' + label + '/' + data_type + '/' + item):
29 |                     views.append(root + '/' + label + '/' + data_type + '/' + item + '/' + view)
30 | 
31 |                 self.x.append(views)
32 |                 self.y.append(self.class_to_idx[label])
33 | 
34 |     # Override to give PyTorch access to any image on the dataset
35 |     def __getitem__(self, index):
36 |         orginal_views = self.x[index]
37 |         views = []
38 | 
39 |         for view in orginal_views:
40 |             im = Image.open(view)
41 |             im = im.convert('RGB')
42 |             if self.transform is not None:
43 |                 im = self.transform(im)
44 |             views.append(im)
45 | 
46 |         return views, self.y[index]
47 | 
48 |     # Override to give PyTorch size of dataset
49 |     def __len__(self):
50 |         return len(self.x)
51 | 


--------------------------------------------------------------------------------
/mhbn.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | import torch.nn as nn
  3 | import torch.utils.model_zoo as model_zoo
  4 | from torch.autograd import Variable
  5 | 
  6 | __all__ = ['MHBNN', 'mhbnn'] #can only import MHBNN class and mhbnn function
  7 | 
  8 | model_urls = {
  9 |     'alexnet': 'https://download.pytorch.org/models/alexnet-owt-4df8aa71.pth',
 10 | }
 11 | 
 12 | 
 13 | class MHBNN(nn.Module):
 14 | 
 15 |     def __init__(self, num_classes, num_local_features):
 16 |         super(MHBNN, self).__init__()
 17 |         self.features = nn.Sequential(
 18 |             nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2),
 19 |             nn.ReLU(inplace=True),
 20 |             nn.MaxPool2d(kernel_size=3, stride=2),
 21 |             nn.Conv2d(64, 192, kernel_size=5, padding=2),
 22 |             nn.ReLU(inplace=True),
 23 |             nn.MaxPool2d(kernel_size=3, stride=2),
 24 |             nn.Conv2d(192, 384, kernel_size=3, padding=1),
 25 |             nn.ReLU(inplace=True),
 26 |             nn.Conv2d(384, 256, kernel_size=3, padding=1),
 27 |             nn.ReLU(inplace=True),
 28 |             nn.Conv2d(256, 256, kernel_size=3, padding=1),
 29 |             nn.ReLU(inplace=True),
 30 |             nn.MaxPool2d(kernel_size=3, stride=2),
 31 |         )
 32 |         #number of local features, d in original paper
 33 |         self.one_by_one_conv_output = num_local_features
 34 |         #nn.Parameter to register the parameter in model.parameters() to make them trainable
 35 |         #if Variable, not trainable
 36 |         self.lambdas = nn.Parameter(torch.ones(self.one_by_one_conv_output)*0.5, requires_grad=True)
 37 |         self.classifier = nn.Sequential(
 38 |             nn.Dropout(),
 39 |             nn.Linear(self.one_by_one_conv_output**2, 4096),
 40 |             nn.ReLU(inplace=True),
 41 |             nn.Dropout(),
 42 |             nn.Linear(4096, 4096),
 43 |             nn.ReLU(inplace=True),
 44 |             nn.Linear(4096, num_classes),
 45 |         )
 46 |         self.one_by_one_conv = nn.Conv2d(256, self.one_by_one_conv_output, kernel_size=1)
 47 |         
 48 |         
 49 |     
 50 |     def sign_sqrt(self, x):
 51 |         x = torch.mul(torch.sign(x),torch.sqrt(torch.abs(x)+1e-12))
 52 |         return x
 53 |     
 54 |     
 55 |     def bilinear_pooling(self, x):
 56 |         '''
 57 |         x = [x1, x2, ..., xn], size d*N, d is number of features of each patch, N is number of patches of all views 
 58 |         return: d*d
 59 |         '''
 60 |         return torch.mm(x, x.t())
 61 |     
 62 |     def harmonize(self, s):
 63 |         '''
 64 |         s: (d, ) tensor, sigular values
 65 |         return: (d, ) tensor after harmonized by box-cox transform
 66 |         '''
 67 |         harmonized_s = torch.zeros_like(s)
 68 |         n = s.size(0)
 69 |         for i in range(n):
 70 |             if torch.abs(self.lambdas[i]) > 1e-12:
 71 |                 harmonized_s[i] = (s[i].pow(self.lambdas[i]) - 1) / self.lambdas[i]
 72 |             else:
 73 |                 harmonized_s[i] = torch.log(s[i])
 74 |         return harmonized_s
 75 | 
 76 | 
 77 |     def forward(self, x):
 78 |         x = x.transpose(0, 1)
 79 |         
 80 |         view_pool = []
 81 |         
 82 |         for v in x:
 83 |             v = self.features(v)
 84 |             v = self.sign_sqrt(v) #early sqrt sublayer
 85 |             v = self.one_by_one_conv(v) #conv sublayer
 86 |             view_pool.append(v)
 87 |         y = torch.stack(view_pool)
 88 |         y = y.transpose(0, 1)
 89 |         res = []
 90 |         #each member in batch
 91 |         for b in y:
 92 |             #[num_local_features, views, w, h]
 93 |             b = b.transpose(0, 1).contiguous()
 94 |             #[num_local_features, views*w*h]
 95 |             b = b.view(b.size(0), -1)
 96 |             #[num_local_features, num_local_features]
 97 |             b = self.bilinear_pooling(b) #bininear pooling
 98 |             u, s, v = torch.svd(b)
 99 |             harmonized_s = self.harmonize(s) #harmonize singular values
100 |             b = torch.mm(torch.mm(u, torch.diag(harmonized_s)), v.t())
101 |             #[num_local_features*num_local_features]
102 |             b = b.view(-1) #vectorized
103 |             b = self.sign_sqrt(b) #late sqrt layer
104 |             b = b / (torch.norm(b, 2)+(1e-8)) #l2 norm sub-layer
105 |             res.append(b)
106 | 
107 |         pooled_view = torch.stack(res) #assembly into batch, [batch_size, num_local_features**2]
108 |         
109 |         pooled_view = self.classifier(pooled_view)
110 |         return pooled_view
111 | 
112 | 
113 | def mhbnn(pretrained=False, num_classes=40, num_local_features = 10):
114 |     r"""
115 |     Args:
116 |         pretrained (bool): If True, returns a model pre-trained on ImageNet
117 |     """
118 |     model = MHBNN(num_classes, num_local_features)
119 |     if pretrained:
120 |         pretrained_dict = model_zoo.load_url(model_urls['alexnet'])
121 |         model_dict = model.state_dict()
122 |         # 1. filter out unnecessary keys
123 |         pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict and v.shape == model_dict[k].shape}
124 |         # 2. overwrite entries in the existing state dict
125 |         model_dict.update(pretrained_dict)
126 |         # 3. load the new state dict
127 |         model.load_state_dict(model_dict)
128 |     return model


--------------------------------------------------------------------------------
/main.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import os
  3 | import torch
  4 | import torch.nn as nn
  5 | import torch.backends.cudnn as cudnn
  6 | from torch.utils.data import DataLoader
  7 | from torch.autograd import Variable
  8 | import torchvision.transforms as transforms
  9 | from custom_dataset import *
 10 | from mhbn import *
 11 | import time
 12 | import util
 13 | import argparse
 14 | 
 15 | 
 16 | transform = transforms.Compose([
 17 |     transforms.CenterCrop(500),
 18 |     transforms.Resize(224),
 19 |     transforms.ToTensor(),
 20 |     transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
 21 | ])
 22 | 
 23 | 
 24 | 
 25 | #cpu or gpu
 26 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 27 | 
 28 | parser = argparse.ArgumentParser(description='MHBNN-PyTorch')
 29 | parser.add_argument('--data', metavar='DIR', help='path to dataset')
 30 | parser.add_argument('--epochs', default=100, type=int, metavar='N', help='number of total epochs to run (default: 100)')
 31 | parser.add_argument('-nlocal' ,'--n-local-features', default=10, type=int, metavar='N', help='number of local features(default: 10)')
 32 | parser.add_argument('-b', '--batch-size', default=4, type=int,
 33 |                     metavar='N', help='mini-batch size (default: 4)')
 34 | parser.add_argument('--lr', '--learning-rate', default=0.0001, type=float,
 35 |                     metavar='LR', help='initial learning rate (default: 0.0001)')
 36 | parser.add_argument('--lr-decay-freq', default=30, type=float,
 37 |                     metavar='W', help='learning rate decay (default: 30)')
 38 | parser.add_argument('--lr-decay', default=0.1, type=float,
 39 |                     metavar='W', help='learning rate decay (default: 0.1)')
 40 | parser.add_argument('--print-freq', '-p', default=10, type=int,
 41 |                     metavar='N', help='print frequency (default: 10)')
 42 | parser.add_argument('-r', '--resume', default='', type=str, metavar='PATH',
 43 |                     help='path to latest checkpoint (default: none)')
 44 | parser.add_argument('--pretrained', dest='pretrained', action='store_true', help='use pre-trained model')
 45 | 
 46 | args = parser.parse_args()
 47 | 
 48 | 
 49 | #root path of data
 50 | root = args.data
 51 | 
 52 | 
 53 | #the batch size of train and validation
 54 | batch_size = args.batch_size
 55 | 
 56 | 
 57 | # Load dataset
 58 | dset_train = MultiViewDataSet(root, 'train', transform=transform)
 59 | train_loader = DataLoader(dset_train, batch_size=batch_size, shuffle=True, num_workers=2)
 60 | 
 61 | dset_val = MultiViewDataSet(root, 'test', transform=transform)
 62 | val_loader = DataLoader(dset_val, batch_size=batch_size, shuffle=True, num_workers=2)
 63 | 
 64 | 
 65 | classes, class_to_idx = dset_train.classes, dset_train.class_to_idx
 66 | 
 67 | 
 68 | #number of local features
 69 | num_local_features = args.n_local_features
 70 | 
 71 | 
 72 | model = mhbnn(True, len(classes), num_local_features)
 73 | model.to(device)
 74 | cudnn.benchmark = True
 75 | print('Running on ' + str(device))
 76 | 
 77 | 
 78 | lr = args.lr
 79 | lr_decay = args.lr_decay
 80 | lr_decay_freq = args.lr_decay_freq
 81 | n_epochs = args.epochs
 82 | criterion = nn.CrossEntropyLoss()
 83 | optimizer = torch.optim.Adam(model.parameters(), lr=lr)
 84 | 
 85 | best_acc = 0.0
 86 | best_loss = 0.0
 87 | start_epoch = 0
 88 | print_freq = args.print_freq
 89 | 
 90 | 
 91 | resume_path = './checkpoint'
 92 | resume_ckp_path = args.resume
 93 | 
 94 | 
 95 | 
 96 | def load_checkpoint():
 97 |     global best_acc, start_epoch
 98 |     # Load checkpoint.
 99 |     print('\n==> Loading checkpoint..')
100 |     assert os.path.isfile(resume_ckp_path), 'Error: no checkpoint file found!'
101 | 
102 |     checkpoint = torch.load(resume_ckp_path)
103 |     best_acc = checkpoint['best_acc']
104 |     start_epoch = checkpoint['epoch']
105 |     model.load_state_dict(checkpoint['state_dict'])
106 |     optimizer.load_state_dict(checkpoint['optimizer'])
107 | 
108 | 
109 | 
110 | def train():
111 |     train_size = len(train_loader)
112 | 
113 |     for i, (inputs, targets) in enumerate(train_loader):
114 |         # Convert from list of 3D to 4D
115 |         #convert to shape [batch_size, views, channels, width, height]
116 |         inputs = np.stack(inputs, axis=1)
117 | 
118 |         inputs = torch.from_numpy(inputs)
119 | 
120 |         inputs, targets = inputs.cuda(device), targets.cuda(device)
121 |         inputs, targets = Variable(inputs), Variable(targets)
122 | 
123 |         # compute output
124 |         outputs = model(inputs)
125 |         loss = criterion(outputs, targets)
126 | 
127 |         # compute gradient and do SGD step
128 |         optimizer.zero_grad()
129 |         loss.backward()
130 |         optimizer.step()
131 | 
132 |         if (i + 1) %  print_freq == 0:
133 |             print("\tIter [%d/%d] Loss: %.4f" % (i + 1, train_size, loss.item()))
134 | 
135 | 
136 | def eval(data_loader, is_test=False):
137 |     if is_test:
138 |        load_checkpoint()
139 | 
140 |     # Eval
141 |     total = 0.0
142 |     correct = 0.0
143 | 
144 |     total_loss = 0.0
145 |     n = 0
146 | 
147 |     for i, (inputs, targets) in enumerate(data_loader):
148 |         with torch.no_grad():
149 |             # Convert from list of 3D to 4D
150 |             inputs = np.stack(inputs, axis=1)
151 | 
152 |             inputs = torch.from_numpy(inputs)
153 | 
154 |             inputs, targets = inputs.cuda(device), targets.cuda(device)
155 |             inputs, targets = Variable(inputs), Variable(targets)
156 | 
157 |             # compute output
158 |             outputs = model(inputs)
159 |             loss = criterion(outputs, targets)
160 | 
161 |             total_loss += loss
162 |             n += 1
163 | 
164 |             _, predicted = torch.max(outputs.data, 1)
165 |             total += targets.size(0)
166 |             correct += (predicted.cpu() == targets.cpu()).sum()
167 |     avg_test_acc = 100 * correct / total
168 |     avg_loss = total_loss / n
169 | 
170 |     return avg_test_acc, avg_loss
171 | 
172 | 
173 | if resume_ckp_path:
174 |     load_checkpoint()
175 | 
176 | 
177 | 
178 | for epoch in range(start_epoch, n_epochs):
179 |     print('\n-----------------------------------')
180 |     print('Epoch: [%d/%d]' % (epoch+1, n_epochs))
181 |     start = time.time()
182 | 
183 |     model.train()
184 |     train()
185 |     print('Time taken: %.2f sec.' % (time.time() - start))
186 | 
187 |     model.eval()
188 |     avg_test_acc, avg_loss = eval(val_loader)
189 | 
190 |     print('\nEvaluation:')
191 |     print('\tVal Acc: %.2f - Loss: %.4f' % (avg_test_acc.item(), avg_loss.item()))
192 |     print('\tCurrent best  val acc: %.2f' % best_acc)
193 |     
194 | 
195 |     # Log epoch to tensorboard
196 |     # See log using: tensorboard --logdir='logs' --port=6006
197 | #     util.logEpoch(logger, model, epoch + 1, avg_loss, avg_test_acc)
198 | 
199 |     # Save model
200 |     if avg_test_acc > best_acc:
201 |         print('\tSaving checkpoint - Acc: %.2f' % avg_test_acc)
202 |         best_acc = avg_test_acc
203 |         best_loss = avg_loss
204 |         util.save_checkpoint({
205 |             'epoch': epoch + 1,
206 |             'state_dict': model.state_dict(),
207 |             'acc': avg_test_acc,
208 |             'best_acc': best_acc,
209 |             'optimizer': optimizer.state_dict(),
210 |         }, 'mhbnn_'+str(num_local_features))
211 | 
212 |     # Decaying Learning Rate
213 |     if (epoch + 1) % lr_decay_freq == 0:
214 |         lr *= lr_decay
215 |         optimizer = torch.optim.Adam(model.parameters(), lr=lr)
216 |         print('Learning rate:', lr)
217 | 
218 | 


--------------------------------------------------------------------------------
/resnet.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | import torch.nn as nn
  3 | import math
  4 | import torch.utils.model_zoo as model_zoo
  5 | 
  6 | 
  7 | __all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101',
  8 |            'resnet152']
  9 | 
 10 | 
 11 | model_urls = {
 12 |     'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
 13 |     'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
 14 |     'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
 15 |     'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
 16 |     'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
 17 | }
 18 | 
 19 | 
 20 | def conv3x3(in_planes, out_planes, stride=1):
 21 |     """3x3 convolution with padding"""
 22 |     return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
 23 |                      padding=1, bias=False)
 24 | 
 25 | 
 26 | class BasicBlock(nn.Module):
 27 |     expansion = 1
 28 | 
 29 |     def __init__(self, inplanes, planes, stride=1, downsample=None):
 30 |         super(BasicBlock, self).__init__()
 31 |         self.conv1 = conv3x3(inplanes, planes, stride)
 32 |         self.bn1 = nn.BatchNorm2d(planes)
 33 |         self.relu = nn.ReLU(inplace=True)
 34 |         self.conv2 = conv3x3(planes, planes)
 35 |         self.bn2 = nn.BatchNorm2d(planes)
 36 |         self.downsample = downsample
 37 |         self.stride = stride
 38 | 
 39 |     def forward(self, x):
 40 |         residual = x
 41 | 
 42 |         out = self.conv1(x)
 43 |         out = self.bn1(out)
 44 |         out = self.relu(out)
 45 | 
 46 |         out = self.conv2(out)
 47 |         out = self.bn2(out)
 48 | 
 49 |         if self.downsample is not None:
 50 |             residual = self.downsample(x)
 51 | 
 52 |         out += residual
 53 |         out = self.relu(out)
 54 | 
 55 |         return out
 56 | 
 57 | 
 58 | class Bottleneck(nn.Module):
 59 |     expansion = 4
 60 | 
 61 |     def __init__(self, inplanes, planes, stride=1, downsample=None):
 62 |         super(Bottleneck, self).__init__()
 63 |         self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
 64 |         self.bn1 = nn.BatchNorm2d(planes)
 65 |         self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
 66 |                                padding=1, bias=False)
 67 |         self.bn2 = nn.BatchNorm2d(planes)
 68 |         self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
 69 |         self.bn3 = nn.BatchNorm2d(planes * 4)
 70 |         self.relu = nn.ReLU(inplace=True)
 71 |         self.downsample = downsample
 72 |         self.stride = stride
 73 | 
 74 |     def forward(self, x):
 75 |         residual = x
 76 | 
 77 |         out = self.conv1(x)
 78 |         out = self.bn1(out)
 79 |         out = self.relu(out)
 80 | 
 81 |         out = self.conv2(out)
 82 |         out = self.bn2(out)
 83 |         out = self.relu(out)
 84 | 
 85 |         out = self.conv3(out)
 86 |         out = self.bn3(out)
 87 | 
 88 |         if self.downsample is not None:
 89 |             residual = self.downsample(x)
 90 | 
 91 |         out += residual
 92 |         out = self.relu(out)
 93 | 
 94 |         return out
 95 | 
 96 | 
 97 | class ResNet(nn.Module):
 98 | 
 99 |     def __init__(self, block, layers, num_classes=3):
100 |         self.inplanes = 64
101 |         super(ResNet, self).__init__()
102 |         self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False)
103 |         self.bn1 = nn.BatchNorm2d(64)
104 |         self.relu = nn.ReLU(inplace=True)
105 |         self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
106 |         self.layer1 = self._make_layer(block, 64, layers[0])
107 |         self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
108 |         self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
109 |         self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
110 |         self.avgpool = nn.AvgPool2d(7, stride=1) #nn.AdaptiveAvgPool2d(7)
111 |         self.fc = nn.Linear(512 * block.expansion, num_classes)
112 | 
113 |         for m in self.modules():
114 |             if isinstance(m, nn.Conv2d):
115 |                 nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
116 |             elif isinstance(m, nn.BatchNorm2d):
117 |                 nn.init.constant_(m.weight, 1)
118 |                 nn.init.constant_(m.bias, 0)
119 | 
120 |     def _make_layer(self, block, planes, blocks, stride=1):
121 |         downsample = None
122 |         if stride != 1 or self.inplanes != planes * block.expansion:
123 |             downsample = nn.Sequential(
124 |                 nn.Conv2d(self.inplanes, planes * block.expansion,
125 |                           kernel_size=1, stride=stride, bias=False),
126 |                 nn.BatchNorm2d(planes * block.expansion),
127 |             )
128 | 
129 |         layers = []
130 |         layers.append(block(self.inplanes, planes, stride, downsample))
131 |         self.inplanes = planes * block.expansion
132 |         for i in range(1, blocks):
133 |             layers.append(block(self.inplanes, planes))
134 | 
135 |         return nn.Sequential(*layers)
136 | 
137 |     def forward(self, x):
138 |         # Swap batch and views dims
139 |         x = x.transpose(0, 1)
140 |         #print(x.shape)
141 | 
142 |         # View pool
143 |         view_pool = []
144 | 
145 |         for v in x:
146 |             v = self.conv1(v)
147 |             v = self.bn1(v)
148 |             v = self.relu(v)
149 |             v = self.maxpool(v)
150 | 
151 |             v = self.layer1(v)
152 |             v = self.layer2(v)
153 |             v = self.layer3(v)
154 |             v = self.layer4(v)
155 | 
156 |             v = self.avgpool(v)
157 |             v = v.view(v.size(0), -1)
158 | 
159 |             view_pool.append(v)
160 | 
161 |         pooled_view = view_pool[0]
162 |         for i in range(1, len(view_pool)):
163 |             pooled_view = torch.max(pooled_view, view_pool[i])
164 | 
165 |         pooled_view = self.fc(pooled_view)
166 | 
167 |         return pooled_view
168 | 
169 | 
170 | def resnet18(pretrained=False, **kwargs):
171 |     """Constructs a ResNet-18 model.
172 | 
173 |     Args:
174 |         pretrained (bool): If True, returns a model pre-trained on ImageNet
175 |     """
176 |     model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs)
177 |     if pretrained:
178 |         pretrained_dict = model_zoo.load_url(model_urls['resnet18'])
179 |         model_dict = model.state_dict()
180 |         pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict and v.shape == model_dict[k].shape}
181 |         model_dict.update(pretrained_dict)
182 |         model.load_state_dict(model_dict)
183 |     return model
184 | 
185 | 
186 | def resnet34(pretrained=False, **kwargs):
187 |     """Constructs a ResNet-34 model.
188 | 
189 |     Args:
190 |         pretrained (bool): If True, returns a model pre-trained on ImageNet
191 |     """
192 |     model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs)
193 |     if pretrained:
194 |         pretrained_dict = model_zoo.load_url(model_urls['resnet34'])
195 |         model_dict = model.state_dict()
196 |         pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict and v.shape == model_dict[k].shape}
197 |         model_dict.update(pretrained_dict)
198 |         model.load_state_dict(model_dict)
199 |     return model
200 | 
201 | 
202 | def resnet50(pretrained=False, **kwargs):
203 |     """Constructs a ResNet-50 model.
204 | 
205 |     Args:
206 |         pretrained (bool): If True, returns a model pre-trained on ImageNet
207 |     """
208 |     model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs)
209 |     if pretrained:
210 |         pretrained_dict = model_zoo.load_url(model_urls['resnet50'])
211 |         model_dict = model.state_dict()
212 |         pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict and v.shape == model_dict[k].shape}
213 |         model_dict.update(pretrained_dict)
214 |         model.load_state_dict(model_dict)
215 |     return model
216 | 
217 | 
218 | def resnet101(pretrained=False, **kwargs):
219 |     """Constructs a ResNet-101 model.
220 | 
221 |     Args:
222 |         pretrained (bool): If True, returns a model pre-trained on ImageNet
223 |     """
224 |     model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs)
225 |     if pretrained:
226 |         pretrained_dict = model_zoo.load_url(model_urls['resnet101'])
227 |         model_dict = model.state_dict()
228 |         pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict and v.shape == model_dict[k].shape}
229 |         model_dict.update(pretrained_dict)
230 |         model.load_state_dict(model_dict)
231 |     return model
232 | 
233 | 
234 | def resnet152(pretrained=False, **kwargs):
235 |     """Constructs a ResNet-152 model.
236 | 
237 |     Args:
238 |         pretrained (bool): If True, returns a model pre-trained on ImageNet
239 |     """
240 |     model = ResNet(Bottleneck, [3, 8, 36, 3], **kwargs)
241 |     if pretrained:
242 |         pretrained_dict = model_zoo.load_url(model_urls['resnet152'])
243 |         model_dict = model.state_dict()
244 |         pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict and v.shape == model_dict[k].shape}
245 |         model_dict.update(pretrained_dict)
246 |         model.load_state_dict(model_dict)
247 |     return model
248 | 


--------------------------------------------------------------------------------
/main.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 1,
  6 |    "metadata": {},
  7 |    "outputs": [],
  8 |    "source": [
  9 |     "import numpy as np\n",
 10 |     "import os\n",
 11 |     "import torch\n",
 12 |     "import torch.nn as nn\n",
 13 |     "import torch.backends.cudnn as cudnn\n",
 14 |     "from torch.utils.data import DataLoader\n",
 15 |     "from torch.autograd import Variable\n",
 16 |     "import torchvision.transforms as transforms\n",
 17 |     "from custom_dataset import *\n",
 18 |     "from mhbn import *\n",
 19 |     "import time\n",
 20 |     "import util"
 21 |    ]
 22 |   },
 23 |   {
 24 |    "cell_type": "code",
 25 |    "execution_count": 2,
 26 |    "metadata": {},
 27 |    "outputs": [
 28 |     {
 29 |      "name": "stdout",
 30 |      "output_type": "stream",
 31 |      "text": [
 32 |       "Loading data\n"
 33 |      ]
 34 |     }
 35 |    ],
 36 |    "source": [
 37 |     "print('Loading data')\n",
 38 |     "transform = transforms.Compose([\n",
 39 |     "    transforms.CenterCrop(500),\n",
 40 |     "    transforms.Resize(224),\n",
 41 |     "    transforms.ToTensor(),\n",
 42 |     "    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
 43 |     "])"
 44 |    ]
 45 |   },
 46 |   {
 47 |    "cell_type": "code",
 48 |    "execution_count": 3,
 49 |    "metadata": {},
 50 |    "outputs": [],
 51 |    "source": [
 52 |     "#cpu or gpu\n",
 53 |     "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
 54 |    ]
 55 |   },
 56 |   {
 57 |    "cell_type": "code",
 58 |    "execution_count": 4,
 59 |    "metadata": {},
 60 |    "outputs": [],
 61 |    "source": [
 62 |     "#root path of data\n",
 63 |     "root = \"/home/alan/Desktop/view/classes/\""
 64 |    ]
 65 |   },
 66 |   {
 67 |    "cell_type": "code",
 68 |    "execution_count": 5,
 69 |    "metadata": {},
 70 |    "outputs": [],
 71 |    "source": [
 72 |     "#the batch size of train and validation\n",
 73 |     "batch_size = 4"
 74 |    ]
 75 |   },
 76 |   {
 77 |    "cell_type": "code",
 78 |    "execution_count": 6,
 79 |    "metadata": {},
 80 |    "outputs": [],
 81 |    "source": [
 82 |     "# Load dataset\n",
 83 |     "dset_train = MultiViewDataSet(root, 'train', transform=transform)\n",
 84 |     "train_loader = DataLoader(dset_train, batch_size=batch_size, shuffle=True, num_workers=2)\n",
 85 |     "\n",
 86 |     "dset_val = MultiViewDataSet(root, 'test', transform=transform)\n",
 87 |     "val_loader = DataLoader(dset_val, batch_size=batch_size, shuffle=True, num_workers=2)"
 88 |    ]
 89 |   },
 90 |   {
 91 |    "cell_type": "code",
 92 |    "execution_count": 7,
 93 |    "metadata": {},
 94 |    "outputs": [],
 95 |    "source": [
 96 |     "classes, class_to_idx = dset_train.classes, dset_train.class_to_idx"
 97 |    ]
 98 |   },
 99 |   {
100 |    "cell_type": "code",
101 |    "execution_count": 8,
102 |    "metadata": {},
103 |    "outputs": [],
104 |    "source": [
105 |     "#number of local features\n",
106 |     "num_local_features = 10"
107 |    ]
108 |   },
109 |   {
110 |    "cell_type": "code",
111 |    "execution_count": 9,
112 |    "metadata": {},
113 |    "outputs": [
114 |     {
115 |      "name": "stdout",
116 |      "output_type": "stream",
117 |      "text": [
118 |       "Running on cuda:0\n"
119 |      ]
120 |     }
121 |    ],
122 |    "source": [
123 |     "model = mhbnn(True, len(classes), num_local_features)\n",
124 |     "model.to(device)\n",
125 |     "cudnn.benchmark = True\n",
126 |     "print('Running on ' + str(device))"
127 |    ]
128 |   },
129 |   {
130 |    "cell_type": "code",
131 |    "execution_count": 10,
132 |    "metadata": {},
133 |    "outputs": [],
134 |    "source": [
135 |     "lr = 0.0001\n",
136 |     "lr_decay = 0.1\n",
137 |     "lr_decay_freq = 30\n",
138 |     "n_epochs = 100\n",
139 |     "criterion = nn.CrossEntropyLoss()\n",
140 |     "optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
141 |     "\n",
142 |     "best_acc = 0.0\n",
143 |     "best_loss = 0.0\n",
144 |     "start_epoch = 0\n",
145 |     "print_freq = 10"
146 |    ]
147 |   },
148 |   {
149 |    "cell_type": "code",
150 |    "execution_count": 11,
151 |    "metadata": {},
152 |    "outputs": [],
153 |    "source": [
154 |     "resume_path = './checkpoint'\n",
155 |     "resume_ckp_path = ''"
156 |    ]
157 |   },
158 |   {
159 |    "cell_type": "code",
160 |    "execution_count": 12,
161 |    "metadata": {},
162 |    "outputs": [],
163 |    "source": [
164 |     "def load_checkpoint():\n",
165 |     "    global best_acc, start_epoch\n",
166 |     "    # Load checkpoint.\n",
167 |     "    print('\\n==> Loading checkpoint..')\n",
168 |     "    assert os.path.isfile(resume_ckp_path), 'Error: no checkpoint file found!'\n",
169 |     "\n",
170 |     "    checkpoint = torch.load(resume_ckp_path)\n",
171 |     "    best_acc = checkpoint['best_acc']\n",
172 |     "    start_epoch = checkpoint['epoch']\n",
173 |     "    model.load_state_dict(checkpoint['state_dict'])\n",
174 |     "    optimizer.load_state_dict(checkpoint['optimizer'])"
175 |    ]
176 |   },
177 |   {
178 |    "cell_type": "code",
179 |    "execution_count": 13,
180 |    "metadata": {},
181 |    "outputs": [],
182 |    "source": [
183 |     "def train():\n",
184 |     "    train_size = len(train_loader)\n",
185 |     "\n",
186 |     "    for i, (inputs, targets) in enumerate(train_loader):\n",
187 |     "        # Convert from list of 3D to 4D\n",
188 |     "        #convert to shape [batch_size, views, channels, width, height]\n",
189 |     "        inputs = np.stack(inputs, axis=1)\n",
190 |     "\n",
191 |     "        inputs = torch.from_numpy(inputs)\n",
192 |     "\n",
193 |     "        inputs, targets = inputs.cuda(device), targets.cuda(device)\n",
194 |     "        inputs, targets = Variable(inputs), Variable(targets)\n",
195 |     "\n",
196 |     "        # compute output\n",
197 |     "        outputs = model(inputs)\n",
198 |     "        loss = criterion(outputs, targets)\n",
199 |     "\n",
200 |     "        # compute gradient and do SGD step\n",
201 |     "        optimizer.zero_grad()\n",
202 |     "        loss.backward()\n",
203 |     "        optimizer.step()\n",
204 |     "\n",
205 |     "        if (i + 1) %  print_freq == 0:\n",
206 |     "            print(\"\\tIter [%d/%d] Loss: %.4f\" % (i + 1, train_size, loss.item()))"
207 |    ]
208 |   },
209 |   {
210 |    "cell_type": "code",
211 |    "execution_count": 14,
212 |    "metadata": {},
213 |    "outputs": [],
214 |    "source": [
215 |     "def eval(data_loader, is_test=False):\n",
216 |     "    if is_test:\n",
217 |     "       load_checkpoint()\n",
218 |     "\n",
219 |     "    # Eval\n",
220 |     "    total = 0.0\n",
221 |     "    correct = 0.0\n",
222 |     "\n",
223 |     "    total_loss = 0.0\n",
224 |     "    n = 0\n",
225 |     "\n",
226 |     "    for i, (inputs, targets) in enumerate(data_loader):\n",
227 |     "        with torch.no_grad():\n",
228 |     "            # Convert from list of 3D to 4D\n",
229 |     "            inputs = np.stack(inputs, axis=1)\n",
230 |     "\n",
231 |     "            inputs = torch.from_numpy(inputs)\n",
232 |     "\n",
233 |     "            inputs, targets = inputs.cuda(device), targets.cuda(device)\n",
234 |     "            inputs, targets = Variable(inputs), Variable(targets)\n",
235 |     "\n",
236 |     "            # compute output\n",
237 |     "            outputs = model(inputs)\n",
238 |     "            loss = criterion(outputs, targets)\n",
239 |     "\n",
240 |     "            total_loss += loss\n",
241 |     "            n += 1\n",
242 |     "\n",
243 |     "            _, predicted = torch.max(outputs.data, 1)\n",
244 |     "            total += targets.size(0)\n",
245 |     "            correct += (predicted.cpu() == targets.cpu()).sum()\n",
246 |     "    avg_test_acc = 100 * correct / total\n",
247 |     "    avg_loss = total_loss / n\n",
248 |     "\n",
249 |     "    return avg_test_acc, avg_loss"
250 |    ]
251 |   },
252 |   {
253 |    "cell_type": "code",
254 |    "execution_count": 15,
255 |    "metadata": {},
256 |    "outputs": [],
257 |    "source": [
258 |     "if resume_ckp_path:\n",
259 |     "    load_checkpoint()"
260 |    ]
261 |   },
262 |   {
263 |    "cell_type": "code",
264 |    "execution_count": 16,
265 |    "metadata": {},
266 |    "outputs": [
267 |     {
268 |      "name": "stdout",
269 |      "output_type": "stream",
270 |      "text": [
271 |       "\n",
272 |       "-----------------------------------\n",
273 |       "Epoch: [1/100]\n",
274 |       "\tIter [10/2461] Loss: 3.6883\n",
275 |       "\tIter [20/2461] Loss: 3.6968\n",
276 |       "\tIter [30/2461] Loss: 3.6757\n",
277 |       "\tIter [40/2461] Loss: 3.6590\n",
278 |       "\tIter [50/2461] Loss: 3.5795\n",
279 |       "\tIter [60/2461] Loss: 3.6115\n",
280 |       "\tIter [70/2461] Loss: 3.4769\n",
281 |       "\tIter [80/2461] Loss: 3.5802\n",
282 |       "\tIter [90/2461] Loss: 3.6035\n",
283 |       "\tIter [100/2461] Loss: 3.5225\n",
284 |       "\tIter [110/2461] Loss: 3.0842\n",
285 |       "\tIter [120/2461] Loss: 3.2837\n",
286 |       "\tIter [130/2461] Loss: 3.5027\n",
287 |       "\tIter [140/2461] Loss: 3.4837\n",
288 |       "\tIter [150/2461] Loss: 3.4527\n",
289 |       "\tIter [160/2461] Loss: 3.3980\n",
290 |       "\tIter [170/2461] Loss: 3.3681\n",
291 |       "\tIter [180/2461] Loss: 3.8943\n",
292 |       "\tIter [190/2461] Loss: 3.4648\n",
293 |       "\tIter [200/2461] Loss: 3.3594\n",
294 |       "\tIter [210/2461] Loss: 3.6478\n",
295 |       "\tIter [220/2461] Loss: 3.1512\n",
296 |       "\tIter [230/2461] Loss: 3.6209\n",
297 |       "\tIter [240/2461] Loss: 3.6048\n",
298 |       "\tIter [250/2461] Loss: 2.7274\n",
299 |       "\tIter [260/2461] Loss: 3.3344\n",
300 |       "\tIter [270/2461] Loss: 3.4610\n",
301 |       "\tIter [280/2461] Loss: 2.8464\n",
302 |       "\tIter [290/2461] Loss: 2.9396\n",
303 |       "\tIter [300/2461] Loss: 2.1151\n",
304 |       "\tIter [310/2461] Loss: 3.2575\n",
305 |       "\tIter [320/2461] Loss: 3.4276\n",
306 |       "\tIter [330/2461] Loss: 2.8965\n",
307 |       "\tIter [340/2461] Loss: 3.6647\n",
308 |       "\tIter [350/2461] Loss: 2.5075\n",
309 |       "\tIter [360/2461] Loss: 3.2576\n",
310 |       "\tIter [370/2461] Loss: 3.6031\n",
311 |       "\tIter [380/2461] Loss: 2.9933\n",
312 |       "\tIter [390/2461] Loss: 3.3318\n",
313 |       "\tIter [400/2461] Loss: 2.3040\n",
314 |       "\tIter [410/2461] Loss: 1.8501\n",
315 |       "\tIter [420/2461] Loss: 2.8625\n",
316 |       "\tIter [430/2461] Loss: 3.5399\n",
317 |       "\tIter [440/2461] Loss: 3.4031\n",
318 |       "\tIter [450/2461] Loss: 2.1144\n",
319 |       "\tIter [460/2461] Loss: 2.5815\n",
320 |       "\tIter [470/2461] Loss: 3.5014\n",
321 |       "\tIter [480/2461] Loss: 2.4705\n",
322 |       "\tIter [490/2461] Loss: 3.2672\n",
323 |       "\tIter [500/2461] Loss: 2.0191\n",
324 |       "\tIter [510/2461] Loss: 2.7211\n",
325 |       "\tIter [520/2461] Loss: 3.0238\n",
326 |       "\tIter [530/2461] Loss: 2.9658\n",
327 |       "\tIter [540/2461] Loss: 2.0828\n",
328 |       "\tIter [550/2461] Loss: 2.2403\n",
329 |       "\tIter [560/2461] Loss: 3.5115\n",
330 |       "\tIter [570/2461] Loss: 2.2371\n",
331 |       "\tIter [580/2461] Loss: 2.2086\n",
332 |       "\tIter [590/2461] Loss: 2.1343\n",
333 |       "\tIter [600/2461] Loss: 2.5443\n",
334 |       "\tIter [610/2461] Loss: 2.0975\n",
335 |       "\tIter [620/2461] Loss: 2.4053\n",
336 |       "\tIter [630/2461] Loss: 2.4311\n",
337 |       "\tIter [640/2461] Loss: 2.0281\n",
338 |       "\tIter [650/2461] Loss: 1.8169\n",
339 |       "\tIter [660/2461] Loss: 2.3873\n",
340 |       "\tIter [670/2461] Loss: 2.9960\n",
341 |       "\tIter [680/2461] Loss: 2.6585\n",
342 |       "\tIter [690/2461] Loss: 1.2482\n",
343 |       "\tIter [700/2461] Loss: 2.4506\n",
344 |       "\tIter [710/2461] Loss: 2.4764\n",
345 |       "\tIter [720/2461] Loss: 3.2262\n",
346 |       "\tIter [730/2461] Loss: 2.0014\n",
347 |       "\tIter [740/2461] Loss: 2.4954\n",
348 |       "\tIter [750/2461] Loss: 1.3195\n",
349 |       "\tIter [760/2461] Loss: 2.0731\n",
350 |       "\tIter [770/2461] Loss: 2.9335\n",
351 |       "\tIter [780/2461] Loss: 2.5198\n",
352 |       "\tIter [790/2461] Loss: 1.4060\n",
353 |       "\tIter [800/2461] Loss: 1.9878\n",
354 |       "\tIter [810/2461] Loss: 1.7995\n",
355 |       "\tIter [820/2461] Loss: 1.8057\n",
356 |       "\tIter [830/2461] Loss: 1.5230\n",
357 |       "\tIter [840/2461] Loss: 1.9433\n",
358 |       "\tIter [850/2461] Loss: 3.0614\n",
359 |       "\tIter [860/2461] Loss: 2.5209\n",
360 |       "\tIter [870/2461] Loss: 1.7934\n",
361 |       "\tIter [880/2461] Loss: 1.5844\n",
362 |       "\tIter [890/2461] Loss: 1.9234\n",
363 |       "\tIter [900/2461] Loss: 1.1359\n",
364 |       "\tIter [910/2461] Loss: 2.1720\n",
365 |       "\tIter [920/2461] Loss: 1.4051\n",
366 |       "\tIter [930/2461] Loss: 3.1418\n",
367 |       "\tIter [940/2461] Loss: 2.3136\n",
368 |       "\tIter [950/2461] Loss: 2.5286\n",
369 |       "\tIter [960/2461] Loss: 0.8228\n",
370 |       "\tIter [970/2461] Loss: 0.9342\n",
371 |       "\tIter [980/2461] Loss: 1.4226\n",
372 |       "\tIter [990/2461] Loss: 2.0243\n",
373 |       "\tIter [1000/2461] Loss: 1.2501\n",
374 |       "\tIter [1010/2461] Loss: 0.7800\n",
375 |       "\tIter [1020/2461] Loss: 3.7287\n",
376 |       "\tIter [1030/2461] Loss: 0.8400\n",
377 |       "\tIter [1040/2461] Loss: 1.7997\n",
378 |       "\tIter [1050/2461] Loss: 3.1451\n",
379 |       "\tIter [1060/2461] Loss: 0.8357\n",
380 |       "\tIter [1070/2461] Loss: 4.0061\n",
381 |       "\tIter [1080/2461] Loss: 2.9797\n",
382 |       "\tIter [1090/2461] Loss: 2.6271\n",
383 |       "\tIter [1100/2461] Loss: 1.2233\n",
384 |       "\tIter [1110/2461] Loss: 1.6945\n",
385 |       "\tIter [1120/2461] Loss: 3.0201\n",
386 |       "\tIter [1130/2461] Loss: 1.9713\n",
387 |       "\tIter [1140/2461] Loss: 2.4396\n",
388 |       "\tIter [1150/2461] Loss: 1.6910\n",
389 |       "\tIter [1160/2461] Loss: 3.5742\n",
390 |       "\tIter [1170/2461] Loss: 0.4053\n",
391 |       "\tIter [1180/2461] Loss: 0.8415\n",
392 |       "\tIter [1190/2461] Loss: 1.9848\n",
393 |       "\tIter [1200/2461] Loss: 1.3966\n",
394 |       "\tIter [1210/2461] Loss: 1.2791\n",
395 |       "\tIter [1220/2461] Loss: 1.5532\n",
396 |       "\tIter [1230/2461] Loss: 1.7672\n",
397 |       "\tIter [1240/2461] Loss: 1.2599\n",
398 |       "\tIter [1250/2461] Loss: 1.2625\n",
399 |       "\tIter [1260/2461] Loss: 1.3076\n",
400 |       "\tIter [1270/2461] Loss: 1.2042\n",
401 |       "\tIter [1280/2461] Loss: 2.1488\n",
402 |       "\tIter [1290/2461] Loss: 1.2337\n",
403 |       "\tIter [1300/2461] Loss: 2.6432\n",
404 |       "\tIter [1310/2461] Loss: 2.0122\n",
405 |       "\tIter [1320/2461] Loss: 2.3672\n",
406 |       "\tIter [1330/2461] Loss: 3.6324\n",
407 |       "\tIter [1340/2461] Loss: 1.9373\n",
408 |       "\tIter [1350/2461] Loss: 1.9891\n",
409 |       "\tIter [1360/2461] Loss: 2.1384\n",
410 |       "\tIter [1370/2461] Loss: 2.0627\n",
411 |       "\tIter [1380/2461] Loss: 1.2508\n",
412 |       "\tIter [1390/2461] Loss: 1.7329\n",
413 |       "\tIter [1400/2461] Loss: 0.9439\n",
414 |       "\tIter [1410/2461] Loss: 1.8203\n",
415 |       "\tIter [1420/2461] Loss: 1.7847\n",
416 |       "\tIter [1430/2461] Loss: 1.5063\n",
417 |       "\tIter [1440/2461] Loss: 2.8591\n",
418 |       "\tIter [1450/2461] Loss: 1.5119\n",
419 |       "\tIter [1460/2461] Loss: 3.2594\n",
420 |       "\tIter [1470/2461] Loss: 3.1464\n",
421 |       "\tIter [1480/2461] Loss: 2.6167\n",
422 |       "\tIter [1490/2461] Loss: 2.0678\n",
423 |       "\tIter [1500/2461] Loss: 1.1151\n",
424 |       "\tIter [1510/2461] Loss: 3.3321\n",
425 |       "\tIter [1520/2461] Loss: 1.1079\n",
426 |       "\tIter [1530/2461] Loss: 2.0831\n",
427 |       "\tIter [1540/2461] Loss: 1.8686\n",
428 |       "\tIter [1550/2461] Loss: 1.3374\n",
429 |       "\tIter [1560/2461] Loss: 1.8512\n",
430 |       "\tIter [1570/2461] Loss: 1.5932\n",
431 |       "\tIter [1580/2461] Loss: 2.1007\n",
432 |       "\tIter [1590/2461] Loss: 0.8625\n",
433 |       "\tIter [1600/2461] Loss: 2.4042\n",
434 |       "\tIter [1610/2461] Loss: 1.9851\n",
435 |       "\tIter [1620/2461] Loss: 2.6881\n",
436 |       "\tIter [1630/2461] Loss: 1.5568\n",
437 |       "\tIter [1640/2461] Loss: 1.7749\n",
438 |       "\tIter [1650/2461] Loss: 3.2325\n",
439 |       "\tIter [1660/2461] Loss: 2.4645\n",
440 |       "\tIter [1670/2461] Loss: 1.4254\n",
441 |       "\tIter [1680/2461] Loss: 1.7026\n",
442 |       "\tIter [1690/2461] Loss: 2.0576\n",
443 |       "\tIter [1700/2461] Loss: 2.1249\n",
444 |       "\tIter [1710/2461] Loss: 1.9619\n",
445 |       "\tIter [1720/2461] Loss: 1.4691\n",
446 |       "\tIter [1730/2461] Loss: 1.5726\n",
447 |       "\tIter [1740/2461] Loss: 0.7356\n",
448 |       "\tIter [1750/2461] Loss: 0.4405\n",
449 |       "\tIter [1760/2461] Loss: 2.4073\n",
450 |       "\tIter [1770/2461] Loss: 2.2401\n",
451 |       "\tIter [1780/2461] Loss: 0.6892\n",
452 |       "\tIter [1790/2461] Loss: 1.4002\n",
453 |       "\tIter [1800/2461] Loss: 0.4048\n",
454 |       "\tIter [1810/2461] Loss: 2.0727\n",
455 |       "\tIter [1820/2461] Loss: 0.6470\n",
456 |       "\tIter [1830/2461] Loss: 1.2344\n",
457 |       "\tIter [1840/2461] Loss: 0.8181\n",
458 |       "\tIter [1850/2461] Loss: 1.9000\n",
459 |       "\tIter [1860/2461] Loss: 1.0141\n",
460 |       "\tIter [1870/2461] Loss: 0.9508\n",
461 |       "\tIter [1880/2461] Loss: 3.5383\n",
462 |       "\tIter [1890/2461] Loss: 1.9355\n",
463 |       "\tIter [1900/2461] Loss: 1.1119\n",
464 |       "\tIter [1910/2461] Loss: 2.2507\n",
465 |       "\tIter [1920/2461] Loss: 0.7424\n",
466 |       "\tIter [1930/2461] Loss: 1.1117\n",
467 |       "\tIter [1940/2461] Loss: 1.2224\n",
468 |       "\tIter [1950/2461] Loss: 1.3931\n",
469 |       "\tIter [1960/2461] Loss: 1.9083\n",
470 |       "\tIter [1970/2461] Loss: 2.5647\n",
471 |       "\tIter [1980/2461] Loss: 0.6747\n",
472 |       "\tIter [1990/2461] Loss: 1.4130\n",
473 |       "\tIter [2000/2461] Loss: 1.6327\n",
474 |       "\tIter [2010/2461] Loss: 1.4419\n",
475 |       "\tIter [2020/2461] Loss: 0.4077\n",
476 |       "\tIter [2030/2461] Loss: 1.4769\n",
477 |       "\tIter [2040/2461] Loss: 1.1526\n",
478 |       "\tIter [2050/2461] Loss: 2.0156\n",
479 |       "\tIter [2060/2461] Loss: 2.2939\n",
480 |       "\tIter [2070/2461] Loss: 0.7684\n",
481 |       "\tIter [2080/2461] Loss: 1.1506\n",
482 |       "\tIter [2090/2461] Loss: 1.2534\n",
483 |       "\tIter [2100/2461] Loss: 0.8084\n",
484 |       "\tIter [2110/2461] Loss: 1.2233\n",
485 |       "\tIter [2120/2461] Loss: 1.4552\n",
486 |       "\tIter [2130/2461] Loss: 2.0513\n",
487 |       "\tIter [2140/2461] Loss: 2.3375\n",
488 |       "\tIter [2150/2461] Loss: 1.7544\n",
489 |       "\tIter [2160/2461] Loss: 1.7033\n",
490 |       "\tIter [2170/2461] Loss: 0.8690\n",
491 |       "\tIter [2180/2461] Loss: 1.9208\n",
492 |       "\tIter [2190/2461] Loss: 1.1514\n",
493 |       "\tIter [2200/2461] Loss: 1.3290\n",
494 |       "\tIter [2210/2461] Loss: 1.9603\n",
495 |       "\tIter [2220/2461] Loss: 1.6498\n",
496 |       "\tIter [2230/2461] Loss: 3.1993\n",
497 |       "\tIter [2240/2461] Loss: 1.8922\n",
498 |       "\tIter [2250/2461] Loss: 0.7850\n",
499 |       "\tIter [2260/2461] Loss: 0.5452\n",
500 |       "\tIter [2270/2461] Loss: 0.7313\n",
501 |       "\tIter [2280/2461] Loss: 2.4882\n",
502 |       "\tIter [2290/2461] Loss: 1.8277\n",
503 |       "\tIter [2300/2461] Loss: 0.7386\n",
504 |       "\tIter [2310/2461] Loss: 1.8763\n",
505 |       "\tIter [2320/2461] Loss: 0.3686\n",
506 |       "\tIter [2330/2461] Loss: 0.3019\n",
507 |       "\tIter [2340/2461] Loss: 1.8997\n",
508 |       "\tIter [2350/2461] Loss: 1.5108\n",
509 |       "\tIter [2360/2461] Loss: 1.1133\n",
510 |       "\tIter [2370/2461] Loss: 0.3528\n",
511 |       "\tIter [2380/2461] Loss: 1.3412\n",
512 |       "\tIter [2390/2461] Loss: 1.7411\n",
513 |       "\tIter [2400/2461] Loss: 1.2908\n",
514 |       "\tIter [2410/2461] Loss: 0.6166\n",
515 |       "\tIter [2420/2461] Loss: 0.4670\n",
516 |       "\tIter [2430/2461] Loss: 1.0936\n",
517 |       "\tIter [2440/2461] Loss: 1.7904\n",
518 |       "\tIter [2450/2461] Loss: 1.8195\n",
519 |       "\tIter [2460/2461] Loss: 0.9394\n",
520 |       "Time taken: 226.65 sec.\n",
521 |       "\n",
522 |       "Evaluation:\n",
523 |       "\tVal Acc: 65.00 - Loss: 1.1527\n",
524 |       "\tCurrent best  val acc: 0.00\n",
525 |       "\tSaving checkpoint - Acc: 65.00\n",
526 |       "\n",
527 |       "-----------------------------------\n",
528 |       "Epoch: [2/100]\n",
529 |       "\tIter [10/2461] Loss: 1.2999\n",
530 |       "\tIter [20/2461] Loss: 2.0038\n",
531 |       "\tIter [30/2461] Loss: 1.0422\n",
532 |       "\tIter [40/2461] Loss: 2.2150\n",
533 |       "\tIter [50/2461] Loss: 0.6337\n",
534 |       "\tIter [60/2461] Loss: 2.1248\n",
535 |       "\tIter [70/2461] Loss: 1.0289\n",
536 |       "\tIter [80/2461] Loss: 2.0062\n",
537 |       "\tIter [90/2461] Loss: 0.9409\n",
538 |       "\tIter [100/2461] Loss: 1.1977\n",
539 |       "\tIter [110/2461] Loss: 1.5402\n",
540 |       "\tIter [120/2461] Loss: 1.2572\n",
541 |       "\tIter [130/2461] Loss: 2.0992\n",
542 |       "\tIter [140/2461] Loss: 0.2727\n",
543 |       "\tIter [150/2461] Loss: 0.8716\n",
544 |       "\tIter [160/2461] Loss: 1.2568\n"
545 |      ]
546 |     },
547 |     {
548 |      "name": "stdout",
549 |      "output_type": "stream",
550 |      "text": [
551 |       "\tIter [170/2461] Loss: 0.4038\n",
552 |       "\tIter [180/2461] Loss: 1.5911\n",
553 |       "\tIter [190/2461] Loss: 1.8787\n",
554 |       "\tIter [200/2461] Loss: 0.9013\n"
555 |      ]
556 |     },
557 |     {
558 |      "name": "stderr",
559 |      "output_type": "stream",
560 |      "text": [
561 |       "Process Process-6:\n",
562 |       "Traceback (most recent call last):\n",
563 |       "Process Process-5:\n",
564 |       "  File \"/home/alan/anaconda3/envs/ml/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
565 |       "    self.run()\n",
566 |       "  File \"/home/alan/anaconda3/envs/ml/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
567 |       "    self._target(*self._args, **self._kwargs)\n",
568 |       "  File \"/home/alan/anaconda3/envs/ml/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/transforms/transforms.py\", line 49, in __call__\n",
569 |       "    img = t(img)\n",
570 |       "  File \"/home/alan/anaconda3/envs/ml/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 106, in _worker_loop\n",
571 |       "    samples = collate_fn([dataset[i] for i in batch_indices])\n",
572 |       "  File \"/home/alan/anaconda3/envs/ml/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 106, in <listcomp>\n",
573 |       "    samples = collate_fn([dataset[i] for i in batch_indices])\n",
574 |       "Traceback (most recent call last):\n",
575 |       "  File \"/home/alan/Desktop/MHBN/custom_dataset.py\", line 43, in __getitem__\n",
576 |       "    im = self.transform(im)\n",
577 |       "  File \"/home/alan/anaconda3/envs/ml/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
578 |       "    self.run()\n",
579 |       "  File \"/home/alan/anaconda3/envs/ml/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/transforms/transforms.py\", line 49, in __call__\n",
580 |       "    img = t(img)\n",
581 |       "  File \"/home/alan/anaconda3/envs/ml/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
582 |       "    self._target(*self._args, **self._kwargs)\n",
583 |       "  File \"/home/alan/anaconda3/envs/ml/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/transforms/transforms.py\", line 175, in __call__\n",
584 |       "    return F.resize(img, self.size, self.interpolation)\n",
585 |       "  File \"/home/alan/anaconda3/envs/ml/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 106, in _worker_loop\n",
586 |       "    samples = collate_fn([dataset[i] for i in batch_indices])\n",
587 |       "  File \"/home/alan/anaconda3/envs/ml/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 106, in <listcomp>\n",
588 |       "    samples = collate_fn([dataset[i] for i in batch_indices])\n",
589 |       "  File \"/home/alan/Desktop/MHBN/custom_dataset.py\", line 43, in __getitem__\n",
590 |       "    im = self.transform(im)\n",
591 |       "  File \"/home/alan/anaconda3/envs/ml/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/transforms/transforms.py\", line 175, in __call__\n",
592 |       "    return F.resize(img, self.size, self.interpolation)\n",
593 |       "  File \"/home/alan/anaconda3/envs/ml/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/transforms/functional.py\", line 204, in resize\n",
594 |       "    return img.resize((ow, oh), interpolation)\n",
595 |       "  File \"/home/alan/anaconda3/envs/ml/lib/python3.6/site-packages/PIL/Image.py\", line 1765, in resize\n",
596 |       "    return self._new(self.im.resize(size, resample, box))\n",
597 |       "  File \"/home/alan/anaconda3/envs/ml/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/transforms/functional.py\", line 204, in resize\n",
598 |       "    return img.resize((ow, oh), interpolation)\n",
599 |       "KeyboardInterrupt\n",
600 |       "  File \"/home/alan/anaconda3/envs/ml/lib/python3.6/site-packages/PIL/Image.py\", line 1765, in resize\n",
601 |       "    return self._new(self.im.resize(size, resample, box))\n",
602 |       "KeyboardInterrupt\n"
603 |      ]
604 |     },
605 |     {
606 |      "ename": "KeyboardInterrupt",
607 |      "evalue": "",
608 |      "output_type": "error",
609 |      "traceback": [
610 |       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
611 |       "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
612 |       "\u001b[0;32m<ipython-input-16-98d1265f1571>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Time taken: %.2f sec.'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
613 |       "\u001b[0;32m<ipython-input-13-392d35a33e25>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m()\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0minputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m         \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m         \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtargets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
614 |       "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
615 |      ]
616 |     }
617 |    ],
618 |    "source": [
619 |     "for epoch in range(start_epoch, n_epochs):\n",
620 |     "    print('\\n-----------------------------------')\n",
621 |     "    print('Epoch: [%d/%d]' % (epoch+1, n_epochs))\n",
622 |     "    start = time.time()\n",
623 |     "\n",
624 |     "    model.train()\n",
625 |     "    train()\n",
626 |     "    print('Time taken: %.2f sec.' % (time.time() - start))\n",
627 |     "\n",
628 |     "    model.eval()\n",
629 |     "    avg_test_acc, avg_loss = eval(val_loader)\n",
630 |     "\n",
631 |     "    print('\\nEvaluation:')\n",
632 |     "    print('\\tVal Acc: %.2f - Loss: %.4f' % (avg_test_acc.item(), avg_loss.item()))\n",
633 |     "    print('\\tCurrent best  val acc: %.2f' % best_acc)\n",
634 |     "    \n",
635 |     "\n",
636 |     "    # Log epoch to tensorboard\n",
637 |     "    # See log using: tensorboard --logdir='logs' --port=6006\n",
638 |     "#     util.logEpoch(logger, model, epoch + 1, avg_loss, avg_test_acc)\n",
639 |     "\n",
640 |     "    # Save model\n",
641 |     "    if avg_test_acc > best_acc:\n",
642 |     "        print('\\tSaving checkpoint - Acc: %.2f' % avg_test_acc)\n",
643 |     "        best_acc = avg_test_acc\n",
644 |     "        best_loss = avg_loss\n",
645 |     "        util.save_checkpoint({\n",
646 |     "            'epoch': epoch + 1,\n",
647 |     "            'state_dict': model.state_dict(),\n",
648 |     "            'acc': avg_test_acc,\n",
649 |     "            'best_acc': best_acc,\n",
650 |     "            'optimizer': optimizer.state_dict(),\n",
651 |     "        }, 'mhbnn_'+str(num_local_features))\n",
652 |     "\n",
653 |     "    # Decaying Learning Rate\n",
654 |     "    if (epoch + 1) % lr_decay_freq == 0:\n",
655 |     "        lr *= lr_decay\n",
656 |     "        optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
657 |     "        print('Learning rate:', lr)"
658 |    ]
659 |   },
660 |   {
661 |    "cell_type": "code",
662 |    "execution_count": null,
663 |    "metadata": {},
664 |    "outputs": [],
665 |    "source": []
666 |   }
667 |  ],
668 |  "metadata": {
669 |   "kernelspec": {
670 |    "display_name": "ml",
671 |    "language": "python",
672 |    "name": "ml"
673 |   },
674 |   "language_info": {
675 |    "codemirror_mode": {
676 |     "name": "ipython",
677 |     "version": 3
678 |    },
679 |    "file_extension": ".py",
680 |    "mimetype": "text/x-python",
681 |    "name": "python",
682 |    "nbconvert_exporter": "python",
683 |    "pygments_lexer": "ipython3",
684 |    "version": "3.6.6"
685 |   }
686 |  },
687 |  "nbformat": 4,
688 |  "nbformat_minor": 2
689 | }
690 | 


--------------------------------------------------------------------------------