├── README.md ├── miniImageNet_TapNet ├── data │ └── __init__.py ├── scripts │ └── train_TapNet_miniImageNet.py └── utils │ ├── __pycache__ │ ├── generators.cpython-36.pyc │ ├── model_TapNet_ResNet12.cpython-36.pyc │ └── rank_nullspace.cpython-36.pyc │ ├── generators.py │ ├── model_TapNet_ResNet12.py │ └── rank_nullspace.py └── tieredImageNet_TapNet ├── data └── __init__.py ├── scripts └── train_TapNet_tieredImageNet.py └── utils ├── __pycache__ ├── generators.cpython-36.pyc ├── model_TapNet_ResNet12.cpython-36.pyc └── rank_nullspace.cpython-36.pyc ├── generators.py ├── model_TapNet_ResNet12.py └── rank_nullspace.py /README.md: -------------------------------------------------------------------------------- 1 | # TapNet: Neural Network Augmented with Task-Adaptive Projection for Few-Shot Learning 2 | 3 | Code for the ICML 2019 paper TapNet: Neural Network Augmented with Task-Adaptive Projection for Few-Shot Learning 4 | 5 | ## Dependencies 6 | * This code is tested on Ubuntu 16.04 with Python 3.6 and chainer 5.20 7 | 8 | ## Data 9 | ### miniImageNet 10 | #Download and unzip "mini-imagenet.tar.gz" from Google Drive link [[mini-ImageNet](https://drive.google.com/file/d/1DvYd7LMa0zvlqTM8oBdCWwQSxpZdf_D5/view?usp=sharing)] 11 | 12 | #Place ``train.npz``, ``val.npz``, ``test.npz`` files in ``TapNet/miniImageNet_TapNet/data`` 13 | 14 | 15 | ### tieredImageNet 16 | #Download and unzip "tiered-imagenet.tar.gz" from Google Drive link [[tiered-ImageNet](https://drive.google.com/file/d/1zz7bAYus7EeoMokwUQlLc3OY_eoII8B7/view?usp=sharing)] 17 | 18 | #Place images ``.npz`` and labels ``.pkl`` files in ``TapNet/tieredImageNet_TapNet/data`` 19 | 20 | ## Running the code 21 | 22 | ``` 23 | #For miniImageNet experiment 24 | 25 | cd /TapNet/miniImageNet_TapNet/scripts 26 | python train_TapNet_miniImageNet.py --gpu {GPU device number} 27 | --n_shot {n_shot} 28 | --nb_class_train {number of classes in training} 29 | --nb_class_test {number of classes in test} 30 | --n_query_train {number of queries per class in training} 31 | --n_query_test {number of queries per class in test} 32 | --wd_rate {Weight decay rate} 33 | 34 | 35 | #For tieredImageNet experiment 36 | 37 | cd /TapNet/tieredImageNet_TapNet/scripts 38 | python train_TapNet_tieredImageNet.py --gpu {GPU device number} 39 | --n_shot {n_shot} 40 | --nb_class_train {number of classes in training} 41 | --nb_class_test {number of classes in test} 42 | --n_query_train {number of queries per class in training} 43 | --n_query_test {number of queries per class in test} 44 | --wd_rate {Weight decay rate} 45 | ``` 46 | 47 | -------------------------------------------------------------------------------- /miniImageNet_TapNet/data/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /miniImageNet_TapNet/scripts/train_TapNet_miniImageNet.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('../') 4 | import argparse 5 | 6 | import numpy as np 7 | import scipy.io as sio 8 | 9 | import chainer.functions as F 10 | from chainer import optimizers 11 | from chainer import cuda 12 | from chainer import serializers 13 | 14 | from utils.generators import miniImageNetGenerator 15 | from utils.model_TapNet_ResNet12 import TapNet 16 | 17 | if __name__ == '__main__': 18 | parser = argparse.ArgumentParser() 19 | parser.add_argument('--gpu', type=int, default=0, 20 | help='gpu device number. -1 for cpu.') 21 | parser.add_argument('--n_shot', type=int, default=5, 22 | help='Number of shots.') 23 | parser.add_argument('--nb_class_train', type=int, default=20, 24 | help='Number of training classes .') 25 | parser.add_argument('--nb_class_test', type=int, default=5, 26 | help='Number of test classes .') 27 | parser.add_argument('--n_query_train', type=int, default=8, 28 | help='Number of queries per class in training.') 29 | parser.add_argument('--n_query_test', type=int, default=15, 30 | help='Number of queries per class in test.') 31 | parser.add_argument('--wd_rate', type=float, default=5e-4, 32 | help='Weight decay rate in Adam optimizer') 33 | # set params 34 | # ----------- 35 | args = parser.parse_args() 36 | if args.gpu < 0: 37 | xp = np 38 | else: 39 | import cupy as cp 40 | os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" 41 | os.environ["CUDA_VISIBLE_DEVICES"]="%d" %args.gpu 42 | xp = cp 43 | dimension=512 44 | max_iter=50001 45 | lrdecay = True 46 | lrstep = 40000 47 | n_shot=args.n_shot 48 | n_query=args.n_query_train 49 | n_query_test = args.n_query_test 50 | nb_class_train=args.nb_class_train 51 | nb_class_test=args.nb_class_test 52 | wd_rate=args.wd_rate 53 | savefile_name='save/TapNet_miniImageNet_ResNet12.mat' 54 | filename_5shot='save/TapNet_miniImageNet_ResNet12' 55 | filename_5shot_last='save/TapNet_miniImageNet_ResNet12_last' 56 | 57 | # set up training 58 | # ------------------ 59 | model = TapNet(nb_class_train=nb_class_train, nb_class_test=nb_class_test, input_size=3*84*84, 60 | dimension=dimension, n_shot=n_shot, gpu=args.gpu) 61 | 62 | optimizer = optimizers.Adam(alpha=1e-3, weight_decay_rate=wd_rate) 63 | model.set_optimizer(optimizer) 64 | 65 | train_generator = miniImageNetGenerator(data_file='../data/Imagenet/train.npz', 66 | nb_classes=nb_class_train, nb_samples_per_class=n_shot+n_query, 67 | max_iter=max_iter, xp=xp) 68 | 69 | # Result analysis list 70 | # ----------------- 71 | 72 | loss_h=[] 73 | accuracy_h_val=[] 74 | accuracy_h_test=[] 75 | 76 | acc_best=0 77 | epoch_best=0 78 | 79 | # start training 80 | # ---------------- 81 | 82 | for t, (images, labels) in train_generator: 83 | # train 84 | loss = model.train(images, labels) 85 | # logging 86 | loss_h.extend([loss.tolist()]) 87 | if (t % 50 == 0): 88 | print("Episode: %d, Train Loss: %f "%(t, loss)) 89 | 90 | if (t != 0) and (t % 500 == 0): 91 | print('Evaluation in Validation data') 92 | test_generator = miniImageNetGenerator(data_file='../data/Imagenet/val.npz', 93 | nb_classes=nb_class_test, nb_samples_per_class=n_shot+n_query_test, 94 | max_iter=600, xp=xp) 95 | scores = [] 96 | for i, (images, labels) in test_generator: 97 | accs = model.evaluate(images, labels) 98 | accs_ = [cuda.to_cpu(acc) for acc in accs] 99 | score = np.asarray(accs_, dtype=int) 100 | scores.append(score) 101 | print(('Accuracy 5 shot ={:.2f}%').format(100*np.mean(np.array(scores)))) 102 | accuracy_t=100*np.mean(np.array(scores)) 103 | 104 | if acc_best < accuracy_t: 105 | acc_best = accuracy_t 106 | epoch_best=t 107 | serializers.save_npz(filename_5shot,model.chain) 108 | 109 | accuracy_h_val.extend([accuracy_t.tolist()]) 110 | del(test_generator) 111 | del(accs) 112 | del(accs_) 113 | del(accuracy_t) 114 | 115 | print('Evaluation in Test data') 116 | test_generator = miniImageNetGenerator(data_file='../data/Imagenet/test.npz', 117 | nb_classes=nb_class_test, nb_samples_per_class=n_shot+n_query_test, 118 | max_iter=600, xp=xp) 119 | scores = [] 120 | for i, (images, labels) in test_generator: 121 | accs = model.evaluate(images, labels) 122 | accs_ = [cuda.to_cpu(acc) for acc in accs] 123 | score = np.asarray(accs_, dtype=int) 124 | scores.append(score) 125 | print(('Accuracy 5 shot ={:.2f}%').format(100*np.mean(np.array(scores)))) 126 | accuracy_t=100*np.mean(np.array(scores)) 127 | accuracy_h_test.extend([accuracy_t.tolist()]) 128 | del(test_generator) 129 | del(accs) 130 | del(accs_) 131 | del(accuracy_t) 132 | sio.savemat(savefile_name, {'accuracy_h_val':accuracy_h_val, 'accuracy_h_test':accuracy_h_test, 'epoch_best':epoch_best,'acc_best':acc_best}) 133 | if len(accuracy_h_val) >10: 134 | print('***Average accuracy on past 10 evaluation***') 135 | print('Best epoch =',epoch_best,'Best 5 shot acc=',acc_best) 136 | 137 | serializers.save_npz(filename_5shot_last,model.chain) 138 | 139 | if (t != 0) and (t % lrstep == 0) and lrdecay: 140 | model.decay_learning_rate(0.1) 141 | 142 | 143 | accuracy_h5=[] 144 | 145 | serializers.load_npz(filename_5shot, model.chain) 146 | print('Evaluating the best 5shot model...') 147 | for i in range(50): 148 | test_generator = miniImageNetGenerator(data_file='../data/Imagenet/test.npz', 149 | nb_classes=nb_class_test, nb_samples_per_class=n_shot+n_query_test, 150 | max_iter=600, xp=xp) 151 | scores=[] 152 | for j, (images, labels) in test_generator: 153 | accs = model.evaluate(images, labels) 154 | accs_ = [cuda.to_cpu(acc) for acc in accs] 155 | score = np.asarray(accs_, dtype=int) 156 | scores.append(score) 157 | accuracy_t=100*np.mean(np.array(scores)) 158 | accuracy_h5.extend([accuracy_t.tolist()]) 159 | print(('600 episodes with 15-query accuracy: 5-shot ={:.2f}%').format(accuracy_t)) 160 | del(test_generator) 161 | del(accs) 162 | del(accs_) 163 | del(accuracy_t) 164 | sio.savemat(savefile_name, {'accuracy_h_val':accuracy_h_val, 'accuracy_h_test':accuracy_h_test, 'epoch_best':epoch_best,'acc_best':acc_best, 'accuracy_h5':accuracy_h5}) 165 | print(('Accuracy_test 5 shot ={:.2f}%').format(np.mean(accuracy_h5))) 166 | 167 | 168 | -------------------------------------------------------------------------------- /miniImageNet_TapNet/utils/__pycache__/generators.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/istarjun/TapNet/87f7c16112bb180a575c8b6345202bbf5b6bb15e/miniImageNet_TapNet/utils/__pycache__/generators.cpython-36.pyc -------------------------------------------------------------------------------- /miniImageNet_TapNet/utils/__pycache__/model_TapNet_ResNet12.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/istarjun/TapNet/87f7c16112bb180a575c8b6345202bbf5b6bb15e/miniImageNet_TapNet/utils/__pycache__/model_TapNet_ResNet12.cpython-36.pyc -------------------------------------------------------------------------------- /miniImageNet_TapNet/utils/__pycache__/rank_nullspace.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/istarjun/TapNet/87f7c16112bb180a575c8b6345202bbf5b6bb15e/miniImageNet_TapNet/utils/__pycache__/rank_nullspace.cpython-36.pyc -------------------------------------------------------------------------------- /miniImageNet_TapNet/utils/generators.py: -------------------------------------------------------------------------------- 1 | """ 2 | This code based on codes from https://github.com/tristandeleu/ntm-one-shot 3 | """ 4 | import numpy as np 5 | import random 6 | 7 | class miniImageNetGenerator(object): 8 | """miniImageNetGenerator 9 | 10 | Args: 11 | data_file (str): 'data/train.npz' or 'data/test.npz' 12 | nb_classes (int): number of classes in an episode 13 | nb_samples_per_class (int): nuber of samples per class in an episode 14 | max_iter (int): max number of episode generation 15 | xp: numpy or cupy 16 | """ 17 | def __init__(self, data_file, nb_classes=5, nb_samples_per_class=10, 18 | max_iter=None, xp=np): 19 | super(miniImageNetGenerator, self).__init__() 20 | self.data_file = data_file 21 | self.nb_classes = nb_classes 22 | self.nb_samples_per_class = nb_samples_per_class 23 | self.max_iter = max_iter 24 | self.xp = xp 25 | self.num_iter = 0 26 | self.data = self._load_data(self.data_file) 27 | 28 | def _load_data(self, data_file): 29 | data_dict = np.load(data_file) 30 | return {key: np.array(val) for (key, val) in data_dict.items()} 31 | 32 | def __iter__(self): 33 | return self 34 | 35 | def __next__(self): 36 | return self.next() 37 | 38 | def next(self): 39 | if (self.max_iter is None) or (self.num_iter < self.max_iter): 40 | self.num_iter += 1 41 | images, labels = self.sample(self.nb_classes, self.nb_samples_per_class) 42 | 43 | return (self.num_iter - 1), (images, labels) 44 | else: 45 | raise StopIteration() 46 | 47 | def sample(self, nb_classes, nb_samples_per_class): 48 | sampled_characters = random.sample(self.data.keys(), nb_classes) 49 | labels_and_images = [] 50 | for (k, char) in enumerate(sampled_characters): 51 | _imgs = self.data[char] 52 | _ind = random.sample(range(len(_imgs)), nb_samples_per_class) 53 | labels_and_images.extend([(k, self.xp.array(_imgs[i].flatten())) for i in _ind]) 54 | arg_labels_and_images = [] 55 | for i in range(self.nb_samples_per_class): 56 | for j in range(self.nb_classes): 57 | arg_labels_and_images.extend([labels_and_images[i+j*self.nb_samples_per_class]]) 58 | 59 | labels, images = zip(*arg_labels_and_images) 60 | return images, labels 61 | 62 | -------------------------------------------------------------------------------- /miniImageNet_TapNet/utils/model_TapNet_ResNet12.py: -------------------------------------------------------------------------------- 1 | import cupy as cp 2 | import numpy as np 3 | 4 | import chainer 5 | import chainer.links as L 6 | import chainer.functions as F 7 | from chainer import cuda 8 | 9 | from utils.rank_nullspace import nullspace_gpu 10 | 11 | class TapNet(object): 12 | def __init__(self, nb_class_train, nb_class_test, input_size, dimension, 13 | n_shot, gpu=-1): 14 | """ 15 | Args 16 | nb_class_train (int): number of classes in a training episode 17 | nb_class_test (int): number of classes in a test episode 18 | input_size (int): dimension of input vector 19 | dimension (int) : dimension of embedding space 20 | n_shot (int) : number of shots 21 | """ 22 | self.nb_class_train = nb_class_train 23 | self.nb_class_test = nb_class_test 24 | self.input_size = input_size 25 | self.dimension = dimension 26 | self.n_shot = n_shot 27 | # create chain 28 | self.chain = self._create_chain() 29 | self.set_gpu(gpu) 30 | 31 | 32 | # Set up methods 33 | # --------------- 34 | @property 35 | def xp(self): 36 | if self.gpu<0: 37 | return np 38 | else: 39 | return cp 40 | 41 | def set_gpu(self, gpu): 42 | self.gpu = gpu 43 | if self.gpu < 0: 44 | self.chain.to_cpu() 45 | else: 46 | self.chain.to_gpu() 47 | 48 | 49 | def set_optimizer(self, optimizer): 50 | self.optimizer = optimizer 51 | self.optimizer.setup(self.chain) 52 | self.optimizer.use_cleargrads(use=False) 53 | 54 | 55 | 56 | def _create_chain(self): 57 | chain = chainer.Chain( 58 | l_conv1_1=L.Convolution2D(None,64,(3,3), pad=1), 59 | l_norm1_1=L.BatchNormalization(64), 60 | l_conv1_2=L.Convolution2D(64,64,(3,3), pad=1), 61 | l_norm1_2=L.BatchNormalization(64), 62 | l_conv1_3=L.Convolution2D(64,64,(3,3), pad=1), 63 | l_norm1_3=L.BatchNormalization(64), 64 | l_conv1_r=L.Convolution2D(None,64,(3,3), pad=1), 65 | l_norm1_r=L.BatchNormalization(64), 66 | 67 | l_conv2_1=L.Convolution2D(64,128,(3,3), pad=1), 68 | l_norm2_1=L.BatchNormalization(128), 69 | l_conv2_2=L.Convolution2D(128,128,(3,3), pad=1), 70 | l_norm2_2=L.BatchNormalization(128), 71 | l_conv2_3=L.Convolution2D(128,128,(3,3), pad=1), 72 | l_norm2_3=L.BatchNormalization(128), 73 | l_conv2_r=L.Convolution2D(64,128,(3,3), pad=1), 74 | l_norm2_r=L.BatchNormalization(128), 75 | 76 | l_conv3_1=L.Convolution2D(128,256,(3,3), pad=1), 77 | l_norm3_1=L.BatchNormalization(256), 78 | l_conv3_2=L.Convolution2D(256,256,(3,3), pad=1), 79 | l_norm3_2=L.BatchNormalization(256), 80 | l_conv3_3=L.Convolution2D(256,256,(3,3), pad=1), 81 | l_norm3_3=L.BatchNormalization(256), 82 | l_conv3_r=L.Convolution2D(128,256,(3,3), pad=1), 83 | l_norm3_r=L.BatchNormalization(256), 84 | 85 | l_conv4_1=L.Convolution2D(256,512,(3,3), pad=1), 86 | l_norm4_1=L.BatchNormalization(512), 87 | l_conv4_2=L.Convolution2D(512,512,(3,3), pad=1), 88 | l_norm4_2=L.BatchNormalization(512), 89 | l_conv4_3=L.Convolution2D(512,512,(3,3), pad=1), 90 | l_norm4_3=L.BatchNormalization(512), 91 | l_conv4_r=L.Convolution2D(256,512,(3,3), pad=1), 92 | l_norm4_r=L.BatchNormalization(512), 93 | 94 | l_phi=L.Linear(self.dimension, self.nb_class_train), 95 | ) 96 | return chain 97 | 98 | 99 | # Train methods 100 | # --------------- 101 | 102 | def encoder(self, x, batchsize, train=True): 103 | with chainer.using_config('train', train): 104 | x2 = F.reshape(x, (batchsize,84,84,3)) 105 | x3 = F.transpose(x2, [0,3,1,2]) 106 | 107 | c1_r=self.chain.l_conv1_r(x3) 108 | n1_r=self.chain.l_norm1_r(c1_r) 109 | 110 | c1_1=self.chain.l_conv1_1(x3) 111 | n1_1=self.chain.l_norm1_1(c1_1) 112 | a1_1=F.relu(n1_1) 113 | 114 | c1_2=self.chain.l_conv1_2(a1_1) 115 | n1_2=self.chain.l_norm1_2(c1_2) 116 | a1_2=F.relu(n1_2) 117 | 118 | c1_3=self.chain.l_conv1_3(a1_2) 119 | n1_3=self.chain.l_norm1_3(c1_3) 120 | 121 | a1_3=F.relu(n1_3+n1_r) 122 | 123 | p1=F.max_pooling_2d(a1_3,2) 124 | p1=F.dropout(p1,ratio=0.3) 125 | 126 | c2_r=self.chain.l_conv2_r(p1) 127 | n2_r=self.chain.l_norm2_r(c2_r) 128 | 129 | c2_1=self.chain.l_conv2_1(p1) 130 | n2_1=self.chain.l_norm2_1(c2_1) 131 | a2_1=F.relu(n2_1) 132 | 133 | c2_2=self.chain.l_conv2_2(a2_1) 134 | n2_2=self.chain.l_norm2_2(c2_2) 135 | a2_2=F.relu(n2_2) 136 | 137 | c2_3=self.chain.l_conv2_3(a2_2) 138 | n2_3=self.chain.l_norm2_3(c2_3) 139 | 140 | a2_3=F.relu(n2_3+n2_r) 141 | 142 | p2=F.max_pooling_2d(a2_3,2) 143 | p2=F.dropout(p2, ratio=0.2) 144 | c3_r=self.chain.l_conv3_r(p2) 145 | n3_r=self.chain.l_norm3_r(c3_r) 146 | 147 | c3_1=self.chain.l_conv3_1(p2) 148 | n3_1=self.chain.l_norm3_1(c3_1) 149 | a3_1=F.relu(n3_1) 150 | 151 | c3_2=self.chain.l_conv3_2(a3_1) 152 | n3_2=self.chain.l_norm3_2(c3_2) 153 | a3_2=F.relu(n3_2) 154 | 155 | c3_3=self.chain.l_conv3_3(a3_2) 156 | n3_3=self.chain.l_norm3_3(c3_3) 157 | 158 | a3_3=F.relu(n3_3+n3_r) 159 | 160 | p3=F.max_pooling_2d(a3_3,2) 161 | p3=F.dropout(p3,ratio=0.2) 162 | 163 | c4_r=self.chain.l_conv4_r(p3) 164 | n4_r=self.chain.l_norm4_r(c4_r) 165 | 166 | c4_1=self.chain.l_conv4_1(p3) 167 | n4_1=self.chain.l_norm4_1(c4_1) 168 | a4_1=F.relu(n4_1) 169 | 170 | c4_2=self.chain.l_conv4_2(a4_1) 171 | n4_2=self.chain.l_norm4_2(c4_2) 172 | a4_2=F.relu(n4_2) 173 | 174 | c4_3=self.chain.l_conv4_3(a4_2) 175 | n4_3=self.chain.l_norm4_3(c4_3) 176 | 177 | a4_3=F.relu(n4_3+n4_r) 178 | 179 | p4=F.max_pooling_2d(a4_3,2) 180 | p4=F.dropout(p4, ratio=0.2) 181 | 182 | p5=F.average_pooling_2d(p4,6) 183 | h_t=F.reshape(p5, (batchsize,-1)) 184 | return h_t 185 | 186 | 187 | 188 | def Projection_Space(self, average_key, batchsize, nb_class, train=True, phi_ind=None): 189 | c_t = average_key 190 | eps=1e-6 191 | if train == True: 192 | Phi_tmp = self.chain.l_phi.W 193 | else: 194 | Phi_data = self.chain.l_phi.W.data 195 | Phi_tmp = chainer.Variable(Phi_data[phi_ind,:]) 196 | for i in range(nb_class): 197 | if i == 0: 198 | Phi_sum = Phi_tmp[i] 199 | else: 200 | Phi_sum += Phi_tmp[i] 201 | Phi = nb_class*(Phi_tmp)-F.broadcast_to(Phi_sum,(nb_class,self.dimension)) 202 | 203 | power_Phi = F.sqrt(F.sum(Phi*Phi, axis=1)) 204 | power_Phi = F.transpose(F.broadcast_to(power_Phi, [self.dimension,nb_class])) 205 | 206 | Phi = Phi/(power_Phi+eps) 207 | 208 | power_c = F.sqrt(F.sum(c_t*c_t, axis=1)) 209 | power_c = F.transpose(F.broadcast_to(power_c, [self.dimension,nb_class])) 210 | c_tmp = c_t/(power_c+eps) 211 | 212 | null=Phi - c_tmp 213 | M = nullspace_gpu(null.data) 214 | M = F.broadcast_to(M,[batchsize, self.dimension, self.dimension-nb_class]) 215 | 216 | return M 217 | 218 | def compute_power(self, batchsize,key,M, nb_class, train=True,phi_ind=None): 219 | if train == True: 220 | Phi_out = self.chain.l_phi.W 221 | else: 222 | Phi_data = self.chain.l_phi.W.data 223 | Phi_out = chainer.Variable(Phi_data[phi_ind,:]) 224 | Phi_out_batch = F.broadcast_to(Phi_out,[batchsize,nb_class, self.dimension]) 225 | PhiM = F.batch_matmul(Phi_out_batch,M) 226 | PhiMs = F.sum(PhiM*PhiM,axis=2) 227 | 228 | key_t = F.reshape(key,[batchsize,1,self.dimension]) 229 | keyM = F.batch_matmul(key_t,M) 230 | keyMs = F.sum(keyM*keyM, axis=2) 231 | keyMs = F.broadcast_to(keyMs, [batchsize,nb_class]) 232 | 233 | pow_t = PhiMs + keyMs 234 | 235 | return pow_t 236 | 237 | 238 | def compute_power_avg_phi(self, batchsize, nb_class, average_key, train=False): 239 | avg_pow = F.sum(average_key*average_key,axis=1) 240 | Phi = self.chain.l_phi.W 241 | Phis = F.sum(Phi*Phi,axis=1) 242 | 243 | avg_pow_bd = F.broadcast_to(F.reshape(avg_pow,[len(avg_pow),1]),[len(avg_pow),len(Phis)]) 244 | wzs_bd = F.broadcast_to(F.reshape(Phis,[1,len(Phis)]),[len(avg_pow),len(Phis)]) 245 | 246 | pow_avg = avg_pow_bd + wzs_bd 247 | 248 | return pow_avg 249 | 250 | 251 | def compute_loss(self, t_data, r_t, pow_t, batchsize,nb_class, train=True): 252 | t = chainer.Variable(self.xp.array(t_data, dtype=self.xp.int32)) 253 | u = 2*self.chain.l_phi(r_t)-pow_t 254 | return F.softmax_cross_entropy(u,t) 255 | 256 | def compute_accuracy(self, t_data, r_t, pow_t,batchsize, nb_class, phi_ind=None): 257 | ro = 2*self.chain.l_phi(r_t) 258 | ro_t = chainer.Variable(ro.data[:,phi_ind]) 259 | u = ro_t-pow_t 260 | 261 | t_est = self.xp.argmax(F.softmax(u).data, axis=1) 262 | 263 | return (t_est == self.xp.array(t_data)) 264 | 265 | def select_phi(self, average_key, avg_pow): 266 | u_avg = 2*self.chain.l_phi(average_key).data 267 | u_avg = u_avg - avg_pow.data 268 | u_avg_ind = cp.asnumpy(self.xp.argsort(u_avg, axis=1)) 269 | phi_ind = np.zeros(self.nb_class_test) 270 | for i in range(self.nb_class_test): 271 | if i == 0: 272 | phi_ind[i] = np.int(u_avg_ind[i, self.nb_class_train-1]) 273 | else: 274 | k=self.nb_class_train-1 275 | while u_avg_ind[i,k] in phi_ind[:i]: 276 | k = k-1 277 | phi_ind[i] = np.int(u_avg_ind[i,k]) 278 | return phi_ind.tolist() 279 | 280 | def train(self, images, labels): 281 | """ 282 | Train a minibatch of episodes 283 | """ 284 | images = self.xp.stack(images) 285 | batchsize = images.shape[0] 286 | loss = 0 287 | 288 | key = self.encoder(images, batchsize, train=True) 289 | support_set = key[:self.nb_class_train*self.n_shot,:] 290 | query_set = key[self.nb_class_train*self.n_shot:,:] 291 | average_key = F.mean(F.reshape(support_set,[self.n_shot,self.nb_class_train,-1]),axis=0) 292 | 293 | batchsize_q = len(query_set.data) 294 | M = self.Projection_Space(average_key, batchsize_q, self.nb_class_train) 295 | 296 | r_t = F.reshape(F.batch_matmul(M,F.batch_matmul(M,query_set,transa=True)),(batchsize_q,-1)) 297 | 298 | pow_t = self.compute_power(batchsize_q,query_set,M,self.nb_class_train) 299 | 300 | loss = self.compute_loss(labels[self.nb_class_train*self.n_shot:], r_t, pow_t, batchsize_q,self.nb_class_train) 301 | 302 | self.chain.zerograds() 303 | loss.backward() 304 | self.optimizer.update() 305 | 306 | return loss.data 307 | 308 | 309 | def evaluate(self, images, labels): 310 | """ 311 | Evaluate accuracy score 312 | """ 313 | nb_class = self.nb_class_test 314 | 315 | images = self.xp.stack(images) 316 | batchsize = images.shape[0] 317 | accs = [] 318 | 319 | key= self.encoder(images,batchsize, train=False) 320 | support_set = key[:nb_class*self.n_shot,:] 321 | query_set = key[nb_class*self.n_shot:,:] 322 | average_key = F.mean(F.reshape(support_set,[self.n_shot,nb_class,-1]),axis=0) 323 | batchsize_q = len(query_set.data) 324 | pow_avg = self.compute_power_avg_phi(batchsize_q, nb_class, average_key, train=False) 325 | 326 | phi_ind = [np.int(ind) for ind in self.select_phi(average_key,pow_avg)] 327 | 328 | M = self.Projection_Space(average_key, batchsize_q,nb_class, train=False, phi_ind=phi_ind) 329 | r_t = F.reshape(F.batch_matmul(M,F.batch_matmul(M,query_set,transa=True)),(batchsize_q,-1)) 330 | pow_t = self.compute_power(batchsize_q,query_set,M,nb_class, train=False, phi_ind=phi_ind) 331 | 332 | accs_tmp = self.compute_accuracy(labels[nb_class*self.n_shot:], r_t, pow_t, batchsize_q, nb_class, phi_ind=phi_ind) 333 | 334 | accs.append(accs_tmp) 335 | 336 | 337 | return accs 338 | 339 | 340 | 341 | def decay_learning_rate(self, decaying_parameter=0.5): 342 | self.optimizer.alpha=self.optimizer.alpha*decaying_parameter 343 | -------------------------------------------------------------------------------- /miniImageNet_TapNet/utils/rank_nullspace.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from numpy.linalg import svd 3 | 4 | import cupy as cp 5 | from cupy.linalg import svd as svd_gpu 6 | from cupy import core 7 | 8 | def rank(A, atol=1e-13, rtol=0): 9 | A = np.atleast_2d(A) 10 | s = svd(A, compute_uv=False) 11 | tol = max(atol, rtol*s[0]) 12 | rank = int((s >= tol).sum()) 13 | return rank 14 | 15 | def nullspace(A, tol=1e-13): 16 | A=np.atleast_2d(A) 17 | u, s, vh = svd(A) 18 | if len(A.shape) == 2: 19 | nnz = (s >= tol).sum() 20 | ns = vh[nnz:].conj().T 21 | elif len(A.shape) == 3: 22 | nnz = (s >= tol).sum(axis=-1) 23 | nnz = max(nnz) 24 | ns = np.transpose(vh[:,nnz:,:].conj(), axes=[0,2,1]) 25 | return ns 26 | 27 | def nullspace_gpu(A, tol=1e-13): 28 | A = cp.atleast_2d(A) 29 | u, s, vh =svd_gpu(A) 30 | nnz = (s >= tol).sum() 31 | ns = vh[nnz:].conj().T 32 | return ns 33 | -------------------------------------------------------------------------------- /tieredImageNet_TapNet/data/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /tieredImageNet_TapNet/scripts/train_TapNet_tieredImageNet.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('../') 4 | import argparse 5 | 6 | import numpy as np 7 | import scipy.io as sio 8 | 9 | import chainer.functions as F 10 | from chainer import optimizers 11 | from chainer import cuda 12 | from chainer import serializers 13 | 14 | from utils.generators import tieredImageNetGenerator 15 | from utils.model_TapNet_ResNet12 import TapNet 16 | 17 | if __name__ == '__main__': 18 | parser = argparse.ArgumentParser() 19 | parser.add_argument('--gpu', type=int, default=0, 20 | help='gpu device number. -1 for cpu.') 21 | parser.add_argument('--n_shot', type=int, default=5, 22 | help='Number of shots.') 23 | parser.add_argument('--nb_class_train', type=int, default=20, 24 | help='Number of training classes .') 25 | parser.add_argument('--nb_class_test', type=int, default=5, 26 | help='Number of test classes .') 27 | parser.add_argument('--n_query_train', type=int, default=8, 28 | help='Number of queries per class in training.') 29 | parser.add_argument('--n_query_test', type=int, default=15, 30 | help='Number of queries per class in test.') 31 | parser.add_argument('--wd_rate', type=float, default=0, 32 | help='Weight decay rate in Adam optimizer') 33 | # set params 34 | # ----------- 35 | args = parser.parse_args() 36 | if args.gpu < 0: 37 | xp = np 38 | else: 39 | import cupy as cp 40 | os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" 41 | os.environ["CUDA_VISIBLE_DEVICES"]="%d" %args.gpu 42 | xp = cp 43 | dimension=512 44 | max_iter=50001 45 | lrdecay = True 46 | lrstep = 40000 47 | n_shot=args.n_shot 48 | n_query=args.n_query_train 49 | n_query_test = args.n_query_test 50 | nb_class_train=args.nb_class_train 51 | nb_class_test=args.nb_class_test 52 | wd_rate=args.wd_rate 53 | savefile_name='save/TapNet_tieredImageNet_ResNet12.mat' 54 | filename_5shot='save/TapNet_tieredImageNet_ResNet12' 55 | filename_5shot_last='save/TapNet_tieredImageNet_ResNet12_last' 56 | 57 | # set up training 58 | # ------------------ 59 | model = TapNet(nb_class_train=nb_class_train, nb_class_test=nb_class_test, input_size=3*84*84, 60 | dimension=dimension, n_shot=n_shot, gpu=args.gpu) 61 | 62 | optimizer = optimizers.Adam(alpha=1e-3, weight_decay_rate=wd_rate) 63 | model.set_optimizer(optimizer) 64 | 65 | train_generator = tieredImageNetGenerator(image_file='../data/train_images.npz', label_file='../data/train_labels.pkl', 66 | nb_classes=nb_class_train, nb_samples_per_class=n_shot+n_query, 67 | max_iter=max_iter, xp=xp) 68 | 69 | # Result analysis list 70 | # ----------------- 71 | 72 | loss_h=[] 73 | accuracy_h_val=[] 74 | accuracy_h_test=[] 75 | 76 | acc_best=0 77 | epoch_best=0 78 | 79 | # start training 80 | # ---------------- 81 | 82 | for t, (images, labels) in train_generator: 83 | # train 84 | loss = model.train(images, labels) 85 | # logging 86 | loss_h.extend([loss.tolist()]) 87 | if (t % 50 == 0): 88 | print("Episode: %d, Train Loss: %f "%(t, loss)) 89 | 90 | if (t != 0) and (t % 500 == 0): 91 | print('Evaluation in Validation data') 92 | test_generator = tieredImageNetGenerator(image_file='../data/val_images.npz', label_file='../data/val_labels.pkl', 93 | nb_classes=nb_class_test, nb_samples_per_class=n_shot+n_query_test, 94 | max_iter=600, xp=xp) 95 | scores = [] 96 | for i, (images, labels) in test_generator: 97 | accs = model.evaluate(images, labels) 98 | accs_ = [cuda.to_cpu(acc) for acc in accs] 99 | score = np.asarray(accs_, dtype=int) 100 | scores.append(score) 101 | print(('Accuracy 5 shot ={:.2f}%').format(100*np.mean(np.array(scores)))) 102 | accuracy_t=100*np.mean(np.array(scores)) 103 | 104 | if acc_best < accuracy_t: 105 | acc_best = accuracy_t 106 | epoch_best=t 107 | serializers.save_npz(filename_5shot,model.chain) 108 | 109 | accuracy_h_val.extend([accuracy_t.tolist()]) 110 | del(test_generator) 111 | del(accs) 112 | del(accs_) 113 | del(accuracy_t) 114 | 115 | print('Evaluation in Test data') 116 | test_generator = tieredImageNetGenerator(image_file='../data/test_images.npz', label_file='../data/test_labels.pkl', 117 | nb_classes=nb_class_test, nb_samples_per_class=n_shot+n_query_test, 118 | max_iter=600, xp=xp) 119 | scores = [] 120 | for i, (images, labels) in test_generator: 121 | accs = model.evaluate(images, labels) 122 | accs_ = [cuda.to_cpu(acc) for acc in accs] 123 | score = np.asarray(accs_, dtype=int) 124 | scores.append(score) 125 | print(('Accuracy 5 shot ={:.2f}%').format(100*np.mean(np.array(scores)))) 126 | accuracy_t=100*np.mean(np.array(scores)) 127 | accuracy_h_test.extend([accuracy_t.tolist()]) 128 | del(test_generator) 129 | del(accs) 130 | del(accs_) 131 | del(accuracy_t) 132 | sio.savemat(savefile_name, {'accuracy_h_val':accuracy_h_val, 'accuracy_h_test':accuracy_h_test, 'epoch_best':epoch_best,'acc_best':acc_best}) 133 | if len(accuracy_h_val) >10: 134 | print('***Average accuracy on past 10 evaluation***') 135 | print('Best epoch =',epoch_best,'Best 5 shot acc=',acc_best) 136 | 137 | serializers.save_npz(filename_5shot_last,model.chain) 138 | 139 | if (t != 0) and (t % lrstep == 0) and lrdecay: 140 | model.decay_learning_rate(0.1) 141 | 142 | 143 | accuracy_h5=[] 144 | 145 | serializers.load_npz(filename_5shot, model.chain) 146 | print('Evaluating the best 5shot model...') 147 | for i in range(50): 148 | test_generator = tieredImageNetGenerator(image_file='../data/test_images.npz', label_file='../data/test_labels.pkl', 149 | nb_classes=nb_class_test, nb_samples_per_class=n_shot+n_query_test, 150 | max_iter=600, xp=xp) 151 | scores=[] 152 | for j, (images, labels) in test_generator: 153 | accs = model.evaluate(images, labels) 154 | accs_ = [cuda.to_cpu(acc) for acc in accs] 155 | score = np.asarray(accs_, dtype=int) 156 | scores.append(score) 157 | accuracy_t=100*np.mean(np.array(scores)) 158 | accuracy_h5.extend([accuracy_t.tolist()]) 159 | print(('600 episodes with 15-query accuracy: 5-shot ={:.2f}%').format(accuracy_t)) 160 | del(test_generator) 161 | del(accs) 162 | del(accs_) 163 | del(accuracy_t) 164 | sio.savemat(savefile_name, {'accuracy_h_val':accuracy_h_val, 'accuracy_h_test':accuracy_h_test, 'epoch_best':epoch_best,'acc_best':acc_best, 'accuracy_h5':accuracy_h5}) 165 | print(('Accuracy_test 5 shot ={:.2f}%').format(np.mean(accuracy_h5))) 166 | 167 | 168 | -------------------------------------------------------------------------------- /tieredImageNet_TapNet/utils/__pycache__/generators.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/istarjun/TapNet/87f7c16112bb180a575c8b6345202bbf5b6bb15e/tieredImageNet_TapNet/utils/__pycache__/generators.cpython-36.pyc -------------------------------------------------------------------------------- /tieredImageNet_TapNet/utils/__pycache__/model_TapNet_ResNet12.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/istarjun/TapNet/87f7c16112bb180a575c8b6345202bbf5b6bb15e/tieredImageNet_TapNet/utils/__pycache__/model_TapNet_ResNet12.cpython-36.pyc -------------------------------------------------------------------------------- /tieredImageNet_TapNet/utils/__pycache__/rank_nullspace.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/istarjun/TapNet/87f7c16112bb180a575c8b6345202bbf5b6bb15e/tieredImageNet_TapNet/utils/__pycache__/rank_nullspace.cpython-36.pyc -------------------------------------------------------------------------------- /tieredImageNet_TapNet/utils/generators.py: -------------------------------------------------------------------------------- 1 | """ 2 | This code based on codes from https://github.com/tristandeleu/ntm-one-shot 3 | """ 4 | import numpy as np 5 | import random 6 | import pickle as pkl 7 | 8 | class tieredImageNetGenerator(object): 9 | """tieredImageNetGenerator 10 | 11 | Args: 12 | image_file (str): 'data/train_images.npz' or 'data/test_images.npz' or 'data/val_images.npz' 13 | label_file (str): 'data/train_labels.pkl' or 'data/test_labels.pkl' or 'data/val_labels.pkl' 14 | nb_classes (int): number of classes in an episode 15 | nb_samples_per_class (int): nuber of samples per class in an episode 16 | max_iter (int): max number of episode generation 17 | xp: numpy or cupy 18 | """ 19 | def __init__(self, image_file, label_file, nb_classes=5, nb_samples_per_class=10, 20 | max_iter=None, xp=np): 21 | super(tieredImageNetGenerator, self).__init__() 22 | self.image_file = image_file 23 | self.label_file = label_file 24 | self.nb_classes = nb_classes 25 | self.nb_samples_per_class = nb_samples_per_class 26 | self.max_iter = max_iter 27 | self.xp = xp 28 | self.num_iter = 0 29 | self._load_data(self.image_file, self.label_file) 30 | 31 | def _load_data(self, image_file, label_file): 32 | with np.load(image_file, mmap_mode="r", encoding='latin1') as data: 33 | images=data["images"] 34 | with open(label_file,'rb') as f: 35 | data=pkl.load(f, encoding='bytes') 36 | label_specific = data[b"label_specific"] 37 | label_specific_str = data[b"label_specific_str"] 38 | num_ex = label_specific.shape[0] 39 | ex_ids = np.arange(num_ex) 40 | num_label_cls_specific = len(label_specific_str) 41 | self.label_specific_idict={} 42 | for cc in range(num_label_cls_specific): 43 | self.label_specific_idict[cc]=ex_ids[label_specific == cc] 44 | self.images=images 45 | 46 | def __iter__(self): 47 | return self 48 | 49 | def __next__(self): 50 | return self.next() 51 | 52 | def next(self): 53 | if (self.max_iter is None) or (self.num_iter < self.max_iter): 54 | self.num_iter += 1 55 | images, labels = self.sample(self.nb_classes, self.nb_samples_per_class) 56 | 57 | return (self.num_iter - 1), (images, labels) 58 | else: 59 | raise StopIteration() 60 | 61 | def sample(self, nb_classes, nb_samples_per_class): 62 | sampled_characters = random.sample(self.label_specific_idict.keys(), nb_classes) 63 | labels_and_images = [] 64 | for (k, char) in enumerate(sampled_characters): 65 | _ind = random.sample(list(self.label_specific_idict[char]), nb_samples_per_class) 66 | labels_and_images.extend([(k, self.xp.array(self.images[i]/np.float32(255).flatten())) for i in _ind]) 67 | arg_labels_and_images = [] 68 | for i in range(self.nb_samples_per_class): 69 | for j in range(self.nb_classes): 70 | arg_labels_and_images.extend([labels_and_images[i+j*self.nb_samples_per_class]]) 71 | 72 | labels, images = zip(*arg_labels_and_images) 73 | return images, labels 74 | -------------------------------------------------------------------------------- /tieredImageNet_TapNet/utils/model_TapNet_ResNet12.py: -------------------------------------------------------------------------------- 1 | import cupy as cp 2 | import numpy as np 3 | 4 | import chainer 5 | import chainer.links as L 6 | import chainer.functions as F 7 | from chainer import cuda 8 | 9 | from utils.rank_nullspace import nullspace_gpu 10 | 11 | class TapNet(object): 12 | def __init__(self, nb_class_train, nb_class_test, input_size, dimension, 13 | n_shot, gpu=-1): 14 | """ 15 | Args 16 | nb_class_train (int): number of classes in a training episode 17 | nb_class_test (int): number of classes in a test episode 18 | input_size (int): dimension of input vector 19 | dimension (int) : dimension of embedding space 20 | n_shot (int) : number of shots 21 | """ 22 | self.nb_class_train = nb_class_train 23 | self.nb_class_test = nb_class_test 24 | self.input_size = input_size 25 | self.dimension = dimension 26 | self.n_shot = n_shot 27 | # create chain 28 | self.chain = self._create_chain() 29 | self.set_gpu(gpu) 30 | 31 | 32 | # Set up methods 33 | # --------------- 34 | @property 35 | def xp(self): 36 | if self.gpu<0: 37 | return np 38 | else: 39 | return cp 40 | 41 | def set_gpu(self, gpu): 42 | self.gpu = gpu 43 | if self.gpu < 0: 44 | self.chain.to_cpu() 45 | else: 46 | self.chain.to_gpu() 47 | 48 | 49 | def set_optimizer(self, optimizer): 50 | self.optimizer = optimizer 51 | self.optimizer.setup(self.chain) 52 | self.optimizer.use_cleargrads(use=False) 53 | 54 | 55 | 56 | def _create_chain(self): 57 | chain = chainer.Chain( 58 | l_conv1_1=L.Convolution2D(None,64,(3,3), pad=1), 59 | l_norm1_1=L.BatchNormalization(64), 60 | l_conv1_2=L.Convolution2D(64,64,(3,3), pad=1), 61 | l_norm1_2=L.BatchNormalization(64), 62 | l_conv1_3=L.Convolution2D(64,64,(3,3), pad=1), 63 | l_norm1_3=L.BatchNormalization(64), 64 | l_conv1_r=L.Convolution2D(None,64,(3,3), pad=1), 65 | l_norm1_r=L.BatchNormalization(64), 66 | 67 | l_conv2_1=L.Convolution2D(64,128,(3,3), pad=1), 68 | l_norm2_1=L.BatchNormalization(128), 69 | l_conv2_2=L.Convolution2D(128,128,(3,3), pad=1), 70 | l_norm2_2=L.BatchNormalization(128), 71 | l_conv2_3=L.Convolution2D(128,128,(3,3), pad=1), 72 | l_norm2_3=L.BatchNormalization(128), 73 | l_conv2_r=L.Convolution2D(64,128,(3,3), pad=1), 74 | l_norm2_r=L.BatchNormalization(128), 75 | 76 | l_conv3_1=L.Convolution2D(128,256,(3,3), pad=1), 77 | l_norm3_1=L.BatchNormalization(256), 78 | l_conv3_2=L.Convolution2D(256,256,(3,3), pad=1), 79 | l_norm3_2=L.BatchNormalization(256), 80 | l_conv3_3=L.Convolution2D(256,256,(3,3), pad=1), 81 | l_norm3_3=L.BatchNormalization(256), 82 | l_conv3_r=L.Convolution2D(128,256,(3,3), pad=1), 83 | l_norm3_r=L.BatchNormalization(256), 84 | 85 | l_conv4_1=L.Convolution2D(256,512,(3,3), pad=1), 86 | l_norm4_1=L.BatchNormalization(512), 87 | l_conv4_2=L.Convolution2D(512,512,(3,3), pad=1), 88 | l_norm4_2=L.BatchNormalization(512), 89 | l_conv4_3=L.Convolution2D(512,512,(3,3), pad=1), 90 | l_norm4_3=L.BatchNormalization(512), 91 | l_conv4_r=L.Convolution2D(256,512,(3,3), pad=1), 92 | l_norm4_r=L.BatchNormalization(512), 93 | 94 | l_phi=L.Linear(self.dimension, self.nb_class_train), 95 | ) 96 | return chain 97 | 98 | 99 | # Train methods 100 | # --------------- 101 | 102 | def encoder(self, x, batchsize, train=True): 103 | with chainer.using_config('train', train): 104 | x2 = F.reshape(x, (batchsize,84,84,3)) 105 | x3 = F.transpose(x2, [0,3,1,2]) 106 | 107 | c1_r=self.chain.l_conv1_r(x3) 108 | n1_r=self.chain.l_norm1_r(c1_r) 109 | 110 | c1_1=self.chain.l_conv1_1(x3) 111 | n1_1=self.chain.l_norm1_1(c1_1) 112 | a1_1=F.relu(n1_1) 113 | 114 | c1_2=self.chain.l_conv1_2(a1_1) 115 | n1_2=self.chain.l_norm1_2(c1_2) 116 | a1_2=F.relu(n1_2) 117 | 118 | c1_3=self.chain.l_conv1_3(a1_2) 119 | n1_3=self.chain.l_norm1_3(c1_3) 120 | 121 | a1_3=F.relu(n1_3+n1_r) 122 | 123 | p1=F.max_pooling_2d(a1_3,2) 124 | p1=F.dropout(p1,ratio=0.2) 125 | 126 | c2_r=self.chain.l_conv2_r(p1) 127 | n2_r=self.chain.l_norm2_r(c2_r) 128 | 129 | c2_1=self.chain.l_conv2_1(p1) 130 | n2_1=self.chain.l_norm2_1(c2_1) 131 | a2_1=F.relu(n2_1) 132 | 133 | c2_2=self.chain.l_conv2_2(a2_1) 134 | n2_2=self.chain.l_norm2_2(c2_2) 135 | a2_2=F.relu(n2_2) 136 | 137 | c2_3=self.chain.l_conv2_3(a2_2) 138 | n2_3=self.chain.l_norm2_3(c2_3) 139 | 140 | a2_3=F.relu(n2_3+n2_r) 141 | 142 | p2=F.max_pooling_2d(a2_3,2) 143 | p2=F.dropout(p2, ratio=0.2) 144 | c3_r=self.chain.l_conv3_r(p2) 145 | n3_r=self.chain.l_norm3_r(c3_r) 146 | 147 | c3_1=self.chain.l_conv3_1(p2) 148 | n3_1=self.chain.l_norm3_1(c3_1) 149 | a3_1=F.relu(n3_1) 150 | 151 | c3_2=self.chain.l_conv3_2(a3_1) 152 | n3_2=self.chain.l_norm3_2(c3_2) 153 | a3_2=F.relu(n3_2) 154 | 155 | c3_3=self.chain.l_conv3_3(a3_2) 156 | n3_3=self.chain.l_norm3_3(c3_3) 157 | 158 | a3_3=F.relu(n3_3+n3_r) 159 | 160 | p3=F.max_pooling_2d(a3_3,2) 161 | p3=F.dropout(p3,ratio=0.2) 162 | 163 | c4_r=self.chain.l_conv4_r(p3) 164 | n4_r=self.chain.l_norm4_r(c4_r) 165 | 166 | c4_1=self.chain.l_conv4_1(p3) 167 | n4_1=self.chain.l_norm4_1(c4_1) 168 | a4_1=F.relu(n4_1) 169 | 170 | c4_2=self.chain.l_conv4_2(a4_1) 171 | n4_2=self.chain.l_norm4_2(c4_2) 172 | a4_2=F.relu(n4_2) 173 | 174 | c4_3=self.chain.l_conv4_3(a4_2) 175 | n4_3=self.chain.l_norm4_3(c4_3) 176 | 177 | a4_3=F.relu(n4_3+n4_r) 178 | 179 | p4=F.max_pooling_2d(a4_3,2) 180 | p4=F.dropout(p4, ratio=0.2) 181 | 182 | p5=F.average_pooling_2d(p4,6) 183 | h_t=F.reshape(p5, (batchsize,-1)) 184 | return h_t 185 | 186 | 187 | 188 | def Projection_Space(self, average_key, batchsize, nb_class, train=True, phi_ind=None): 189 | c_t = average_key 190 | eps=1e-6 191 | if train == True: 192 | Phi_tmp = self.chain.l_phi.W 193 | else: 194 | Phi_data = self.chain.l_phi.W.data 195 | Phi_tmp = chainer.Variable(Phi_data[phi_ind,:]) 196 | for i in range(nb_class): 197 | if i == 0: 198 | Phi_sum = Phi_tmp[i] 199 | else: 200 | Phi_sum += Phi_tmp[i] 201 | Phi = nb_class*(Phi_tmp)-F.broadcast_to(Phi_sum,(nb_class,self.dimension)) 202 | 203 | power_Phi = F.sqrt(F.sum(Phi*Phi, axis=1)) 204 | power_Phi = F.transpose(F.broadcast_to(power_Phi, [self.dimension,nb_class])) 205 | 206 | Phi = Phi/(power_Phi+eps) 207 | 208 | power_c = F.sqrt(F.sum(c_t*c_t, axis=1)) 209 | power_c = F.transpose(F.broadcast_to(power_c, [self.dimension,nb_class])) 210 | c_tmp = c_t/(power_c+eps) 211 | 212 | null=Phi - c_tmp 213 | M = nullspace_gpu(null.data) 214 | M = F.broadcast_to(M,[batchsize, self.dimension, self.dimension-nb_class]) 215 | 216 | return M 217 | 218 | def compute_power(self, batchsize,key,M, nb_class, train=True,phi_ind=None): 219 | if train == True: 220 | Phi_out = self.chain.l_phi.W 221 | else: 222 | Phi_data = self.chain.l_phi.W.data 223 | Phi_out = chainer.Variable(Phi_data[phi_ind,:]) 224 | Phi_out_batch = F.broadcast_to(Phi_out,[batchsize,nb_class, self.dimension]) 225 | PhiM = F.batch_matmul(Phi_out_batch,M) 226 | PhiMs = F.sum(PhiM*PhiM,axis=2) 227 | 228 | key_t = F.reshape(key,[batchsize,1,self.dimension]) 229 | keyM = F.batch_matmul(key_t,M) 230 | keyMs = F.sum(keyM*keyM, axis=2) 231 | keyMs = F.broadcast_to(keyMs, [batchsize,nb_class]) 232 | 233 | pow_t = PhiMs + keyMs 234 | 235 | return pow_t 236 | 237 | 238 | def compute_power_avg_phi(self, batchsize, nb_class, average_key, train=False): 239 | avg_pow = F.sum(average_key*average_key,axis=1) 240 | Phi = self.chain.l_phi.W 241 | Phis = F.sum(Phi*Phi,axis=1) 242 | 243 | avg_pow_bd = F.broadcast_to(F.reshape(avg_pow,[len(avg_pow),1]),[len(avg_pow),len(Phis)]) 244 | wzs_bd = F.broadcast_to(F.reshape(Phis,[1,len(Phis)]),[len(avg_pow),len(Phis)]) 245 | 246 | pow_avg = avg_pow_bd + wzs_bd 247 | 248 | return pow_avg 249 | 250 | 251 | def compute_loss(self, t_data, r_t, pow_t, batchsize,nb_class, train=True): 252 | t = chainer.Variable(self.xp.array(t_data, dtype=self.xp.int32)) 253 | u = 2*self.chain.l_phi(r_t)-pow_t 254 | return F.softmax_cross_entropy(u,t) 255 | 256 | def compute_accuracy(self, t_data, r_t, pow_t,batchsize, nb_class, phi_ind=None): 257 | ro = 2*self.chain.l_phi(r_t) 258 | ro_t = chainer.Variable(ro.data[:,phi_ind]) 259 | u = ro_t-pow_t 260 | 261 | t_est = self.xp.argmax(F.softmax(u).data, axis=1) 262 | 263 | return (t_est == self.xp.array(t_data)) 264 | 265 | def select_phi(self, average_key, avg_pow): 266 | u_avg = 2*self.chain.l_phi(average_key).data 267 | u_avg = u_avg - avg_pow.data 268 | u_avg_ind = cp.asnumpy(self.xp.argsort(u_avg, axis=1)) 269 | phi_ind = np.zeros(self.nb_class_test) 270 | for i in range(self.nb_class_test): 271 | if i == 0: 272 | phi_ind[i] = np.int(u_avg_ind[i, self.nb_class_train-1]) 273 | else: 274 | k=self.nb_class_train-1 275 | while u_avg_ind[i,k] in phi_ind[:i]: 276 | k = k-1 277 | phi_ind[i] = np.int(u_avg_ind[i,k]) 278 | return phi_ind.tolist() 279 | 280 | def train(self, images, labels): 281 | """ 282 | Train a minibatch of episodes 283 | """ 284 | images = self.xp.stack(images) 285 | batchsize = images.shape[0] 286 | loss = 0 287 | 288 | key = self.encoder(images, batchsize, train=True) 289 | support_set = key[:self.nb_class_train*self.n_shot,:] 290 | query_set = key[self.nb_class_train*self.n_shot:,:] 291 | average_key = F.mean(F.reshape(support_set,[self.n_shot,self.nb_class_train,-1]),axis=0) 292 | 293 | batchsize_q = len(query_set.data) 294 | M = self.Projection_Space(average_key, batchsize_q, self.nb_class_train) 295 | 296 | r_t = F.reshape(F.batch_matmul(M,F.batch_matmul(M,query_set,transa=True)),(batchsize_q,-1)) 297 | 298 | pow_t = self.compute_power(batchsize_q,query_set,M,self.nb_class_train) 299 | 300 | loss = self.compute_loss(labels[self.nb_class_train*self.n_shot:], r_t, pow_t, batchsize_q,self.nb_class_train) 301 | 302 | self.chain.zerograds() 303 | loss.backward() 304 | self.optimizer.update() 305 | 306 | return loss.data 307 | 308 | 309 | def evaluate(self, images, labels): 310 | """ 311 | Evaluate accuracy score 312 | """ 313 | nb_class = self.nb_class_test 314 | 315 | images = self.xp.stack(images) 316 | batchsize = images.shape[0] 317 | accs = [] 318 | 319 | key= self.encoder(images,batchsize, train=False) 320 | support_set = key[:nb_class*self.n_shot,:] 321 | query_set = key[nb_class*self.n_shot:,:] 322 | average_key = F.mean(F.reshape(support_set,[self.n_shot,nb_class,-1]),axis=0) 323 | batchsize_q = len(query_set.data) 324 | pow_avg = self.compute_power_avg_phi(batchsize_q, nb_class, average_key, train=False) 325 | 326 | phi_ind = [np.int(ind) for ind in self.select_phi(average_key,pow_avg)] 327 | 328 | M = self.Projection_Space(average_key, batchsize_q,nb_class, train=False, phi_ind=phi_ind) 329 | r_t = F.reshape(F.batch_matmul(M,F.batch_matmul(M,query_set,transa=True)),(batchsize_q,-1)) 330 | pow_t = self.compute_power(batchsize_q,query_set,M,nb_class, train=False, phi_ind=phi_ind) 331 | 332 | accs_tmp = self.compute_accuracy(labels[nb_class*self.n_shot:], r_t, pow_t, batchsize_q, nb_class, phi_ind=phi_ind) 333 | 334 | accs.append(accs_tmp) 335 | 336 | 337 | return accs 338 | 339 | 340 | 341 | def decay_learning_rate(self, decaying_parameter=0.5): 342 | self.optimizer.alpha=self.optimizer.alpha*decaying_parameter 343 | -------------------------------------------------------------------------------- /tieredImageNet_TapNet/utils/rank_nullspace.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from numpy.linalg import svd 3 | 4 | import cupy as cp 5 | from cupy.linalg import svd as svd_gpu 6 | from cupy import core 7 | 8 | def rank(A, atol=1e-13, rtol=0): 9 | A = np.atleast_2d(A) 10 | s = svd(A, compute_uv=False) 11 | tol = max(atol, rtol*s[0]) 12 | rank = int((s >= tol).sum()) 13 | return rank 14 | 15 | def nullspace(A, tol=1e-13): 16 | A=np.atleast_2d(A) 17 | u, s, vh = svd(A) 18 | if len(A.shape) == 2: 19 | nnz = (s >= tol).sum() 20 | ns = vh[nnz:].conj().T 21 | elif len(A.shape) == 3: 22 | nnz = (s >= tol).sum(axis=-1) 23 | nnz = max(nnz) 24 | ns = np.transpose(vh[:,nnz:,:].conj(), axes=[0,2,1]) 25 | return ns 26 | 27 | def nullspace_gpu(A, tol=1e-13): 28 | A = cp.atleast_2d(A) 29 | u, s, vh =svd_gpu(A) 30 | nnz = (s >= tol).sum() 31 | ns = vh[nnz:].conj().T 32 | return ns 33 | --------------------------------------------------------------------------------