├── PT-BOSS ├── README.md ├── cifar.py ├── ema.py ├── label_guessor.py ├── lr_scheduler.py ├── model.py ├── model.pyc ├── randaugment.py ├── run_script.sh ├── sampler.py ├── train.py └── transform.py ├── README.md └── TF-BOSS ├── README.md ├── cifarData.sh ├── countTest.py ├── cta ├── __pycache__ │ ├── cta_remixmatch.cpython-36.pyc │ └── cta_remixmatch.cpython-37.pyc ├── cta_fsmixup.py ├── cta_mixmatch.py ├── cta_remixmatch.py └── lib │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-36.pyc │ ├── __init__.cpython-37.pyc │ ├── train.cpython-36.pyc │ └── train.cpython-37.pyc │ └── train.py ├── fixmatch.py ├── fully_supervised ├── fs_baseline.py ├── fs_mixup.py ├── lib │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-37.pyc │ │ └── train.cpython-37.pyc │ ├── data.py │ └── train.py └── runs │ └── all.sh ├── ict.py ├── installData.sh ├── iter.sh ├── libml ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-37.pyc │ ├── augment.cpython-37.pyc │ ├── ctaugment.cpython-37.pyc │ ├── data.cpython-37.pyc │ ├── layers.cpython-37.pyc │ ├── models.cpython-37.pyc │ ├── train.cpython-37.pyc │ └── utils.cpython-37.pyc ├── augment.py ├── ctaugment.py ├── data.py ├── layers.py ├── models.py ├── train.py └── utils.py ├── loopTensor.py ├── mean_teacher.py ├── mixmatch.py ├── mixup.py ├── newTest.py ├── pi_model.py ├── pseudo_label.py ├── remixmatch_no_cta.py ├── requirements.txt ├── run.sh ├── scripts ├── accuracy_table.py ├── aggregate_accuracy.py ├── check_split.py ├── cifar10_iteration.py ├── cifar10_prototypes.py ├── createSummary.py ├── create_datasets.py ├── create_prototypicals.py ├── create_prototypicals.py~ ├── create_split.py ├── create_unlabeled.py ├── extract_accuracy.py ├── inspect_dataset.py ├── iteration1.py ├── submit4.sh ├── svhn_iteration.py ├── svhn_prototypes.py ├── z.pbs └── z2.pbs ├── setup.sh ├── submit.sh ├── svhnData.sh └── third_party └── auto_augment ├── augmentations.py ├── custom_ops.py ├── policies.py ├── shake_drop.py ├── shake_shake.py └── wrn.py /PT-BOSS/README.md: -------------------------------------------------------------------------------- 1 | 2 | # BOSS 3 | 4 | ## Dataset 5 | ## Before training, download cifar-10 dataset: 6 | 7 | mkdir -p dataset && cd dataset 8 | wget -c http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz 9 | tar -xzvf cifar-10-python.tar.gz 10 | 11 | ## Train the model; parameters including balance method and seed, can be set in the train.py flags 12 | bash run_script.sh 13 | -------------------------------------------------------------------------------- /PT-BOSS/ema.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.distributed as dist 3 | 4 | 5 | class EMA(object): 6 | def __init__(self, model, alpha): 7 | self.step = 0 8 | self.model = model 9 | self.alpha = alpha 10 | self.shadow = self.get_model_state() 11 | self.backup = {} 12 | self.param_keys = [k for k, _ in self.model.named_parameters()] 13 | self.buffer_keys = [k for k, _ in self.model.named_buffers()] 14 | 15 | def update_params(self): 16 | decay = min(self.alpha, (self.step + 1) / (self.step + 10)) 17 | state = self.model.state_dict() 18 | for name in self.param_keys: 19 | self.shadow[name].copy_( 20 | decay * self.shadow[name] 21 | + (1 - decay) * state[name] 22 | ) 23 | # for name in self.buffer_keys: 24 | # self.shadow[name].copy_( 25 | # decay * self.shadow[name] 26 | # + (1 - decay) * state[name] 27 | # ) 28 | self.step += 1 29 | 30 | def update_buffer(self): 31 | state = self.model.state_dict() 32 | for name in self.buffer_keys: 33 | self.shadow[name].copy_(state[name]) 34 | 35 | def apply_shadow(self): 36 | self.backup = self.get_model_state() 37 | self.model.load_state_dict(self.shadow) 38 | 39 | def restore(self): 40 | self.model.load_state_dict(self.backup) 41 | 42 | def get_model_state(self): 43 | return { 44 | k: v.clone().detach() 45 | for k, v in self.model.state_dict().items() 46 | } 47 | 48 | 49 | 50 | if __name__ == '__main__': 51 | 52 | print('=====') 53 | model = torch.nn.BatchNorm1d(5) 54 | ema = EMA(model, 0.9, 0.02, 0.002) 55 | inten = torch.randn(10, 5) 56 | out = model(inten) 57 | ema.update_params() 58 | print(model.state_dict()) 59 | ema.update_buffer() 60 | print(model.state_dict()) 61 | -------------------------------------------------------------------------------- /PT-BOSS/label_guessor.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import math 3 | import numpy as np 4 | 5 | class LabelGuessor(object): 6 | 7 | def __init__(self, thresh): 8 | self.thresh = thresh 9 | 10 | def __call__(self, model, ims, balance, delT): 11 | org_state = { 12 | k: v.clone().detach() 13 | for k, v in model.state_dict().items() 14 | } 15 | is_train = model.training 16 | with torch.no_grad(): 17 | model.train() 18 | all_probs = [] 19 | logits = model(ims) 20 | probs = torch.softmax(logits, dim=1) 21 | scores, lbs = torch.max(probs, dim=1) 22 | # print("lbs ", lbs) 23 | # print("scores ", scores) 24 | mask = torch.ones_like(lbs,dtype=torch.float) 25 | labels, counts = np.unique(lbs.cpu(),return_counts=True) 26 | # print("labels", labels) 27 | # print("counts ", counts) 28 | # count_unlabels = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] 29 | # for i in range(len(labels)): 30 | # count_unlabels[labels[i]] = counts[i] 31 | mxCount = max(counts) 32 | stdClass = np.std(counts) 33 | # print("stdClass ", stdClass) 34 | if balance > 0: 35 | if balance == 1 or balance == 4: 36 | idx = (mask == 0.0) 37 | else: 38 | idx = (scores > self.thresh) 39 | delT = 0 40 | if mxCount > 0: 41 | ratios = [x/mxCount for x in counts] 42 | for i in range(len(labels)): 43 | tmp = (scores*(lbs==labels[i]).float()).ge(self.thresh - delT*(1-ratios[i])) # Which elements 44 | idx = idx | tmp 45 | if balance > 2: 46 | labels, counts = np.unique(lbs[idx].cpu(),return_counts=True) 47 | ratio = torch.zeros_like(mask,dtype=torch.float) 48 | for i in range(len(labels)): 49 | ratio += ((1/counts[i])*(lbs==labels[i]).float()) # Magnitude of mask elements 50 | Z = torch.sum(mask[idx]) 51 | # print("ratio ",ratio) 52 | mask = ratio[idx] 53 | if Z > 0: 54 | mask = Z * mask / torch.sum(mask) 55 | else: 56 | idx = (scores > self.thresh) 57 | mask[idx] = 1.0 58 | else: 59 | idx = scores > self.thresh 60 | mask = mask[idx] 61 | lbs = lbs[idx] 62 | # print("1. lbs ", lbs) 63 | # print("2. mask ", mask) 64 | 65 | model.load_state_dict(org_state) 66 | if is_train: 67 | model.train() 68 | else: 69 | model.eval() 70 | return lbs.detach(), idx, mask, stdClass 71 | 72 | -------------------------------------------------------------------------------- /PT-BOSS/lr_scheduler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- encoding: utf-8 -*- 3 | 4 | import torch 5 | from torch.optim.lr_scheduler import _LRScheduler 6 | import numpy as np 7 | 8 | 9 | class WarmupExpLrScheduler(_LRScheduler): 10 | def __init__( 11 | self, 12 | optimizer, 13 | power, 14 | step_interval=1, 15 | warmup_iter=500, 16 | warmup_ratio=5e-4, 17 | warmup='exp', 18 | last_epoch=-1, 19 | ): 20 | self.power = power 21 | self.step_interval = step_interval 22 | self.warmup_iter = warmup_iter 23 | self.warmup_ratio = warmup_ratio 24 | self.warmup = warmup 25 | super(WarmupExpLrScheduler, self).__init__(optimizer, last_epoch) 26 | 27 | def get_lr(self): 28 | ratio = self.get_lr_ratio() 29 | lrs = [ratio * lr for lr in self.base_lrs] 30 | return lrs 31 | 32 | def get_lr_ratio(self): 33 | if self.last_epoch < self.warmup_iter: 34 | ratio = self.get_warmup_ratio() 35 | else: 36 | real_iter = self.last_epoch - self.warmup_iter 37 | ratio = self.power ** (real_iter // self.step_interval) 38 | return ratio 39 | 40 | def get_warmup_ratio(self): 41 | assert self.warmup in ('linear', 'exp') 42 | alpha = self.last_epoch / self.warmup_iter 43 | if self.warmup == 'linear': 44 | ratio = self.warmup_ratio + (1 - self.warmup_ratio) * alpha 45 | elif self.warmup == 'exp': 46 | ratio = self.warmup_ratio ** (1. - alpha) 47 | return ratio 48 | 49 | 50 | class WarmupPolyLrScheduler(_LRScheduler): 51 | def __init__( 52 | self, 53 | optimizer, 54 | power, 55 | max_iter, 56 | warmup_iter, 57 | warmup_ratio=5e-4, 58 | warmup='exp', 59 | last_epoch=-1, 60 | ): 61 | self.power = power 62 | self.max_iter = max_iter 63 | self.warmup_iter = warmup_iter 64 | self.warmup_ratio = warmup_ratio 65 | self.warmup = warmup 66 | super(WarmupPolyLrScheduler, self).__init__(optimizer, last_epoch) 67 | 68 | def get_lr(self): 69 | ratio = self.get_lr_ratio() 70 | lrs = [ratio * lr for lr in self.base_lrs] 71 | return lrs 72 | 73 | def get_lr_ratio(self): 74 | if self.last_epoch < self.warmup_iter: 75 | ratio = self.get_warmup_ratio() 76 | else: 77 | real_iter = self.last_epoch - self.warmup_iter 78 | real_max_iter = self.max_iter - self.warmup_iter 79 | alpha = real_iter / real_max_iter 80 | ratio = (1 - alpha) ** self.power 81 | return ratio 82 | 83 | def get_warmup_ratio(self): 84 | assert self.warmup in ('linear', 'exp') 85 | alpha = self.last_epoch / self.warmup_iter 86 | if self.warmup == 'linear': 87 | ratio = self.warmup_ratio + (1 - self.warmup_ratio) * alpha 88 | elif self.warmup == 'exp': 89 | ratio = self.warmup_ratio ** (1. - alpha) 90 | return ratio 91 | 92 | 93 | class WarmupCosineLrScheduler(_LRScheduler): 94 | ''' 95 | This is different from official definition, this is implemented according to 96 | the paper of fix-match 97 | ''' 98 | def __init__( 99 | self, 100 | optimizer, 101 | max_iter, 102 | warmup_iter, 103 | warmup_ratio=5e-4, 104 | warmup='exp', 105 | last_epoch=-1, 106 | ): 107 | self.max_iter = max_iter 108 | self.warmup_iter = warmup_iter 109 | self.warmup_ratio = warmup_ratio 110 | self.warmup = warmup 111 | super(WarmupCosineLrScheduler, self).__init__(optimizer, last_epoch) 112 | 113 | def get_lr(self): 114 | ratio = self.get_lr_ratio() 115 | lrs = [ratio * lr for lr in self.base_lrs] 116 | return lrs 117 | 118 | def get_lr_ratio(self): 119 | if self.last_epoch < self.warmup_iter: 120 | ratio = self.get_warmup_ratio() 121 | else: 122 | real_iter = self.last_epoch - self.warmup_iter 123 | real_max_iter = self.max_iter - self.warmup_iter 124 | ratio = np.cos((7 * np.pi * real_iter) / (16 * real_max_iter)) 125 | return ratio 126 | 127 | def get_warmup_ratio(self): 128 | assert self.warmup in ('linear', 'exp') 129 | alpha = self.last_epoch / self.warmup_iter 130 | if self.warmup == 'linear': 131 | ratio = self.warmup_ratio + (1 - self.warmup_ratio) * alpha 132 | elif self.warmup == 'exp': 133 | ratio = self.warmup_ratio ** (1. - alpha) 134 | return ratio 135 | 136 | 137 | if __name__ == "__main__": 138 | model = torch.nn.Conv2d(3, 16, 3, 1, 1) 139 | optim = torch.optim.SGD(model.parameters(), lr=1e-3) 140 | 141 | max_iter = 20000 142 | lr_scheduler = WarmupCosineLrScheduler(optim, max_iter, 0, 0.1, 'linear', -1) 143 | lrs = [] 144 | for _ in range(max_iter): 145 | lr = lr_scheduler.get_lr()[0] 146 | print(lr) 147 | lrs.append(lr) 148 | lr_scheduler.step() 149 | import matplotlib 150 | import matplotlib.pyplot as plt 151 | import numpy as np 152 | lrs = np.array(lrs) 153 | n_lrs = len(lrs) 154 | plt.plot(np.arange(n_lrs), lrs) 155 | plt.grid() 156 | plt.show() 157 | 158 | 159 | -------------------------------------------------------------------------------- /PT-BOSS/model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- encoding: utf-8 -*- 3 | 4 | import torch 5 | import torch.nn as nn 6 | 7 | from torch.nn import BatchNorm2d 8 | 9 | 10 | ''' 11 | As in the paper, the wide resnet only considers the resnet of the pre-activated version, and it only considers the basic blocks rather than the bottleneck blocks. 12 | ''' 13 | 14 | 15 | class BasicBlockPreAct(nn.Module): 16 | def __init__( 17 | self, in_chan, out_chan, drop_rate=0, stride=1, pre_res_act=False 18 | ): 19 | super(BasicBlockPreAct, self).__init__() 20 | self.bn1 = BatchNorm2d(in_chan, momentum=0.001) 21 | self.relu1 = nn.LeakyReLU(inplace=True, negative_slope=0.1) 22 | self.conv1 = nn.Conv2d( 23 | in_chan, 24 | out_chan, 25 | kernel_size=3, 26 | stride=stride, 27 | padding=1, 28 | bias=False 29 | ) 30 | self.bn2 = BatchNorm2d(out_chan, momentum=0.001) 31 | self.relu2 = nn.LeakyReLU(inplace=True, negative_slope=0.1) 32 | self.dropout = nn.Dropout(drop_rate) if not drop_rate == 0 else None 33 | self.conv2 = nn.Conv2d( 34 | out_chan, 35 | out_chan, 36 | kernel_size=3, 37 | stride=1, 38 | padding=1, 39 | bias=False 40 | ) 41 | self.downsample = None 42 | if in_chan != out_chan or stride != 1: 43 | self.downsample = nn.Conv2d( 44 | in_chan, out_chan, kernel_size=1, stride=stride, bias=False 45 | ) 46 | self.pre_res_act = pre_res_act 47 | self.init_weight() 48 | 49 | def forward(self, x): 50 | bn1 = self.bn1(x) 51 | act1 = self.relu1(bn1) 52 | residual = self.conv1(act1) 53 | residual = self.bn2(residual) 54 | residual = self.relu2(residual) 55 | if not self.dropout is None: 56 | residual = self.dropout(residual) 57 | residual = self.conv2(residual) 58 | 59 | shortcut = act1 if self.pre_res_act else x 60 | if self.downsample is not None: 61 | shortcut = self.downsample(shortcut) 62 | 63 | out = shortcut + residual 64 | return out 65 | 66 | def init_weight(self): 67 | for _, md in self.named_modules(): 68 | if isinstance(md, nn.Conv2d): 69 | nn.init.kaiming_normal_( 70 | md.weight, a=0, mode='fan_in', nonlinearity='leaky_relu') 71 | if not md.bias is None: nn.init.constant_(md.bias, 0) 72 | 73 | 74 | 75 | class WideResnetBackbone(nn.Module): 76 | def __init__(self, k=1, n=28, drop_rate=0): 77 | super(WideResnetBackbone, self).__init__() 78 | self.k, self.n = k, n 79 | assert (self.n - 4) % 6 == 0 80 | n_blocks = (self.n - 4) // 6 81 | n_layers = [16,] + [self.k*16*(2**i) for i in range(3)] 82 | 83 | self.conv1 = nn.Conv2d( 84 | 3, 85 | n_layers[0], 86 | kernel_size=3, 87 | stride=1, 88 | padding=1, 89 | bias=False 90 | ) 91 | self.layer1 = self.create_layer( 92 | n_layers[0], 93 | n_layers[1], 94 | bnum=n_blocks, 95 | stride=1, 96 | drop_rate=drop_rate, 97 | pre_res_act=True, 98 | ) 99 | self.layer2 = self.create_layer( 100 | n_layers[1], 101 | n_layers[2], 102 | bnum=n_blocks, 103 | stride=2, 104 | drop_rate=drop_rate, 105 | pre_res_act=False, 106 | ) 107 | self.layer3 = self.create_layer( 108 | n_layers[2], 109 | n_layers[3], 110 | bnum=n_blocks, 111 | stride=2, 112 | drop_rate=drop_rate, 113 | pre_res_act=False, 114 | ) 115 | self.bn_last = BatchNorm2d(n_layers[3], momentum=0.001) 116 | self.relu_last = nn.LeakyReLU(inplace=True, negative_slope=0.1) 117 | self.init_weight() 118 | 119 | def create_layer( 120 | self, 121 | in_chan, 122 | out_chan, 123 | bnum, 124 | stride=1, 125 | drop_rate=0, 126 | pre_res_act=False, 127 | ): 128 | layers = [ 129 | BasicBlockPreAct( 130 | in_chan, 131 | out_chan, 132 | drop_rate=drop_rate, 133 | stride=stride, 134 | pre_res_act=pre_res_act),] 135 | for _ in range(bnum-1): 136 | layers.append( 137 | BasicBlockPreAct( 138 | out_chan, 139 | out_chan, 140 | drop_rate=drop_rate, 141 | stride=1, 142 | pre_res_act=False,)) 143 | return nn.Sequential(*layers) 144 | 145 | def forward(self, x): 146 | feat = self.conv1(x) 147 | 148 | feat = self.layer1(feat) 149 | feat2 = self.layer2(feat) # 1/2 150 | feat4 = self.layer3(feat2) # 1/4 151 | 152 | feat4 = self.bn_last(feat4) 153 | feat4 = self.relu_last(feat4) 154 | return feat2, feat4 155 | 156 | def init_weight(self): 157 | for _, child in self.named_children(): 158 | if isinstance(child, nn.Conv2d): 159 | # n = child.kernel_size[0] * child.kernel_size[0] * child.out_channels 160 | # nn.init.normal_(child.weight, 0, (2./n)**0.5) 161 | nn.init.kaiming_normal_( 162 | child.weight, a=0, mode='fan_in', nonlinearity='leaky_relu' 163 | ) 164 | if not child.bias is None: nn.init.constant_(child.bias, 0) 165 | 166 | 167 | class WideResnet(nn.Module): 168 | ''' 169 | for wide-resnet-28-10, the definition should be WideResnet(n_classes, 10, 28) 170 | ''' 171 | def __init__(self, n_classes, k=1, n=28): 172 | super(WideResnet, self).__init__() 173 | self.n_layers, self.k = n, k 174 | self.backbone = WideResnetBackbone(k=k, n=n) 175 | self.classifier = nn.Linear(64 * self.k, n_classes, bias=True) 176 | 177 | def forward(self, x): 178 | feat = self.backbone(x)[-1] 179 | feat = torch.mean(feat, dim=(2, 3)) 180 | feat = self.classifier(feat) 181 | return feat 182 | 183 | def init_weight(self): 184 | nn.init.xavier_normal_(self.classifier.weight) 185 | if not self.classifier.bias is None: 186 | nn.init.constant_(self.classifier.bias, 0) 187 | 188 | 189 | 190 | if __name__ == "__main__": 191 | x = torch.randn(2, 3, 224, 224) 192 | lb = torch.randint(0, 10, (2, )).long() 193 | 194 | net = WideResnetBackbone() 195 | out = net(x) 196 | print(out[0].size()) 197 | del net, out 198 | 199 | net = WideResnet(n_classes=10) 200 | criteria = nn.CrossEntropyLoss() 201 | out = net(x) 202 | loss = criteria(out, lb) 203 | loss.backward() 204 | print(out.size()) 205 | -------------------------------------------------------------------------------- /PT-BOSS/model.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lnsmith54/BOSS/45a202c93ebcab8e9c033cc1ac20ba7e1849fff9/PT-BOSS/model.pyc -------------------------------------------------------------------------------- /PT-BOSS/run_script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | CUDA_VISIBLE_DEVICES=0 python train.py --n-labeled 10 --n-epochs 512 --batchsize 32 --mu 7 --thr 0.95 --lam-u 1 --lr 0.04 --weight-decay 5e-4 --momentum 0.85 --seed 6 --balance 4 3 | 4 | -------------------------------------------------------------------------------- /PT-BOSS/transform.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import cv2 4 | 5 | 6 | class PadandRandomCrop(object): 7 | ''' 8 | Input tensor is expected to have shape of (H, W, 3) 9 | ''' 10 | def __init__(self, border=4, cropsize=(32, 32)): 11 | self.border = border 12 | self.cropsize = cropsize 13 | 14 | def __call__(self, im): 15 | borders = [(self.border, self.border), (self.border, self.border), (0, 0)] 16 | convas = np.pad(im, borders, mode='reflect') 17 | H, W, C = convas.shape 18 | h, w = self.cropsize 19 | dh, dw = max(0, H-h), max(0, W-w) 20 | sh, sw = np.random.randint(0, dh), np.random.randint(0, dw) 21 | out = convas[sh:sh+h, sw:sw+w, :] 22 | return out 23 | 24 | 25 | class RandomHorizontalFlip(object): 26 | def __init__(self, p=0.5): 27 | self.p = p 28 | 29 | def __call__(self, im): 30 | if np.random.rand() < self.p: 31 | im = im[:, ::-1, :] 32 | return im 33 | 34 | 35 | class Resize(object): 36 | def __init__(self, size): 37 | self.size = size 38 | 39 | def __call__(self, im): 40 | im = cv2.resize(im, self.size) 41 | return im 42 | 43 | 44 | class Normalize(object): 45 | ''' 46 | Inputs are pixel values in range of [0, 255], channel order is 'rgb' 47 | ''' 48 | def __init__(self, mean, std): 49 | self.mean = np.array(mean, np.float32).reshape(1, 1, -1) 50 | self.std = np.array(std, np.float32).reshape(1, 1, -1) 51 | 52 | def __call__(self, im): 53 | if len(im.shape) == 4: 54 | mean, std = self.mean[None, ...], self.std[None, ...] 55 | elif len(im.shape) == 3: 56 | mean, std = self.mean, self.std 57 | im = im.astype(np.float32) / 255. 58 | im -= mean 59 | im /= std 60 | return im 61 | 62 | 63 | class ToTensor(object): 64 | def __init__(self): 65 | pass 66 | 67 | def __call__(self, im): 68 | if len(im.shape) == 4: 69 | return torch.from_numpy(im.transpose(0, 3, 1, 2)) 70 | elif len(im.shape) == 3: 71 | return torch.from_numpy(im.transpose(2, 0, 1)) 72 | 73 | 74 | class Compose(object): 75 | def __init__(self, ops): 76 | self.ops = ops 77 | 78 | def __call__(self, im): 79 | for op in self.ops: 80 | im = op(im) 81 | return im 82 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BOSS 2 | This repository provides the code for replicating the experiments in the paper "Building One-Shot Semi-supervised (BOSS) Learning up to Fully Supervised Performance" 3 | 4 | This repository contains two sets of codes: one for the TensorFlow version that is contained in the folder TF-FixMatch and the other for the PyTorch version that is contained in PT-FixMatch. Both folders contain README files that describe how to install dependencies, setup the datasets, and run the codes to replicate our experiments. 5 | 6 | Additional information is available in the Appendix of our paper. 7 | 8 | ## License 9 | 10 | This code is made freely available to academic and non-academic entities for non-commercial purposes such as academic research, teaching, scientific publications, or personal experimentation. Permission is granted to use the data given that you agree: 11 | 1. That the code comes “AS IS”, without express or implied warranty. Although every effort has been made to ensure accuracy, we (NRL) do not accept any responsibility for errors or omissions. 12 | 2. That you include a reference to the BOSS paper in any work that makes use of this code. 13 | 3. You may not use the codebase or any derivative work for commercial purposes such as, for example, licensing or selling the software, or using the software with a purpose to procure a commercial gain. 14 | 4. That all rights not expressly granted to you are reserved by us (NRL). 15 | 16 | ## Citation 17 | 18 | When using the dataset or code, please cite our [paper](https://arxiv.org/abs/2006.09363): 19 | ``` 20 | @article{smith2020building, 21 | title={Building One-Shot Semi-supervised (BOSS) Learning up to Fully Supervised Performance}, 22 | author={Smith, Leslie N and Conovaloff, Adam}, 23 | journal={arXiv preprint arXiv:2006.09363}, 24 | year={2020} 25 | } 26 | ``` 27 | 28 | 29 | 30 | ## Acknowledgements 31 | 32 | The codebase is heavily based off [FixMatch](https://github.com/google-research/fixmatch) and [FixMatch-pytorch](https://github.com/CoinCheung/fixmatch-pytorch). Both are great repositories - have a look! 33 | -------------------------------------------------------------------------------- /TF-BOSS/README.md: -------------------------------------------------------------------------------- 1 | # Instructions to run BOSS methodology using Tensorflow 2 | 3 | # Setup Fixmatch 4 | sudo apt install python3-dev python3-virtualenv python3-tk imagemagick 5 | virtualenv -p python3 --system-site-packages env3 6 | . env3/bin/activate 7 | pip install -r requirements.txt 8 | 9 | # Install datasets 10 | setup.sh #currently set for cirfar10, uncomment necessary lines for other datasets 11 | #for more instructions, see fixmatch github repo: https://github.com/google-research/fixmatch 12 | 13 | # designate prototypes 14 | svhnData.sh #and/or 15 | cifarData.sh 16 | 17 | # run BOSS 18 | run.sh #dataset, seed, balance method and hyperparameters can be set in-script 19 | -------------------------------------------------------------------------------- /TF-BOSS/cifarData.sh: -------------------------------------------------------------------------------- 1 | export ML_DATA="./data" 2 | export PYTHONPATH=$PYTHONPATH:$PWD 3 | 4 | cp $ML_DATA/cifar10-test.tfrecord $ML_DATA/cifar10p-test.tfrecord 5 | cp $ML_DATA/cifar10-train.tfrecord $ML_DATA/cifar10p-train.tfrecord 6 | 7 | mkdir data/pseudolabeled 8 | mkdir data/pseudolabeled/top 9 | cp data/cifar10p-train.tfrecord data/pseudolabeled/ 10 | 11 | mkdir data/pseudolabeled/SSL2 12 | cp data/cifar10p-train.tfrecord data/pseudolabeled/ 13 | cp data/cifar10p-train.tfrecord data/pseudolabeled/cifar10pB1-train.tfrecord 14 | cp data/cifar10p-test.tfrecord data/pseudolabeled/cifar10pB1-test.tfrecord 15 | cp data/SSL2/cifar10p-unlabel.* data/pseudolabeled/SSL2/ 16 | 17 | cp data/pseudolabeled/SSL2/cifar10p-unlabel.json data/pseudolabeled/SSL2/cifar10pB1-unlabel.json 18 | cp data/pseudolabeled/SSL2/cifar10p-unlabel.tfrecord data/pseudolabeled/SSL2/cifar10pB1-unlabel.tfrecord 19 | cp data/pseudolabeled/SSL2/cifar10p-unlabel.json data/pseudolabeled/SSL2/cifar10pB4-unlabel.json 20 | cp data/pseudolabeled/SSL2/cifar10p-unlabel.tfrecord data/pseudolabeled/SSL2/cifar10pB4-unlabel.tfrecord 21 | 22 | cp $ML_DATA/SSL2/cifar10-unlabel.json $ML_DATA/SSL2/cifar10p-unlabel.json 23 | cp $ML_DATA/SSL2/cifar10-unlabel.tfrecord $ML_DATA/SSL2/cifar10p-unlabel.tfrecord 24 | 25 | 26 | date 27 | size=10 28 | for seed in 1 2 3 4 5 6 7; do 29 | # CUDA_VISIBLE_DEVICES= scripts/cifar10_prototypes.py --seed=$seed --size=$size $ML_DATA/SSL2/cifar10p $ML_DATA/cifar10-train.tfrecord 30 | python scripts/cifar10_prototypes.py --seed=$seed --size=$size $ML_DATA/SSL2/cifar10p $ML_DATA/cifar10-train.tfrecord 31 | done 32 | 33 | date 34 | 35 | exit 36 | 37 | -------------------------------------------------------------------------------- /TF-BOSS/countTest.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | pLabels = np.array([0, 1, 2, 3, 2, 1, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3]) 5 | print("pLabels ",pLabels) 6 | 7 | indx = [np.zeros_like(pLabels,dtype=int)] 8 | for i in range(1,4): 9 | tmp=i*np.ones_like(pLabels,dtype=int) 10 | indx = np.concatenate((indx,[tmp]), axis=0) 11 | pLabels = np.tile(pLabels,(4,1)) 12 | print("pLabels ",pLabels) 13 | print("indx ",indx) 14 | 15 | comparing = tf.math.equal(pLabels,indx) 16 | c=tf.math.reduce_sum(tf.cast(comparing,tf.float32),axis=1) 17 | 18 | with tf.Session() as sess: 19 | counts = sess.run(c) 20 | print(counts) -------------------------------------------------------------------------------- /TF-BOSS/cta/__pycache__/cta_remixmatch.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lnsmith54/BOSS/45a202c93ebcab8e9c033cc1ac20ba7e1849fff9/TF-BOSS/cta/__pycache__/cta_remixmatch.cpython-36.pyc -------------------------------------------------------------------------------- /TF-BOSS/cta/__pycache__/cta_remixmatch.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lnsmith54/BOSS/45a202c93ebcab8e9c033cc1ac20ba7e1849fff9/TF-BOSS/cta/__pycache__/cta_remixmatch.cpython-37.pyc -------------------------------------------------------------------------------- /TF-BOSS/cta/cta_fsmixup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | 17 | from absl import app 18 | from absl import flags 19 | 20 | from cta.lib.train import CTAClassifyFullySupervised 21 | from fully_supervised.fs_mixup import FSMixup 22 | from fully_supervised.lib import data 23 | from libml import utils 24 | 25 | FLAGS = flags.FLAGS 26 | 27 | 28 | class CTAFSMixup(FSMixup, CTAClassifyFullySupervised): 29 | pass 30 | 31 | 32 | def main(argv): 33 | utils.setup_main() 34 | del argv # Unused. 35 | dataset = data.DATASETS()[FLAGS.dataset]() 36 | log_width = utils.ilog2(dataset.width) 37 | model = CTAFSMixup( 38 | os.path.join(FLAGS.train_dir, dataset.name, CTAFSMixup.cta_name()), 39 | dataset, 40 | lr=FLAGS.lr, 41 | wd=FLAGS.wd, 42 | arch=FLAGS.arch, 43 | batch=FLAGS.batch, 44 | nclass=dataset.nclass, 45 | ema=FLAGS.ema, 46 | beta=FLAGS.beta, 47 | dropout=FLAGS.dropout, 48 | 49 | scales=FLAGS.scales or (log_width - 2), 50 | filters=FLAGS.filters, 51 | repeat=FLAGS.repeat) 52 | model.train(FLAGS.train_kimg << 10, FLAGS.report_kimg << 10) 53 | 54 | 55 | if __name__ == '__main__': 56 | utils.setup_tf() 57 | flags.DEFINE_float('wd', 0.002, 'Weight decay.') 58 | flags.DEFINE_float('ema', 0.999, 'Exponential moving average of params.') 59 | flags.DEFINE_float('beta', 0.5, 'Mixup beta distribution.') 60 | flags.DEFINE_float('dropout', 0, 'Dropout on embedding layer.') 61 | flags.DEFINE_integer('scales', 0, 'Number of 2x2 downscalings in the classifier.') 62 | flags.DEFINE_integer('filters', 32, 'Filter size of convolutions.') 63 | flags.DEFINE_integer('repeat', 4, 'Number of residual layers per stage.') 64 | FLAGS.set_default('augment', 'd.d') 65 | FLAGS.set_default('dataset', 'cifar10-1') 66 | FLAGS.set_default('batch', 64) 67 | FLAGS.set_default('lr', 0.002) 68 | FLAGS.set_default('train_kimg', 1 << 16) 69 | app.run(main) 70 | -------------------------------------------------------------------------------- /TF-BOSS/cta/cta_mixmatch.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | 17 | from absl import app 18 | from absl import flags 19 | 20 | from cta.lib.train import CTAClassifySemi 21 | from libml import data, utils 22 | from mixmatch import MixMatch 23 | 24 | FLAGS = flags.FLAGS 25 | 26 | 27 | class CTAMixMatch(MixMatch, CTAClassifySemi): 28 | pass 29 | 30 | 31 | def main(argv): 32 | utils.setup_main() 33 | del argv # Unused. 34 | dataset = data.PAIR_DATASETS()[FLAGS.dataset]() 35 | log_width = utils.ilog2(dataset.width) 36 | model = CTAMixMatch( 37 | os.path.join(FLAGS.train_dir, dataset.name, CTAMixMatch.cta_name()), 38 | dataset, 39 | lr=FLAGS.lr, 40 | wd=FLAGS.wd, 41 | arch=FLAGS.arch, 42 | batch=FLAGS.batch, 43 | nclass=dataset.nclass, 44 | ema=FLAGS.ema, 45 | beta=FLAGS.beta, 46 | w_match=FLAGS.w_match, 47 | scales=FLAGS.scales or (log_width - 2), 48 | filters=FLAGS.filters, 49 | repeat=FLAGS.repeat) 50 | model.train(FLAGS.train_kimg << 10, FLAGS.report_kimg << 10) 51 | 52 | 53 | if __name__ == '__main__': 54 | utils.setup_tf() 55 | flags.DEFINE_float('wd', 0.02, 'Weight decay.') 56 | flags.DEFINE_float('ema', 0.999, 'Exponential moving average of params.') 57 | flags.DEFINE_float('beta', 0.75, 'Mixup beta distribution.') 58 | flags.DEFINE_float('w_match', 100, 'Weight for distribution matching loss.') 59 | flags.DEFINE_integer('scales', 0, 'Number of 2x2 downscalings in the classifier.') 60 | flags.DEFINE_integer('filters', 32, 'Filter size of convolutions.') 61 | flags.DEFINE_integer('repeat', 4, 'Number of residual layers per stage.') 62 | FLAGS.set_default('augment', 'd.d.d') 63 | FLAGS.set_default('dataset', 'cifar10.3@250-5000') 64 | FLAGS.set_default('batch', 64) 65 | FLAGS.set_default('lr', 0.002) 66 | FLAGS.set_default('train_kimg', 1 << 16) 67 | app.run(main) 68 | -------------------------------------------------------------------------------- /TF-BOSS/cta/cta_remixmatch.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | 17 | from absl import app 18 | from absl import flags 19 | 20 | from cta.lib.train import CTAClassifySemi 21 | from libml import utils, data 22 | from remixmatch_no_cta import ReMixMatch 23 | 24 | FLAGS = flags.FLAGS 25 | 26 | 27 | class CTAReMixMatch(ReMixMatch, CTAClassifySemi): 28 | pass 29 | 30 | 31 | def main(argv): 32 | utils.setup_main() 33 | del argv # Unused. 34 | dataset = data.MANY_DATASETS()[FLAGS.dataset]() 35 | log_width = utils.ilog2(dataset.width) 36 | model = CTAReMixMatch( 37 | os.path.join(FLAGS.train_dir, dataset.name, CTAReMixMatch.cta_name()), 38 | dataset, 39 | lr=FLAGS.lr, 40 | wd=FLAGS.wd, 41 | arch=FLAGS.arch, 42 | batch=FLAGS.batch, 43 | nclass=dataset.nclass, 44 | 45 | K=FLAGS.K, 46 | beta=FLAGS.beta, 47 | w_kl=FLAGS.w_kl, 48 | w_match=FLAGS.w_match, 49 | w_rot=FLAGS.w_rot, 50 | redux=FLAGS.redux, 51 | use_dm=FLAGS.use_dm, 52 | use_xe=FLAGS.use_xe, 53 | warmup_kimg=FLAGS.warmup_kimg, 54 | 55 | scales=FLAGS.scales or (log_width - 2), 56 | filters=FLAGS.filters, 57 | repeat=FLAGS.repeat) 58 | model.train(FLAGS.train_kimg << 10, FLAGS.report_kimg << 10) 59 | 60 | 61 | if __name__ == '__main__': 62 | utils.setup_tf() 63 | flags.DEFINE_float('wd', 0.02, 'Weight decay.') 64 | flags.DEFINE_float('beta', 0.75, 'Mixup beta distribution.') 65 | flags.DEFINE_float('w_kl', 0.5, 'Weight for KL loss.') 66 | flags.DEFINE_float('w_match', 1.5, 'Weight for distribution matching loss.') 67 | flags.DEFINE_float('w_rot', 0.5, 'Weight for rotation loss.') 68 | flags.DEFINE_integer('scales', 0, 'Number of 2x2 downscalings in the classifier.') 69 | flags.DEFINE_integer('filters', 32, 'Filter size of convolutions.') 70 | flags.DEFINE_integer('repeat', 4, 'Number of residual layers per stage.') 71 | flags.DEFINE_integer('warmup_kimg', 1024, 'Unannealing duration for SSL loss.') 72 | flags.DEFINE_enum('redux', '1st', 'swap mean 1st'.split(), 'Logit selection.') 73 | flags.DEFINE_bool('use_dm', True, 'Whether to use distribution matching.') 74 | flags.DEFINE_bool('use_xe', True, 'Whether to use cross-entropy or Brier.') 75 | FLAGS.set_default('augment', 'd.d.d') 76 | FLAGS.set_default('dataset', 'cifar10.3@250-5000') 77 | FLAGS.set_default('batch', 64) 78 | FLAGS.set_default('lr', 0.002) 79 | FLAGS.set_default('train_kimg', 1 << 16) 80 | app.run(main) 81 | -------------------------------------------------------------------------------- /TF-BOSS/cta/lib/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /TF-BOSS/cta/lib/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lnsmith54/BOSS/45a202c93ebcab8e9c033cc1ac20ba7e1849fff9/TF-BOSS/cta/lib/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /TF-BOSS/cta/lib/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lnsmith54/BOSS/45a202c93ebcab8e9c033cc1ac20ba7e1849fff9/TF-BOSS/cta/lib/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /TF-BOSS/cta/lib/__pycache__/train.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lnsmith54/BOSS/45a202c93ebcab8e9c033cc1ac20ba7e1849fff9/TF-BOSS/cta/lib/__pycache__/train.cpython-36.pyc -------------------------------------------------------------------------------- /TF-BOSS/cta/lib/__pycache__/train.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lnsmith54/BOSS/45a202c93ebcab8e9c033cc1ac20ba7e1849fff9/TF-BOSS/cta/lib/__pycache__/train.cpython-37.pyc -------------------------------------------------------------------------------- /TF-BOSS/fully_supervised/fs_baseline.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Fully supervised training. 15 | """ 16 | 17 | import functools 18 | import os 19 | 20 | import tensorflow as tf 21 | from absl import app 22 | from absl import flags 23 | 24 | from fully_supervised.lib.data import DATASETS 25 | from fully_supervised.lib.train import ClassifyFullySupervised 26 | from libml import models, utils 27 | from libml.utils import EasyDict 28 | 29 | FLAGS = flags.FLAGS 30 | 31 | 32 | class FSBaseline(ClassifyFullySupervised, models.MultiModel): 33 | 34 | def model(self, batch, lr, wd, ema, **kwargs): 35 | hwc = [self.dataset.height, self.dataset.width, self.dataset.colors] 36 | xt_in = tf.placeholder(tf.float32, [batch] + hwc, 'xt') # For training 37 | x_in = tf.placeholder(tf.float32, [None] + hwc, 'x') 38 | l_in = tf.placeholder(tf.int32, [batch], 'labels') 39 | wd *= lr 40 | classifier = lambda x, **kw: self.classifier(x, **kw, **kwargs).logits 41 | 42 | x, labels_x = self.augment(xt_in, tf.one_hot(l_in, self.nclass), **kwargs) 43 | logits_x = classifier(x, training=True) 44 | 45 | loss_xe = tf.nn.softmax_cross_entropy_with_logits_v2(labels=labels_x, logits=logits_x) 46 | loss_xe = tf.reduce_mean(loss_xe) 47 | tf.summary.scalar('losses/xe', loss_xe) 48 | 49 | ema = tf.train.ExponentialMovingAverage(decay=ema) 50 | ema_op = ema.apply(utils.model_vars()) 51 | ema_getter = functools.partial(utils.getter_ema, ema) 52 | post_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) + [ema_op] 53 | post_ops.extend([tf.assign(v, v * (1 - wd)) for v in utils.model_vars('classify') if 'kernel' in v.name]) 54 | 55 | train_op = tf.train.AdamOptimizer(lr).minimize(loss_xe, colocate_gradients_with_ops=True) 56 | with tf.control_dependencies([train_op]): 57 | train_op = tf.group(*post_ops) 58 | 59 | return EasyDict( 60 | xt=xt_in, x=x_in, label=l_in, train_op=train_op, 61 | classify_raw=tf.nn.softmax(classifier(x_in, training=False)), # No EMA, for debugging. 62 | classify_op=tf.nn.softmax(classifier(x_in, getter=ema_getter, training=False))) 63 | 64 | 65 | def main(argv): 66 | utils.setup_main() 67 | del argv # Unused. 68 | dataset = DATASETS()[FLAGS.dataset]() 69 | log_width = utils.ilog2(dataset.width) 70 | model = FSBaseline( 71 | os.path.join(FLAGS.train_dir, dataset.name), 72 | dataset, 73 | lr=FLAGS.lr, 74 | wd=FLAGS.wd, 75 | arch=FLAGS.arch, 76 | batch=FLAGS.batch, 77 | nclass=dataset.nclass, 78 | ema=FLAGS.ema, 79 | dropout=FLAGS.dropout, 80 | smoothing=FLAGS.smoothing, 81 | 82 | scales=FLAGS.scales or (log_width - 2), 83 | filters=FLAGS.filters, 84 | repeat=FLAGS.repeat) 85 | model.train(FLAGS.train_kimg << 10, FLAGS.report_kimg << 10) 86 | 87 | 88 | if __name__ == '__main__': 89 | utils.setup_tf() 90 | flags.DEFINE_float('wd', 0.002, 'Weight decay.') 91 | flags.DEFINE_float('ema', 0.999, 'Exponential moving average of params.') 92 | flags.DEFINE_float('dropout', 0, 'Dropout on embedding layer.') 93 | flags.DEFINE_float('smoothing', 0.001, 'Label smoothing.') 94 | flags.DEFINE_integer('scales', 0, 'Number of 2x2 downscalings in the classifier.') 95 | flags.DEFINE_integer('filters', 32, 'Filter size of convolutions.') 96 | flags.DEFINE_integer('repeat', 4, 'Number of residual layers per stage.') 97 | FLAGS.set_default('dataset', 'cifar10') 98 | FLAGS.set_default('batch', 64) 99 | FLAGS.set_default('lr', 0.002) 100 | FLAGS.set_default('train_kimg', 1 << 16) 101 | app.run(main) 102 | -------------------------------------------------------------------------------- /TF-BOSS/fully_supervised/fs_mixup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Mixup fully supervised training. 15 | """ 16 | 17 | import os 18 | 19 | import tensorflow as tf 20 | from absl import app 21 | from absl import flags 22 | 23 | from fully_supervised.fs_baseline import FSBaseline 24 | from fully_supervised.lib.data import DATASETS 25 | from libml import utils 26 | 27 | FLAGS = flags.FLAGS 28 | 29 | 30 | class FSMixup(FSBaseline): 31 | 32 | def augment(self, x, l, beta, **kwargs): 33 | del kwargs 34 | with tf.device('/cpu'): 35 | mix = tf.distributions.Beta(beta, beta).sample([tf.shape(x)[0], 1, 1, 1]) 36 | mix = tf.maximum(mix, 1 - mix) 37 | xmix = x * mix + x[::-1] * (1 - mix) 38 | lmix = l * mix[:, :, 0, 0] + l[::-1] * (1 - mix[:, :, 0, 0]) 39 | return xmix, lmix 40 | 41 | 42 | def main(argv): 43 | utils.setup_main() 44 | del argv # Unused. 45 | dataset = DATASETS()[FLAGS.dataset]() 46 | log_width = utils.ilog2(dataset.width) 47 | model = FSMixup( 48 | os.path.join(FLAGS.train_dir, dataset.name), 49 | dataset, 50 | lr=FLAGS.lr, 51 | wd=FLAGS.wd, 52 | arch=FLAGS.arch, 53 | batch=FLAGS.batch, 54 | nclass=dataset.nclass, 55 | ema=FLAGS.ema, 56 | beta=FLAGS.beta, 57 | dropout=FLAGS.dropout, 58 | 59 | scales=FLAGS.scales or (log_width - 2), 60 | filters=FLAGS.filters, 61 | repeat=FLAGS.repeat) 62 | model.train(FLAGS.train_kimg << 10, FLAGS.report_kimg << 10) 63 | 64 | 65 | if __name__ == '__main__': 66 | utils.setup_tf() 67 | flags.DEFINE_float('wd', 0.002, 'Weight decay.') 68 | flags.DEFINE_float('ema', 0.999, 'Exponential moving average of params.') 69 | flags.DEFINE_float('beta', 0.5, 'Mixup beta distribution.') 70 | flags.DEFINE_float('dropout', 0, 'Dropout on embedding layer.') 71 | flags.DEFINE_integer('scales', 0, 'Number of 2x2 downscalings in the classifier.') 72 | flags.DEFINE_integer('filters', 32, 'Filter size of convolutions.') 73 | flags.DEFINE_integer('repeat', 4, 'Number of residual layers per stage.') 74 | FLAGS.set_default('dataset', 'cifar10-1') 75 | FLAGS.set_default('batch', 64) 76 | FLAGS.set_default('lr', 0.002) 77 | FLAGS.set_default('train_kimg', 1 << 16) 78 | app.run(main) 79 | -------------------------------------------------------------------------------- /TF-BOSS/fully_supervised/lib/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /TF-BOSS/fully_supervised/lib/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lnsmith54/BOSS/45a202c93ebcab8e9c033cc1ac20ba7e1849fff9/TF-BOSS/fully_supervised/lib/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /TF-BOSS/fully_supervised/lib/__pycache__/train.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lnsmith54/BOSS/45a202c93ebcab8e9c033cc1ac20ba7e1849fff9/TF-BOSS/fully_supervised/lib/__pycache__/train.cpython-37.pyc -------------------------------------------------------------------------------- /TF-BOSS/fully_supervised/lib/data.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | 17 | from absl import flags 18 | 19 | from libml import augment as augment_module 20 | from libml import data 21 | 22 | FLAGS = flags.FLAGS 23 | 24 | 25 | class DataSetsFS(data.DataSets): 26 | @classmethod 27 | def creator(cls, name, train_files, test_files, valid, augment, parse_fn=data.record_parse, do_memoize=True, 28 | nclass=10, height=32, width=32, colors=3): 29 | train_files = [os.path.join(data.DATA_DIR, x) for x in train_files] 30 | test_files = [os.path.join(data.DATA_DIR, x) for x in test_files] 31 | if not isinstance(augment, list): 32 | augment = augment(name) 33 | else: 34 | assert len(augment) == 1 35 | augment = augment[0] 36 | 37 | def create(): 38 | image_shape = [height, width, colors] 39 | kwargs = dict(parse_fn=parse_fn, image_shape=image_shape) 40 | train_labeled = data.DataSet.from_files(train_files, augment, **kwargs) 41 | if do_memoize: 42 | train_labeled = train_labeled.memoize() 43 | if FLAGS.whiten: 44 | mean, std = data.compute_mean_std(train_labeled) 45 | else: 46 | mean, std = 0, 1 47 | 48 | valid_data = data.DataSet.from_files(train_files, augment_module.NOAUGMENT, **kwargs).take(valid) 49 | test_data = data.DataSet.from_files(test_files, augment_module.NOAUGMENT, **kwargs) 50 | 51 | return cls(name + '.' + FLAGS.augment.split('.')[0] + '-' + str(valid), 52 | train_labeled=train_labeled.skip(valid), 53 | train_unlabeled=None, 54 | valid=valid_data, 55 | test=test_data, 56 | nclass=nclass, colors=colors, height=height, width=width, mean=mean, std=std) 57 | 58 | return name + '-' + str(valid), create 59 | 60 | 61 | def augment_function(dataset: str): 62 | return augment_module.get_augmentation(dataset, FLAGS.augment.split('.')[0]) 63 | 64 | 65 | def create_datasets(): 66 | d = {} 67 | d.update([DataSetsFS.creator('cifar10', ['cifar10-train.tfrecord'], ['cifar10-test.tfrecord'], valid, 68 | augment_function) for valid in [1, 5000]]) 69 | d.update([DataSetsFS.creator('cifar100', ['cifar100-train.tfrecord'], ['cifar100-test.tfrecord'], valid, 70 | augment_function, nclass=100) for valid in [1, 5000]]) 71 | d.update([DataSetsFS.creator('fashion_mnist', ['fashion_mnist-train.tfrecord'], ['fashion_mnist-test.tfrecord'], 72 | valid, augment_function, height=32, width=32, colors=1, 73 | parse_fn=data.record_parse_mnist) 74 | for valid in [1, 5000]]) 75 | d.update( 76 | [DataSetsFS.creator('stl10', [], [], valid, augment_function, height=96, width=96, do_memoize=False) 77 | for valid in [1, 5000]]) 78 | d.update([DataSetsFS.creator('svhn', ['svhn-train.tfrecord', 'svhn-extra.tfrecord'], ['svhn-test.tfrecord'], 79 | valid, augment_function, do_memoize=False) for valid in [1, 5000]]) 80 | d.update([DataSetsFS.creator('svhn_noextra', ['svhn-train.tfrecord'], ['svhn-test.tfrecord'], 81 | valid, augment_function, do_memoize=False) for valid in [1, 5000]]) 82 | return d 83 | 84 | 85 | DATASETS = create_datasets 86 | -------------------------------------------------------------------------------- /TF-BOSS/fully_supervised/lib/train.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import tensorflow as tf 16 | from absl import flags 17 | from tqdm import trange 18 | 19 | from libml import utils 20 | from libml.train import ClassifySemi 21 | 22 | FLAGS = flags.FLAGS 23 | 24 | 25 | class ClassifyFullySupervised(ClassifySemi): 26 | """Fully supervised classification. 27 | """ 28 | 29 | def train_step(self, train_session, gen_labeled): 30 | x = gen_labeled() 31 | self.tmp.step = train_session.run([self.ops.train_op, self.ops.update_step], 32 | feed_dict={self.ops.xt: x['image'], 33 | self.ops.label: x['label']})[1] 34 | 35 | def train(self, train_nimg, report_nimg): 36 | if FLAGS.eval_ckpt: 37 | self.eval_checkpoint(FLAGS.eval_ckpt) 38 | return 39 | batch = FLAGS.batch 40 | train_labeled = self.dataset.train_labeled.repeat().shuffle(FLAGS.shuffle).parse().augment() 41 | train_labeled = train_labeled.batch(batch).prefetch(16).make_one_shot_iterator().get_next() 42 | scaffold = tf.train.Scaffold(saver=tf.train.Saver(max_to_keep=FLAGS.keep_ckpt, pad_step_number=10)) 43 | 44 | with tf.Session(config=utils.get_config()) as sess: 45 | self.session = sess 46 | self.cache_eval() 47 | 48 | with tf.train.MonitoredTrainingSession( 49 | scaffold=scaffold, 50 | checkpoint_dir=self.checkpoint_dir, 51 | config=utils.get_config(), 52 | save_checkpoint_steps=FLAGS.save_kimg << 10, 53 | save_summaries_steps=report_nimg - batch) as train_session: 54 | self.session = train_session._tf_sess() 55 | gen_labeled = self.gen_labeled_fn(train_labeled) 56 | self.tmp.step = self.session.run(self.step) 57 | while self.tmp.step < train_nimg: 58 | loop = trange(self.tmp.step % report_nimg, report_nimg, batch, 59 | leave=False, unit='img', unit_scale=batch, 60 | desc='Epoch %d/%d' % (1 + (self.tmp.step // report_nimg), train_nimg // report_nimg)) 61 | for _ in loop: 62 | self.train_step(train_session, gen_labeled) 63 | while self.tmp.print_queue: 64 | loop.write(self.tmp.print_queue.pop(0)) 65 | while self.tmp.print_queue: 66 | print(self.tmp.print_queue.pop(0)) 67 | -------------------------------------------------------------------------------- /TF-BOSS/fully_supervised/runs/all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Fully supervised baseline without mixup (not shown in paper since Mixup is better) 18 | python fully_supervised/fs_baseline.py --train_dir experiments/fs --dataset=cifar10-1 --wd=0.02 --smoothing=0.001 19 | python fully_supervised/fs_baseline.py --train_dir experiments/fs --dataset=cifar100-1 --wd=0.02 --smoothing=0.001 20 | python fully_supervised/fs_baseline.py --train_dir experiments/fs --dataset=svhn-1 --wd=0.002 --smoothing=0.01 21 | python fully_supervised/fs_baseline.py --train_dir experiments/fs --dataset=svhn_noextra-1 --wd=0.002 --smoothing=0.01 22 | 23 | # Fully supervised Mixup baselines (in paper) 24 | # Uses default parameters: --wd=0.002 --beta=0.5 25 | python fully_supervised/fs_mixup.py --train_dir experiments/fs --dataset=cifar10-1 26 | python fully_supervised/fs_mixup.py --train_dir experiments/fs --dataset=svhn-1 27 | python fully_supervised/fs_mixup.py --train_dir experiments/fs --dataset=svhn_noextra-1 28 | 29 | # Fully supervised Mixup baselines on 26M parameter large network (in paper) 30 | # Uses default parameters: --wd=0.002 --beta=0.5 31 | python fully_supervised/fs_mixup.py --train_dir experiments/fs --dataset=cifar10-1 --filters=135 32 | python fully_supervised/fs_mixup.py --train_dir experiments/fs --dataset=cifar100-1 --filters=135 33 | -------------------------------------------------------------------------------- /TF-BOSS/ict.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Interpolation Consistency Training for Semi-Supervised Learning. 15 | 16 | Reimplementation of https://arxiv.org/abs/1903.03825 17 | """ 18 | 19 | import functools 20 | import os 21 | 22 | import tensorflow as tf 23 | from absl import app 24 | from absl import flags 25 | 26 | from libml import models, utils 27 | from libml.data import PAIR_DATASETS 28 | from libml.utils import EasyDict 29 | 30 | FLAGS = flags.FLAGS 31 | 32 | 33 | class ICT(models.MultiModel): 34 | 35 | def model(self, batch, lr, wd, ema, warmup_pos, consistency_weight, beta, **kwargs): 36 | hwc = [self.dataset.height, self.dataset.width, self.dataset.colors] 37 | xt_in = tf.placeholder(tf.float32, [batch] + hwc, 'xt') # For training 38 | x_in = tf.placeholder(tf.float32, [None] + hwc, 'x') 39 | y_in = tf.placeholder(tf.float32, [batch, 2] + hwc, 'y') 40 | l_in = tf.placeholder(tf.int32, [batch], 'labels') 41 | l = tf.one_hot(l_in, self.nclass) 42 | wd *= lr 43 | warmup = tf.clip_by_value(tf.to_float(self.step) / (warmup_pos * (FLAGS.train_kimg << 10)), 0, 1) 44 | 45 | y = tf.reshape(tf.transpose(y_in, [1, 0, 2, 3, 4]), [-1] + hwc) 46 | y_1, y_2 = tf.split(y, 2) 47 | 48 | mix = tf.distributions.Beta(beta, beta).sample([tf.shape(x_in)[0], 1, 1, 1]) 49 | mix = tf.maximum(mix, 1 - mix) 50 | 51 | classifier = lambda x, **kw: self.classifier(x, **kw, **kwargs).logits 52 | logits_x = classifier(xt_in, training=True) 53 | post_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # Take only first call to update batch norm. 54 | 55 | ema = tf.train.ExponentialMovingAverage(decay=ema) 56 | ema_op = ema.apply(utils.model_vars()) 57 | ema_getter = functools.partial(utils.getter_ema, ema) 58 | logits_teacher = classifier(y_1, training=True, getter=ema_getter) 59 | labels_teacher = tf.stop_gradient(tf.nn.softmax(logits_teacher)) 60 | labels_teacher = labels_teacher * mix[:, :, 0, 0] + labels_teacher[::-1] * (1 - mix[:, :, 0, 0]) 61 | logits_student = classifier(y_1 * mix + y_1[::-1] * (1 - mix), training=True) 62 | loss_mt = tf.reduce_mean((labels_teacher - tf.nn.softmax(logits_student)) ** 2, -1) 63 | loss_mt = tf.reduce_mean(loss_mt) 64 | 65 | loss = tf.nn.softmax_cross_entropy_with_logits_v2(labels=l, logits=logits_x) 66 | loss = tf.reduce_mean(loss) 67 | tf.summary.scalar('losses/xe', loss) 68 | tf.summary.scalar('losses/mt', loss_mt) 69 | 70 | post_ops.append(ema_op) 71 | post_ops.extend([tf.assign(v, v * (1 - wd)) for v in utils.model_vars('classify') if 'kernel' in v.name]) 72 | 73 | train_op = tf.train.AdamOptimizer(lr).minimize(loss + loss_mt * warmup * consistency_weight, 74 | colocate_gradients_with_ops=True) 75 | with tf.control_dependencies([train_op]): 76 | train_op = tf.group(*post_ops) 77 | 78 | return EasyDict( 79 | xt=xt_in, x=x_in, y=y_in, label=l_in, train_op=train_op, 80 | classify_raw=tf.nn.softmax(classifier(x_in, training=False)), # No EMA, for debugging. 81 | classify_op=tf.nn.softmax(classifier(x_in, getter=ema_getter, training=False))) 82 | 83 | 84 | def main(argv): 85 | utils.setup_main() 86 | del argv # Unused. 87 | dataset = PAIR_DATASETS()[FLAGS.dataset]() 88 | log_width = utils.ilog2(dataset.width) 89 | model = ICT( 90 | os.path.join(FLAGS.train_dir, dataset.name), 91 | dataset, 92 | lr=FLAGS.lr, 93 | wd=FLAGS.wd, 94 | arch=FLAGS.arch, 95 | warmup_pos=FLAGS.warmup_pos, 96 | batch=FLAGS.batch, 97 | nclass=dataset.nclass, 98 | ema=FLAGS.ema, 99 | beta=FLAGS.beta, 100 | consistency_weight=FLAGS.consistency_weight, 101 | 102 | scales=FLAGS.scales or (log_width - 2), 103 | filters=FLAGS.filters, 104 | repeat=FLAGS.repeat) 105 | model.train(FLAGS.train_kimg << 10, FLAGS.report_kimg << 10) 106 | 107 | 108 | if __name__ == '__main__': 109 | utils.setup_tf() 110 | flags.DEFINE_float('consistency_weight', 50., 'Consistency weight.') 111 | flags.DEFINE_float('warmup_pos', 0.4, 'Relative position at which constraint loss warmup ends.') 112 | flags.DEFINE_float('wd', 0.02, 'Weight decay.') 113 | flags.DEFINE_float('ema', 0.999, 'Exponential moving average of params.') 114 | flags.DEFINE_float('beta', 0.5, 'Mixup beta.') 115 | flags.DEFINE_integer('scales', 0, 'Number of 2x2 downscalings in the classifier.') 116 | flags.DEFINE_integer('filters', 32, 'Filter size of convolutions.') 117 | flags.DEFINE_integer('repeat', 4, 'Number of residual layers per stage.') 118 | FLAGS.set_default('augment', 'd.d.d') 119 | FLAGS.set_default('dataset', 'cifar10.3@250-5000') 120 | FLAGS.set_default('batch', 64) 121 | FLAGS.set_default('lr', 0.002) 122 | FLAGS.set_default('train_kimg', 1 << 16) 123 | app.run(main) 124 | -------------------------------------------------------------------------------- /TF-BOSS/installData.sh: -------------------------------------------------------------------------------- 1 | 2 | module load python36 3 | module load openmpi/cuda/64/3.1.4 4 | module load tensorflow-py36-cuda10.1-gcc/1.15.2 5 | 6 | export ML_DATA="./data" 7 | export PYTHONPATH=$PYTHONPATH:$PWD 8 | 9 | date 10 | size=10 11 | for seed in 1 2 3 4 5; do 12 | CUDA_VISIBLE_DEVICES= scripts/cifar10_prototypes.py --seed=$seed --size=$size $ML_DATA/SSL2/cifar10p $ML_DATA/cifar10-train.tfrecord 13 | done 14 | 15 | date 16 | 17 | exit 18 | 19 | -------------------------------------------------------------------------------- /TF-BOSS/iter.sh: -------------------------------------------------------------------------------- 1 | Seed0=$1 2 | 3 | module load python36 4 | module load openmpi/cuda/64/3.1.4 5 | module load tensorflow-py36-cuda10.1-gcc/1.15.2 6 | 7 | export ML_DATA="./data/pseudolabeled" 8 | export PYTHONPATH=$PYTHONPATH:$PWD 9 | 10 | CUDA_VISIBLE_DEVICES= scripts/iteration1.py --seed=$Seed0 --size=50 --balance $2 --pseudo_file $3 $ML_DATA/SSL2/cifar10p $ML_DATA/cifar10p-train.tfrecord 11 | 12 | CUDA_VISIBLE_DEVICES= scripts/iteration1.py --seed=$Seed0 --size=100 --balance $2 --pseudo_file $3 $ML_DATA/SSL2/cifar10p $ML_DATA/cifar10p-train.tfrecord 13 | 14 | CUDA_VISIBLE_DEVICES= scripts/iteration1.py --seed=$Seed0 --size=200 --balance $2 --pseudo_file $3 $ML_DATA/SSL2/cifar10p $ML_DATA/cifar10p-train.tfrecord 15 | 16 | CUDA_VISIBLE_DEVICES= scripts/iteration1.py --seed=$Seed0 --size=400 --balance $2 --pseudo_file $3 $ML_DATA/SSL2/cifar10p $ML_DATA/cifar10p-train.tfrecord 17 | 18 | wait 19 | 20 | exit 21 | -------------------------------------------------------------------------------- /TF-BOSS/libml/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /TF-BOSS/libml/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lnsmith54/BOSS/45a202c93ebcab8e9c033cc1ac20ba7e1849fff9/TF-BOSS/libml/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /TF-BOSS/libml/__pycache__/augment.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lnsmith54/BOSS/45a202c93ebcab8e9c033cc1ac20ba7e1849fff9/TF-BOSS/libml/__pycache__/augment.cpython-37.pyc -------------------------------------------------------------------------------- /TF-BOSS/libml/__pycache__/ctaugment.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lnsmith54/BOSS/45a202c93ebcab8e9c033cc1ac20ba7e1849fff9/TF-BOSS/libml/__pycache__/ctaugment.cpython-37.pyc -------------------------------------------------------------------------------- /TF-BOSS/libml/__pycache__/data.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lnsmith54/BOSS/45a202c93ebcab8e9c033cc1ac20ba7e1849fff9/TF-BOSS/libml/__pycache__/data.cpython-37.pyc -------------------------------------------------------------------------------- /TF-BOSS/libml/__pycache__/layers.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lnsmith54/BOSS/45a202c93ebcab8e9c033cc1ac20ba7e1849fff9/TF-BOSS/libml/__pycache__/layers.cpython-37.pyc -------------------------------------------------------------------------------- /TF-BOSS/libml/__pycache__/models.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lnsmith54/BOSS/45a202c93ebcab8e9c033cc1ac20ba7e1849fff9/TF-BOSS/libml/__pycache__/models.cpython-37.pyc -------------------------------------------------------------------------------- /TF-BOSS/libml/__pycache__/train.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lnsmith54/BOSS/45a202c93ebcab8e9c033cc1ac20ba7e1849fff9/TF-BOSS/libml/__pycache__/train.cpython-37.pyc -------------------------------------------------------------------------------- /TF-BOSS/libml/__pycache__/utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lnsmith54/BOSS/45a202c93ebcab8e9c033cc1ac20ba7e1849fff9/TF-BOSS/libml/__pycache__/utils.cpython-37.pyc -------------------------------------------------------------------------------- /TF-BOSS/libml/ctaugment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Control Theory based self-augmentation.""" 15 | import random 16 | from collections import namedtuple 17 | 18 | import numpy as np 19 | from PIL import Image, ImageOps, ImageEnhance, ImageFilter 20 | 21 | OPS = {} 22 | OP = namedtuple('OP', ('f', 'bins')) 23 | Sample = namedtuple('Sample', ('train', 'probe')) 24 | 25 | 26 | def register(*bins): 27 | def wrap(f): 28 | OPS[f.__name__] = OP(f, bins) 29 | return f 30 | 31 | return wrap 32 | 33 | 34 | def apply(x, ops): 35 | if ops is None: 36 | return x 37 | y = Image.fromarray(np.round(127.5 * (1 + x)).clip(0, 255).astype('uint8')) 38 | for op, args in ops: 39 | y = OPS[op].f(y, *args) 40 | return np.asarray(y).astype('f') / 127.5 - 1 41 | 42 | 43 | class CTAugment: 44 | def __init__(self, depth=2, th=0.85, decay=0.99): 45 | self.decay = decay 46 | self.depth = depth 47 | self.th = th 48 | self.rates = {} 49 | for k, op in OPS.items(): 50 | self.rates[k] = tuple([np.ones(x, 'f') for x in op.bins]) 51 | 52 | def rate_to_p(self, rate): 53 | p = rate + (1 - self.decay) # Avoid to have all zero. 54 | p = p / p.max() 55 | p[p < self.th] = 0 56 | return p 57 | 58 | def policy(self, probe): 59 | kl = list(OPS.keys()) 60 | v = [] 61 | if probe: 62 | for _ in range(self.depth): 63 | k = random.choice(kl) 64 | bins = self.rates[k] 65 | rnd = np.random.uniform(0, 1, len(bins)) 66 | v.append(OP(k, rnd.tolist())) 67 | return v 68 | for _ in range(self.depth): 69 | vt = [] 70 | k = random.choice(kl) 71 | bins = self.rates[k] 72 | rnd = np.random.uniform(0, 1, len(bins)) 73 | for r, bin in zip(rnd, bins): 74 | p = self.rate_to_p(bin) 75 | value = np.random.choice(p.shape[0], p=p / p.sum()) 76 | vt.append((value + r) / p.shape[0]) 77 | v.append(OP(k, vt)) 78 | return v 79 | 80 | def update_rates(self, policy, proximity): 81 | for k, bins in policy: 82 | for p, rate in zip(bins, self.rates[k]): 83 | p = int(p * len(rate) * 0.999) 84 | rate[p] = rate[p] * self.decay + proximity * (1 - self.decay) 85 | 86 | def stats(self): 87 | return '\n'.join('%-16s %s' % (k, ' / '.join(' '.join('%.2f' % x for x in self.rate_to_p(rate)) 88 | for rate in self.rates[k])) 89 | for k in sorted(OPS.keys())) 90 | 91 | 92 | def _enhance(x, op, level): 93 | return op(x).enhance(0.1 + 1.9 * level) 94 | 95 | 96 | def _imageop(x, op, level): 97 | return Image.blend(x, op(x), level) 98 | 99 | 100 | def _filter(x, op, level): 101 | return Image.blend(x, x.filter(op), level) 102 | 103 | 104 | @register(17) 105 | def autocontrast(x, level): 106 | return _imageop(x, ImageOps.autocontrast, level) 107 | 108 | 109 | @register(17) 110 | def blur(x, level): 111 | return _filter(x, ImageFilter.BLUR, level) 112 | 113 | 114 | @register(17) 115 | def brightness(x, brightness): 116 | return _enhance(x, ImageEnhance.Brightness, brightness) 117 | 118 | 119 | @register(17) 120 | def color(x, color): 121 | return _enhance(x, ImageEnhance.Color, color) 122 | 123 | 124 | @register(17) 125 | def contrast(x, contrast): 126 | return _enhance(x, ImageEnhance.Contrast, contrast) 127 | 128 | 129 | @register(17) 130 | def cutout(x, level): 131 | """Apply cutout to pil_img at the specified level.""" 132 | size = 1 + int(level * min(x.size) * 0.499) 133 | img_height, img_width = x.size 134 | height_loc = np.random.randint(low=0, high=img_height) 135 | width_loc = np.random.randint(low=0, high=img_width) 136 | upper_coord = (max(0, height_loc - size // 2), max(0, width_loc - size // 2)) 137 | lower_coord = (min(img_height, height_loc + size // 2), min(img_width, width_loc + size // 2)) 138 | pixels = x.load() # create the pixel map 139 | for i in range(upper_coord[0], lower_coord[0]): # for every col: 140 | for j in range(upper_coord[1], lower_coord[1]): # For every row 141 | pixels[i, j] = (127, 127, 127) # set the color accordingly 142 | return x 143 | 144 | 145 | @register(17) 146 | def equalize(x, level): 147 | return _imageop(x, ImageOps.equalize, level) 148 | 149 | 150 | @register(17) 151 | def invert(x, level): 152 | return _imageop(x, ImageOps.invert, level) 153 | 154 | 155 | @register() 156 | def identity(x): 157 | return x 158 | 159 | 160 | @register(8) 161 | def posterize(x, level): 162 | level = 1 + int(level * 7.999) 163 | return ImageOps.posterize(x, level) 164 | 165 | 166 | @register(17, 6) 167 | def rescale(x, scale, method): 168 | s = x.size 169 | scale *= 0.25 170 | crop = (scale * s[0], scale * s[1], s[0] * (1 - scale), s[1] * (1 - scale)) 171 | methods = (Image.ANTIALIAS, Image.BICUBIC, Image.BILINEAR, Image.BOX, Image.HAMMING, Image.NEAREST) 172 | method = methods[int(method * 5.99)] 173 | return x.crop(crop).resize(x.size, method) 174 | 175 | 176 | @register(17) 177 | def rotate(x, angle): 178 | angle = int(np.round((2 * angle - 1) * 45)) 179 | return x.rotate(angle) 180 | 181 | 182 | @register(17) 183 | def sharpness(x, sharpness): 184 | return _enhance(x, ImageEnhance.Sharpness, sharpness) 185 | 186 | 187 | @register(17) 188 | def shear_x(x, shear): 189 | shear = (2 * shear - 1) * 0.3 190 | return x.transform(x.size, Image.AFFINE, (1, shear, 0, 0, 1, 0)) 191 | 192 | 193 | @register(17) 194 | def shear_y(x, shear): 195 | shear = (2 * shear - 1) * 0.3 196 | return x.transform(x.size, Image.AFFINE, (1, 0, 0, shear, 1, 0)) 197 | 198 | 199 | @register(17) 200 | def smooth(x, level): 201 | return _filter(x, ImageFilter.SMOOTH, level) 202 | 203 | 204 | @register(17) 205 | def solarize(x, th): 206 | th = int(th * 255.999) 207 | return ImageOps.solarize(x, th) 208 | 209 | 210 | @register(17) 211 | def translate_x(x, delta): 212 | delta = (2 * delta - 1) * 0.3 213 | return x.transform(x.size, Image.AFFINE, (1, 0, delta, 0, 1, 0)) 214 | 215 | 216 | @register(17) 217 | def translate_y(x, delta): 218 | delta = (2 * delta - 1) * 0.3 219 | return x.transform(x.size, Image.AFFINE, (1, 0, 0, 0, 1, delta)) 220 | -------------------------------------------------------------------------------- /TF-BOSS/libml/models.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Classifier architectures.""" 15 | import functools 16 | import itertools 17 | 18 | import tensorflow as tf 19 | from absl import flags 20 | 21 | from libml import layers 22 | from libml.train import ClassifySemi 23 | from libml.utils import EasyDict 24 | 25 | 26 | class CNN13(ClassifySemi): 27 | """Simplified reproduction of the Mean Teacher paper network. filters=128 in original implementation. 28 | Removed dropout, Gaussians, forked dense layers, basically all non-standard things.""" 29 | 30 | def classifier(self, x, scales, filters, training, getter=None, **kwargs): 31 | del kwargs 32 | assert scales == 3 # Only specified for 32x32 inputs. 33 | conv_args = dict(kernel_size=3, activation=tf.nn.leaky_relu, padding='same') 34 | bn_args = dict(training=training, momentum=0.999) 35 | 36 | with tf.variable_scope('classify', reuse=tf.AUTO_REUSE, custom_getter=getter): 37 | y = tf.layers.conv2d((x - self.dataset.mean) / self.dataset.std, filters, **conv_args) 38 | y = tf.layers.batch_normalization(y, **bn_args) 39 | y = tf.layers.conv2d(y, filters, **conv_args) 40 | y = tf.layers.batch_normalization(y, **bn_args) 41 | y = tf.layers.conv2d(y, filters, **conv_args) 42 | y = tf.layers.batch_normalization(y, **bn_args) 43 | y = tf.layers.max_pooling2d(y, 2, 2) 44 | y = tf.layers.conv2d(y, 2 * filters, **conv_args) 45 | y = tf.layers.batch_normalization(y, **bn_args) 46 | y = tf.layers.conv2d(y, 2 * filters, **conv_args) 47 | y = tf.layers.batch_normalization(y, **bn_args) 48 | y = tf.layers.conv2d(y, 2 * filters, **conv_args) 49 | y = tf.layers.batch_normalization(y, **bn_args) 50 | y = tf.layers.max_pooling2d(y, 2, 2) 51 | y = tf.layers.conv2d(y, 4 * filters, kernel_size=3, activation=tf.nn.leaky_relu, padding='valid') 52 | y = tf.layers.batch_normalization(y, **bn_args) 53 | y = tf.layers.conv2d(y, 2 * filters, kernel_size=1, activation=tf.nn.leaky_relu, padding='same') 54 | y = tf.layers.batch_normalization(y, **bn_args) 55 | y = tf.layers.conv2d(y, 1 * filters, kernel_size=1, activation=tf.nn.leaky_relu, padding='same') 56 | y = tf.layers.batch_normalization(y, **bn_args) 57 | y = tf.reduce_mean(y, [1, 2]) # (b, 6, 6, 128) -> (b, 128) 58 | logits = tf.layers.dense(y, self.nclass) 59 | return EasyDict(logits=logits, embeds=y) 60 | 61 | 62 | class ResNet(ClassifySemi): 63 | def classifier(self, x, scales, filters, repeat, training, getter=None, dropout=0, **kwargs): 64 | del kwargs 65 | leaky_relu = functools.partial(tf.nn.leaky_relu, alpha=0.1) 66 | bn_args = dict(training=training, momentum=0.999) 67 | 68 | def conv_args(k, f): 69 | return dict(padding='same', 70 | kernel_initializer=tf.random_normal_initializer(stddev=tf.rsqrt(0.5 * k * k * f)) ) 71 | 72 | def residual(x0, filters, stride=1, activate_before_residual=False): 73 | x = leaky_relu(tf.layers.batch_normalization(x0, **bn_args)) 74 | if activate_before_residual: 75 | x0 = x 76 | 77 | x = tf.layers.conv2d(x, filters, 3, strides=stride, **conv_args(3, filters)) 78 | x = leaky_relu(tf.layers.batch_normalization(x, **bn_args)) 79 | x = tf.layers.conv2d(x, filters, 3, **conv_args(3, filters)) 80 | 81 | if x0.get_shape()[3] != filters: 82 | x0 = tf.layers.conv2d(x0, filters, 1, strides=stride, **conv_args(1, filters)) 83 | 84 | return x0 + x 85 | 86 | with tf.variable_scope('classify', reuse=tf.AUTO_REUSE, custom_getter=getter): 87 | y = tf.layers.conv2d((x - self.dataset.mean) / self.dataset.std, 16, 3, **conv_args(3, 16)) 88 | for scale in range(scales): 89 | y = residual(y, filters << scale, stride=2 if scale else 1, activate_before_residual=scale == 0) 90 | for i in range(repeat - 1): 91 | y = residual(y, filters << scale) 92 | 93 | y = leaky_relu(tf.layers.batch_normalization(y, **bn_args)) 94 | y = embeds = tf.reduce_mean(y, [1, 2]) 95 | if dropout and training: 96 | y = tf.nn.dropout(y, 1 - dropout) 97 | logits = tf.layers.dense(y, self.nclass, kernel_initializer=tf.glorot_normal_initializer()) 98 | return EasyDict(logits=logits, embeds=embeds) 99 | 100 | 101 | class ShakeNet(ClassifySemi): 102 | def classifier(self, x, scales, filters, repeat, training, getter=None, dropout=0, **kwargs): 103 | del kwargs 104 | bn_args = dict(training=training, momentum=0.999) 105 | 106 | def conv_args(k, f): 107 | return dict(padding='same', use_bias=False, 108 | kernel_initializer=tf.random_normal_initializer(stddev=tf.rsqrt(0.5 * k * k * f))) 109 | 110 | def residual(x0, filters, stride=1): 111 | def branch(): 112 | x = tf.nn.relu(x0) 113 | x = tf.layers.conv2d(x, filters, 3, strides=stride, **conv_args(3, filters)) 114 | x = tf.nn.relu(tf.layers.batch_normalization(x, **bn_args)) 115 | x = tf.layers.conv2d(x, filters, 3, **conv_args(3, filters)) 116 | x = tf.layers.batch_normalization(x, **bn_args) 117 | return x 118 | 119 | x = layers.shakeshake(branch(), branch(), training) 120 | 121 | if stride == 2: 122 | x1 = tf.layers.conv2d(tf.nn.relu(x0[:, ::2, ::2]), filters >> 1, 1, **conv_args(1, filters >> 1)) 123 | x2 = tf.layers.conv2d(tf.nn.relu(x0[:, 1::2, 1::2]), filters >> 1, 1, **conv_args(1, filters >> 1)) 124 | x0 = tf.concat([x1, x2], axis=3) 125 | x0 = tf.layers.batch_normalization(x0, **bn_args) 126 | elif x0.get_shape()[3] != filters: 127 | x0 = tf.layers.conv2d(x0, filters, 1, **conv_args(1, filters)) 128 | x0 = tf.layers.batch_normalization(x0, **bn_args) 129 | 130 | return x0 + x 131 | 132 | with tf.variable_scope('classify', reuse=tf.AUTO_REUSE, custom_getter=getter): 133 | y = tf.layers.conv2d((x - self.dataset.mean) / self.dataset.std, 16, 3, **conv_args(3, 16)) 134 | for scale, i in itertools.product(range(scales), range(repeat)): 135 | with tf.variable_scope('layer%d.%d' % (scale + 1, i)): 136 | if i == 0: 137 | y = residual(y, filters << scale, stride=2 if scale else 1) 138 | else: 139 | y = residual(y, filters << scale) 140 | 141 | y = embeds = tf.reduce_mean(y, [1, 2]) 142 | if dropout and training: 143 | y = tf.nn.dropout(y, 1 - dropout) 144 | logits = tf.layers.dense(y, self.nclass, kernel_initializer=tf.glorot_normal_initializer()) 145 | return EasyDict(logits=logits, embeds=embeds) 146 | 147 | 148 | class MultiModel(CNN13, ResNet, ShakeNet): 149 | MODELS = ('cnn13', 'resnet', 'shake') 150 | MODEL_CNN13, MODEL_RESNET, MODEL_SHAKE = MODELS 151 | 152 | def augment(self, x, l, smoothing, **kwargs): 153 | del kwargs 154 | return x, l - smoothing * (l - 1. / self.nclass) 155 | 156 | def classifier(self, x, arch, **kwargs): 157 | if arch == self.MODEL_CNN13: 158 | return CNN13.classifier(self, x, **kwargs) 159 | elif arch == self.MODEL_RESNET: 160 | return ResNet.classifier(self, x, **kwargs) 161 | elif arch == self.MODEL_SHAKE: 162 | return ShakeNet.classifier(self, x, **kwargs) 163 | raise ValueError('Model %s does not exists, available ones are %s' % (arch, self.MODELS)) 164 | 165 | 166 | flags.DEFINE_enum('arch', MultiModel.MODEL_RESNET, MultiModel.MODELS, 'Architecture.') 167 | -------------------------------------------------------------------------------- /TF-BOSS/loopTensor.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | #pseudo_mask = tf.placeholder(tf.float32, shape=(12)) 4 | 5 | delT = 0.2 6 | confidence = 0.95 7 | pLabels = np.array([1, 2, 3, 4, 5, 6, 7, 8, 7, 5, 4, 1]) 8 | #classes, idx, class_count = tf.unique_with_counts(pLabels) 9 | #classes = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) 10 | #class_count = np.array([0, 2, 1, 1, 2, 2, 1, 2, 1, 0]) 11 | pLabels = tf.cast(pLabels,dtype=tf.int64) 12 | classes, idx, counts = tf.unique_with_counts(pLabels) 13 | #class_count = tf.zeros([10],dtype=tf.int32) 14 | shape = tf.constant([10]) 15 | classes = tf.cast(classes,dtype=tf.int32) 16 | class_count = tf.scatter_nd(tf.reshape(classes,[8,1]),counts, shape) 17 | 18 | with tf.Session() as sess: 19 | rs = sess.run(class_count) 20 | print(rs) 21 | 22 | print("class_count ", class_count) 23 | class_count= tf.cast(class_count,dtype=tf.float32) 24 | print("class_count ", class_count) 25 | ratios = tf.math.divide_no_nan(tf.ones_like(class_count,dtype=tf.float32),class_count) 26 | print("pLabels.shape[-1] ",pLabels.shape[-1]) 27 | ratio = tf.gather_nd(ratios, tf.reshape(pLabels,[tf.size(pLabels),1]) ) 28 | ratio = tf.math.divide_no_nan(tf.scalar_mul(12.0, ratio), tf.reduce_sum(ratio)) 29 | with tf.Session() as sess: 30 | rs = sess.run(ratio) 31 | print(rs) 32 | 33 | mxCount = tf.reduce_max(class_count, axis=0) 34 | ratios = 1.0 - tf.math.divide_no_nan(class_count, mxCount) 35 | ratios = confidence - delT*ratios 36 | confidences = tf.gather_nd(ratios, tf.reshape(pLabels,[tf.size(pLabels),1]) ) 37 | 38 | with tf.Session() as sess: 39 | rs = sess.run(confidences) 40 | print(rs) 41 | 42 | pseudo_mask = ratio >= confidences 43 | with tf.Session() as sess: 44 | rs = sess.run(pseudo_mask) 45 | print(rs) 46 | 47 | subLabels = tf.boolean_mask(pLabels,pseudo_mask) 48 | with tf.Session() as sess: 49 | rs = sess.run(subLabels) 50 | print(rs) 51 | 52 | classes, idx, counts = tf.unique_with_counts(pLabels) 53 | 54 | #pred = np.array([[1, 5, 9, 2],[10, 1, 7, 4],[2, 8, 1, 3]]) 55 | #print(pred) 56 | #print(np.argsort(pred,axis=0)) 57 | #print(np.argsort(pred,axis=0).argmax(axis=0)) 58 | 59 | 60 | # for i in range(self.dataset.nclass): 61 | # class_i = classes[i]*tf.ones_like(pLabels,dtype=tf.float32) 62 | # class_i = i*tf.ones_like(pLabels,dtype=tf.float32) 63 | # class_i = tf.cast(tf.math.equal(pLabels,class_i),dtype=tf.float32) 64 | # confidences = tf.math.add(confidences,tf.scalar_mul( confidence - delT*(1.0-ratios[i]) , class_i) ) 65 | -------------------------------------------------------------------------------- /TF-BOSS/mean_teacher.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Mean teachers are better role models: 16 | Weight-averaged consistency targets improve semi-supervised deep learning results. 17 | 18 | Reimplementation of https://arxiv.org/abs/1703.01780 19 | """ 20 | import functools 21 | import os 22 | 23 | import numpy as np 24 | import tensorflow as tf 25 | from absl import app 26 | from absl import flags 27 | 28 | from libml import models, utils 29 | from libml.data import PAIR_DATASETS 30 | from libml.utils import EasyDict 31 | 32 | FLAGS = flags.FLAGS 33 | 34 | 35 | class MeanTeacher(models.MultiModel): 36 | 37 | def model(self, batch, lr, wd, ema, warmup_pos, consistency_weight, **kwargs): 38 | hwc = [self.dataset.height, self.dataset.width, self.dataset.colors] 39 | xt_in = tf.placeholder(tf.float32, [batch] + hwc, 'xt') # For training 40 | x_in = tf.placeholder(tf.float32, [None] + hwc, 'x') 41 | y_in = tf.placeholder(tf.float32, [batch, 2] + hwc, 'y') 42 | l_in = tf.placeholder(tf.int32, [batch], 'labels') 43 | l = tf.one_hot(l_in, self.nclass) 44 | 45 | warmup = tf.clip_by_value(tf.to_float(self.step) / (warmup_pos * (FLAGS.train_kimg << 10)), 0, 1) 46 | lrate = tf.clip_by_value(tf.to_float(self.step) / (FLAGS.train_kimg << 10), 0, 1) 47 | lr *= tf.cos(lrate * (7 * np.pi) / (2 * 8)) 48 | tf.summary.scalar('monitors/lr', lr) 49 | 50 | classifier = lambda x, **kw: self.classifier(x, **kw, **kwargs).logits 51 | logits_x = classifier(xt_in, training=True) 52 | post_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # Take only first call to update batch norm. 53 | y = tf.reshape(tf.transpose(y_in, [1, 0, 2, 3, 4]), [-1] + hwc) 54 | y_1, y_2 = tf.split(y, 2) 55 | ema = tf.train.ExponentialMovingAverage(decay=ema) 56 | ema_op = ema.apply(utils.model_vars()) 57 | ema_getter = functools.partial(utils.getter_ema, ema) 58 | logits_y = classifier(y_1, training=True, getter=ema_getter) 59 | logits_teacher = tf.stop_gradient(logits_y) 60 | logits_student = classifier(y_2, training=True) 61 | loss_mt = tf.reduce_mean((tf.nn.softmax(logits_teacher) - tf.nn.softmax(logits_student)) ** 2, -1) 62 | loss_mt = tf.reduce_mean(loss_mt) 63 | 64 | loss = tf.nn.softmax_cross_entropy_with_logits_v2(labels=l, logits=logits_x) 65 | loss = tf.reduce_mean(loss) 66 | tf.summary.scalar('losses/xe', loss) 67 | tf.summary.scalar('losses/mt', loss_mt) 68 | 69 | # L2 regularization 70 | loss_wd = sum(tf.nn.l2_loss(v) for v in utils.model_vars('classify') if 'kernel' in v.name) 71 | tf.summary.scalar('losses/wd', loss_wd) 72 | 73 | post_ops.append(ema_op) 74 | train_op = tf.train.MomentumOptimizer(lr, 0.9, use_nesterov=True).minimize( 75 | loss + loss_mt * warmup * consistency_weight + wd * loss_wd, colocate_gradients_with_ops=True) 76 | with tf.control_dependencies([train_op]): 77 | train_op = tf.group(*post_ops) 78 | 79 | return EasyDict( 80 | xt=xt_in, x=x_in, y=y_in, label=l_in, train_op=train_op, 81 | classify_raw=tf.nn.softmax(classifier(x_in, training=False)), # No EMA, for debugging. 82 | classify_op=tf.nn.softmax(classifier(x_in, getter=ema_getter, training=False))) 83 | 84 | 85 | def main(argv): 86 | utils.setup_main() 87 | del argv # Unused. 88 | dataset = PAIR_DATASETS()[FLAGS.dataset]() 89 | log_width = utils.ilog2(dataset.width) 90 | model = MeanTeacher( 91 | os.path.join(FLAGS.train_dir, dataset.name), 92 | dataset, 93 | lr=FLAGS.lr, 94 | wd=FLAGS.wd, 95 | arch=FLAGS.arch, 96 | warmup_pos=FLAGS.warmup_pos, 97 | batch=FLAGS.batch, 98 | nclass=dataset.nclass, 99 | ema=FLAGS.ema, 100 | smoothing=FLAGS.smoothing, 101 | consistency_weight=FLAGS.consistency_weight, 102 | 103 | scales=FLAGS.scales or (log_width - 2), 104 | filters=FLAGS.filters, 105 | repeat=FLAGS.repeat) 106 | model.train(FLAGS.train_kimg << 10, FLAGS.report_kimg << 10) 107 | 108 | 109 | if __name__ == '__main__': 110 | utils.setup_tf() 111 | flags.DEFINE_float('consistency_weight', 50., 'Consistency weight.') 112 | flags.DEFINE_float('warmup_pos', 0.4, 'Relative position at which constraint loss warmup ends.') 113 | flags.DEFINE_float('wd', 0.0005, 'Weight decay.') 114 | flags.DEFINE_float('ema', 0.999, 'Exponential moving average of params.') 115 | flags.DEFINE_float('smoothing', 0.001, 'Label smoothing.') 116 | flags.DEFINE_integer('scales', 0, 'Number of 2x2 downscalings in the classifier.') 117 | flags.DEFINE_integer('filters', 32, 'Filter size of convolutions.') 118 | flags.DEFINE_integer('repeat', 4, 'Number of residual layers per stage.') 119 | FLAGS.set_default('augment', 'd.d.d') 120 | FLAGS.set_default('dataset', 'cifar10.3@250-5000') 121 | FLAGS.set_default('batch', 64) 122 | FLAGS.set_default('lr', 0.03) 123 | FLAGS.set_default('train_kimg', 1 << 16) 124 | app.run(main) 125 | -------------------------------------------------------------------------------- /TF-BOSS/mixmatch.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """MixMatch training. 15 | - Ensure class consistency by producing a group of `nu` augmentations of the same image and guessing the label for the 16 | group. 17 | - Sharpen the target distribution. 18 | - Use the sharpened distribution directly as a smooth label in MixUp. 19 | """ 20 | 21 | import functools 22 | import os 23 | 24 | import numpy as np 25 | import tensorflow as tf 26 | from absl import app 27 | from absl import flags 28 | 29 | from libml import layers, utils, models 30 | from libml.data import PAIR_DATASETS 31 | from libml.layers import MixMode 32 | from libml.utils import EasyDict 33 | 34 | FLAGS = flags.FLAGS 35 | 36 | 37 | class MixMatch(models.MultiModel): 38 | 39 | def distribution_summary(self, p_data, p_model, p_target=None): 40 | def kl(p, q): 41 | p /= tf.reduce_sum(p) 42 | q /= tf.reduce_sum(q) 43 | return -tf.reduce_sum(p * tf.log(q / p)) 44 | 45 | tf.summary.scalar('metrics/kld', kl(p_data, p_model)) 46 | if p_target is not None: 47 | tf.summary.scalar('metrics/kld_target', kl(p_data, p_target)) 48 | 49 | for i in range(self.nclass): 50 | tf.summary.scalar('matching/class%d_ratio' % i, p_model[i] / p_data[i]) 51 | for i in range(self.nclass): 52 | tf.summary.scalar('matching/val%d' % i, p_model[i]) 53 | 54 | def augment(self, x, l, beta, **kwargs): 55 | assert 0, 'Do not call.' 56 | 57 | def guess_label(self, y, classifier, T, **kwargs): 58 | del kwargs 59 | logits_y = [classifier(yi, training=True) for yi in y] 60 | logits_y = tf.concat(logits_y, 0) 61 | # Compute predicted probability distribution py. 62 | p_model_y = tf.reshape(tf.nn.softmax(logits_y), [len(y), -1, self.nclass]) 63 | p_model_y = tf.reduce_mean(p_model_y, axis=0) 64 | # Compute the target distribution. 65 | p_target = tf.pow(p_model_y, 1. / T) 66 | p_target /= tf.reduce_sum(p_target, axis=1, keep_dims=True) 67 | return EasyDict(p_target=p_target, p_model=p_model_y) 68 | 69 | def model(self, batch, lr, wd, ema, beta, w_match, warmup_kimg=1024, nu=2, mixmode='xxy.yxy', dbuf=128, **kwargs): 70 | hwc = [self.dataset.height, self.dataset.width, self.dataset.colors] 71 | xt_in = tf.placeholder(tf.float32, [batch] + hwc, 'xt') # For training 72 | x_in = tf.placeholder(tf.float32, [None] + hwc, 'x') 73 | y_in = tf.placeholder(tf.float32, [batch, nu] + hwc, 'y') 74 | l_in = tf.placeholder(tf.int32, [batch], 'labels') 75 | 76 | w_match *= tf.clip_by_value(tf.cast(self.step, tf.float32) / (warmup_kimg << 10), 0, 1) 77 | lrate = tf.clip_by_value(tf.to_float(self.step) / (FLAGS.train_kimg << 10), 0, 1) 78 | lr *= tf.cos(lrate * (7 * np.pi) / (2 * 8)) 79 | tf.summary.scalar('monitors/lr', lr) 80 | augment = MixMode(mixmode) 81 | classifier = lambda x, **kw: self.classifier(x, **kw, **kwargs).logits 82 | 83 | # Moving average of the current estimated label distribution 84 | p_model = layers.PMovingAverage('p_model', self.nclass, dbuf) 85 | p_target = layers.PMovingAverage('p_target', self.nclass, dbuf) # Rectified distribution (only for plotting) 86 | 87 | # Known (or inferred) true unlabeled distribution 88 | p_data = layers.PData(self.dataset) 89 | 90 | y = tf.reshape(tf.transpose(y_in, [1, 0, 2, 3, 4]), [-1] + hwc) 91 | guess = self.guess_label(tf.split(y, nu), classifier, T=0.5, **kwargs) 92 | ly = tf.stop_gradient(guess.p_target) 93 | lx = tf.one_hot(l_in, self.nclass) 94 | xy, labels_xy = augment([xt_in] + tf.split(y, nu), [lx] + [ly] * nu, [beta, beta]) 95 | x, y = xy[0], xy[1:] 96 | labels_x, labels_y = labels_xy[0], tf.concat(labels_xy[1:], 0) 97 | del xy, labels_xy 98 | 99 | batches = layers.interleave([x] + y, batch) 100 | skip_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) 101 | logits = [classifier(batches[0], training=True)] 102 | post_ops = [v for v in tf.get_collection(tf.GraphKeys.UPDATE_OPS) if v not in skip_ops] 103 | for batchi in batches[1:]: 104 | logits.append(classifier(batchi, training=True)) 105 | logits = layers.interleave(logits, batch) 106 | logits_x = logits[0] 107 | logits_y = tf.concat(logits[1:], 0) 108 | 109 | loss_xe = tf.nn.softmax_cross_entropy_with_logits_v2(labels=labels_x, logits=logits_x) 110 | loss_xe = tf.reduce_mean(loss_xe) 111 | loss_l2u = tf.square(labels_y - tf.nn.softmax(logits_y)) 112 | loss_l2u = tf.reduce_mean(loss_l2u) 113 | tf.summary.scalar('losses/xe', loss_xe) 114 | tf.summary.scalar('losses/l2u', loss_l2u) 115 | self.distribution_summary(p_data(), p_model(), p_target()) 116 | 117 | # L2 regularization 118 | loss_wd = sum(tf.nn.l2_loss(v) for v in utils.model_vars('classify') if 'kernel' in v.name) 119 | tf.summary.scalar('losses/wd', loss_wd) 120 | 121 | ema = tf.train.ExponentialMovingAverage(decay=ema) 122 | ema_op = ema.apply(utils.model_vars()) 123 | ema_getter = functools.partial(utils.getter_ema, ema) 124 | post_ops.append(ema_op) 125 | 126 | train_op = tf.train.MomentumOptimizer(lr, 0.9, use_nesterov=True).minimize( 127 | loss_xe + w_match * loss_l2u + wd * loss_wd, colocate_gradients_with_ops=True) 128 | with tf.control_dependencies([train_op]): 129 | train_op = tf.group(*post_ops) 130 | 131 | return EasyDict( 132 | xt=xt_in, x=x_in, y=y_in, label=l_in, train_op=train_op, 133 | classify_raw=tf.nn.softmax(classifier(x_in, training=False)), # No EMA, for debugging. 134 | classify_op=tf.nn.softmax(classifier(x_in, getter=ema_getter, training=False))) 135 | 136 | 137 | def main(argv): 138 | utils.setup_main() 139 | del argv # Unused. 140 | dataset = PAIR_DATASETS()[FLAGS.dataset]() 141 | log_width = utils.ilog2(dataset.width) 142 | model = MixMatch( 143 | os.path.join(FLAGS.train_dir, dataset.name), 144 | dataset, 145 | lr=FLAGS.lr, 146 | wd=FLAGS.wd, 147 | arch=FLAGS.arch, 148 | batch=FLAGS.batch, 149 | nclass=dataset.nclass, 150 | ema=FLAGS.ema, 151 | 152 | beta=FLAGS.beta, 153 | w_match=FLAGS.w_match, 154 | 155 | scales=FLAGS.scales or (log_width - 2), 156 | filters=FLAGS.filters, 157 | repeat=FLAGS.repeat) 158 | model.train(FLAGS.train_kimg << 10, FLAGS.report_kimg << 10) 159 | 160 | 161 | if __name__ == '__main__': 162 | utils.setup_tf() 163 | flags.DEFINE_float('wd', 0.0005, 'Weight decay.') 164 | flags.DEFINE_float('ema', 0.999, 'Exponential moving average of params.') 165 | flags.DEFINE_float('beta', 0.5, 'Mixup beta distribution.') 166 | flags.DEFINE_float('w_match', 100, 'Weight for distribution matching loss.') 167 | flags.DEFINE_integer('scales', 0, 'Number of 2x2 downscalings in the classifier.') 168 | flags.DEFINE_integer('filters', 32, 'Filter size of convolutions.') 169 | flags.DEFINE_integer('repeat', 4, 'Number of residual layers per stage.') 170 | FLAGS.set_default('augment', 'd.d.d') 171 | FLAGS.set_default('dataset', 'cifar10.3@250-5000') 172 | FLAGS.set_default('batch', 64) 173 | FLAGS.set_default('lr', 0.03) 174 | FLAGS.set_default('train_kimg', 1 << 16) 175 | app.run(main) 176 | -------------------------------------------------------------------------------- /TF-BOSS/mixup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """mixup: Beyond Empirical Risk Minimization. 15 | 16 | Adaption to SSL of MixUp: https://arxiv.org/abs/1710.09412 17 | """ 18 | import functools 19 | import os 20 | 21 | import tensorflow as tf 22 | from absl import app 23 | from absl import flags 24 | 25 | from libml import data, utils, models 26 | from libml.utils import EasyDict 27 | 28 | FLAGS = flags.FLAGS 29 | 30 | 31 | class Mixup(models.MultiModel): 32 | 33 | def augment(self, x, l, beta, **kwargs): 34 | del kwargs 35 | mix = tf.distributions.Beta(beta, beta).sample([tf.shape(x)[0], 1, 1, 1]) 36 | mix = tf.maximum(mix, 1 - mix) 37 | xmix = x * mix + x[::-1] * (1 - mix) 38 | lmix = l * mix[:, :, 0, 0] + l[::-1] * (1 - mix[:, :, 0, 0]) 39 | return xmix, lmix 40 | 41 | def model(self, batch, lr, wd, ema, **kwargs): 42 | hwc = [self.dataset.height, self.dataset.width, self.dataset.colors] 43 | xt_in = tf.placeholder(tf.float32, [batch] + hwc, 'xt') # For training 44 | x_in = tf.placeholder(tf.float32, [None] + hwc, 'x') 45 | y_in = tf.placeholder(tf.float32, [batch] + hwc, 'y') 46 | l_in = tf.placeholder(tf.int32, [batch], 'labels') 47 | wd *= lr 48 | classifier = lambda x, **kw: self.classifier(x, **kw, **kwargs).logits 49 | 50 | def get_logits(x): 51 | logits = classifier(x, training=True) 52 | return logits 53 | 54 | x, labels_x = self.augment(xt_in, tf.one_hot(l_in, self.nclass), **kwargs) 55 | logits_x = get_logits(x) 56 | post_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) 57 | y, labels_y = self.augment(y_in, tf.nn.softmax(get_logits(y_in)), **kwargs) 58 | labels_y = tf.stop_gradient(labels_y) 59 | logits_y = get_logits(y) 60 | 61 | loss_xe = tf.nn.softmax_cross_entropy_with_logits_v2(labels=labels_x, logits=logits_x) 62 | loss_xe = tf.reduce_mean(loss_xe) 63 | loss_xeu = tf.nn.softmax_cross_entropy_with_logits_v2(labels=labels_y, logits=logits_y) 64 | loss_xeu = tf.reduce_mean(loss_xeu) 65 | tf.summary.scalar('losses/xe', loss_xe) 66 | tf.summary.scalar('losses/xeu', loss_xeu) 67 | 68 | ema = tf.train.ExponentialMovingAverage(decay=ema) 69 | ema_op = ema.apply(utils.model_vars()) 70 | ema_getter = functools.partial(utils.getter_ema, ema) 71 | post_ops.append(ema_op) 72 | post_ops.extend([tf.assign(v, v * (1 - wd)) for v in utils.model_vars('classify') if 'kernel' in v.name]) 73 | 74 | train_op = tf.train.AdamOptimizer(lr).minimize(loss_xe + loss_xeu, colocate_gradients_with_ops=True) 75 | with tf.control_dependencies([train_op]): 76 | train_op = tf.group(*post_ops) 77 | 78 | return EasyDict( 79 | xt=xt_in, x=x_in, y=y_in, label=l_in, train_op=train_op, 80 | classify_raw=tf.nn.softmax(classifier(x_in, training=False)), # No EMA, for debugging. 81 | classify_op=tf.nn.softmax(classifier(x_in, getter=ema_getter, training=False))) 82 | 83 | 84 | def main(argv): 85 | utils.setup_main() 86 | del argv # Unused. 87 | dataset = data.DATASETS()[FLAGS.dataset]() 88 | log_width = utils.ilog2(dataset.width) 89 | model = Mixup( 90 | os.path.join(FLAGS.train_dir, dataset.name), 91 | dataset, 92 | lr=FLAGS.lr, 93 | wd=FLAGS.wd, 94 | arch=FLAGS.arch, 95 | batch=FLAGS.batch, 96 | nclass=dataset.nclass, 97 | ema=FLAGS.ema, 98 | beta=FLAGS.beta, 99 | 100 | scales=FLAGS.scales or (log_width - 2), 101 | filters=FLAGS.filters, 102 | repeat=FLAGS.repeat) 103 | model.train(FLAGS.train_kimg << 10, FLAGS.report_kimg << 10) 104 | 105 | 106 | if __name__ == '__main__': 107 | utils.setup_tf() 108 | flags.DEFINE_float('wd', 0.02, 'Weight decay.') 109 | flags.DEFINE_float('ema', 0.999, 'Exponential moving average of params.') 110 | flags.DEFINE_float('beta', 0.5, 'Mixup beta distribution.') 111 | flags.DEFINE_integer('scales', 0, 'Number of 2x2 downscalings in the classifier.') 112 | flags.DEFINE_integer('filters', 32, 'Filter size of convolutions.') 113 | flags.DEFINE_integer('repeat', 4, 'Number of residual layers per stage.') 114 | FLAGS.set_default('dataset', 'cifar10.3@250-5000') 115 | FLAGS.set_default('batch', 64) 116 | FLAGS.set_default('lr', 0.002) 117 | FLAGS.set_default('train_kimg', 1 << 16) 118 | app.run(main) 119 | -------------------------------------------------------------------------------- /TF-BOSS/newTest.py: -------------------------------------------------------------------------------- 1 | #import tensorflow as tf 2 | import numpy as np 3 | 4 | train_dir = 'experiments/BOSS/dbg/cifar10p.d.d.d.1@10-1/CTAugment_depth2_th0.80_decay0.990/FixMatch_archresnet_balance2_batch32_confidence0.95_delT0.0_filters32_lr0.04_mom0.88_nclass10_repeat4_scales3_uratio7_wc0.0_wd0.0004_wu1.0' 5 | print(train_dir) 6 | 7 | dir = train_dir.replace("BOSS","preds") 8 | print(dir) 9 | 10 | exit(1) 11 | -------------------------------------------------------------------------------- /TF-BOSS/pi_model.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Temporal Ensembling for Semi-Supervised Learning. 15 | 16 | Pi-model reimplementation of https://arxiv.org/abs/1610.02242 17 | """ 18 | 19 | import functools 20 | import os 21 | 22 | import numpy as np 23 | import tensorflow as tf 24 | from absl import app 25 | from absl import flags 26 | 27 | from libml import models, utils 28 | from libml.data import PAIR_DATASETS 29 | from libml.utils import EasyDict 30 | 31 | FLAGS = flags.FLAGS 32 | 33 | 34 | class PiModel(models.MultiModel): 35 | 36 | def model(self, batch, lr, wd, ema, warmup_pos, consistency_weight, **kwargs): 37 | hwc = [self.dataset.height, self.dataset.width, self.dataset.colors] 38 | xt_in = tf.placeholder(tf.float32, [batch] + hwc, 'xt') # For training 39 | x_in = tf.placeholder(tf.float32, [None] + hwc, 'x') 40 | y_in = tf.placeholder(tf.float32, [batch, 2] + hwc, 'y') 41 | l_in = tf.placeholder(tf.int32, [batch], 'labels') 42 | l = tf.one_hot(l_in, self.nclass) 43 | 44 | warmup = tf.clip_by_value(tf.to_float(self.step) / (warmup_pos * (FLAGS.train_kimg << 10)), 0, 1) 45 | lrate = tf.clip_by_value(tf.to_float(self.step) / (FLAGS.train_kimg << 10), 0, 1) 46 | lr *= tf.cos(lrate * (7 * np.pi) / (2 * 8)) 47 | tf.summary.scalar('monitors/lr', lr) 48 | 49 | classifier = lambda x, **kw: self.classifier(x, **kw, **kwargs).logits 50 | logits_x = classifier(xt_in, training=True) 51 | post_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # Take only first call to update batch norm. 52 | y = tf.reshape(tf.transpose(y_in, [1, 0, 2, 3, 4]), [-1] + hwc) 53 | y_1, y_2 = tf.split(y, 2) 54 | logits_y = classifier(y_1, training=True) 55 | logits_teacher = tf.stop_gradient(logits_y) 56 | logits_student = classifier(y_2, training=True) 57 | loss_pm = tf.reduce_mean((tf.nn.softmax(logits_teacher) - tf.nn.softmax(logits_student)) ** 2, -1) 58 | loss_pm = tf.reduce_mean(loss_pm) 59 | 60 | loss = tf.nn.softmax_cross_entropy_with_logits_v2(labels=l, logits=logits_x) 61 | loss = tf.reduce_mean(loss) 62 | tf.summary.scalar('losses/xe', loss) 63 | tf.summary.scalar('losses/pm', loss_pm) 64 | 65 | # L2 regularization 66 | loss_wd = sum(tf.nn.l2_loss(v) for v in utils.model_vars('classify') if 'kernel' in v.name) 67 | tf.summary.scalar('losses/wd', loss_wd) 68 | 69 | ema = tf.train.ExponentialMovingAverage(decay=ema) 70 | ema_op = ema.apply(utils.model_vars()) 71 | ema_getter = functools.partial(utils.getter_ema, ema) 72 | post_ops.append(ema_op) 73 | 74 | train_op = tf.train.MomentumOptimizer(lr, 0.9, use_nesterov=True).minimize( 75 | loss + loss_pm * warmup * consistency_weight + wd * loss_wd, colocate_gradients_with_ops=True) 76 | with tf.control_dependencies([train_op]): 77 | train_op = tf.group(*post_ops) 78 | 79 | return EasyDict( 80 | xt=xt_in, x=x_in, y=y_in, label=l_in, train_op=train_op, 81 | classify_raw=tf.nn.softmax(classifier(x_in, training=False)), # No EMA, for debugging. 82 | classify_op=tf.nn.softmax(classifier(x_in, getter=ema_getter, training=False))) 83 | 84 | 85 | def main(argv): 86 | utils.setup_main() 87 | del argv # Unused. 88 | dataset = PAIR_DATASETS()[FLAGS.dataset]() 89 | log_width = utils.ilog2(dataset.width) 90 | model = PiModel( 91 | os.path.join(FLAGS.train_dir, dataset.name), 92 | dataset, 93 | lr=FLAGS.lr, 94 | wd=FLAGS.wd, 95 | arch=FLAGS.arch, 96 | warmup_pos=FLAGS.warmup_pos, 97 | batch=FLAGS.batch, 98 | nclass=dataset.nclass, 99 | ema=FLAGS.ema, 100 | smoothing=FLAGS.smoothing, 101 | consistency_weight=FLAGS.consistency_weight, 102 | 103 | scales=FLAGS.scales or (log_width - 2), 104 | filters=FLAGS.filters, 105 | repeat=FLAGS.repeat) 106 | model.train(FLAGS.train_kimg << 10, FLAGS.report_kimg << 10) 107 | 108 | 109 | if __name__ == '__main__': 110 | utils.setup_tf() 111 | flags.DEFINE_float('consistency_weight', 10., 'Consistency weight.') 112 | flags.DEFINE_float('warmup_pos', 0.4, 'Relative position at which constraint loss warmup ends.') 113 | flags.DEFINE_float('wd', 0.0005, 'Weight decay.') 114 | flags.DEFINE_float('ema', 0.999, 'Exponential moving average of params.') 115 | flags.DEFINE_float('smoothing', 0.1, 'Label smoothing.') 116 | flags.DEFINE_integer('scales', 0, 'Number of 2x2 downscalings in the classifier.') 117 | flags.DEFINE_integer('filters', 32, 'Filter size of convolutions.') 118 | flags.DEFINE_integer('repeat', 4, 'Number of residual layers per stage.') 119 | FLAGS.set_default('augment', 'd.d.d') 120 | FLAGS.set_default('dataset', 'cifar10.3@250-5000') 121 | FLAGS.set_default('batch', 64) 122 | FLAGS.set_default('lr', 0.03) 123 | FLAGS.set_default('train_kimg', 1 << 16) 124 | app.run(main) 125 | -------------------------------------------------------------------------------- /TF-BOSS/pseudo_label.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Pseudo-label: The simple and efficient semi-supervised learning method fordeep neural networks. 15 | 16 | Reimplementation of http://deeplearning.net/wp-content/uploads/2013/03/pseudo_label_final.pdf 17 | """ 18 | 19 | import functools 20 | import os 21 | 22 | import numpy as np 23 | import tensorflow as tf 24 | from absl import app 25 | from absl import flags 26 | 27 | from libml import utils, data, models 28 | from libml.utils import EasyDict 29 | 30 | FLAGS = flags.FLAGS 31 | 32 | 33 | class PseudoLabel(models.MultiModel): 34 | 35 | def model(self, batch, lr, wd, ema, warmup_pos, consistency_weight, threshold, **kwargs): 36 | hwc = [self.dataset.height, self.dataset.width, self.dataset.colors] 37 | xt_in = tf.placeholder(tf.float32, [batch] + hwc, 'xt') # For training 38 | x_in = tf.placeholder(tf.float32, [None] + hwc, 'x') 39 | y_in = tf.placeholder(tf.float32, [batch] + hwc, 'y') 40 | l_in = tf.placeholder(tf.int32, [batch], 'labels') 41 | l = tf.one_hot(l_in, self.nclass) 42 | 43 | warmup = tf.clip_by_value(tf.to_float(self.step) / (warmup_pos * (FLAGS.train_kimg << 10)), 0, 1) 44 | lrate = tf.clip_by_value(tf.to_float(self.step) / (FLAGS.train_kimg << 10), 0, 1) 45 | lr *= tf.cos(lrate * (7 * np.pi) / (2 * 8)) 46 | tf.summary.scalar('monitors/lr', lr) 47 | 48 | classifier = lambda x, **kw: self.classifier(x, **kw, **kwargs).logits 49 | logits_x = classifier(xt_in, training=True) 50 | post_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # Take only first call to update batch norm. 51 | logits_y = classifier(y_in, training=True) 52 | # Get the pseudo-label loss 53 | loss_pl = tf.nn.sparse_softmax_cross_entropy_with_logits( 54 | labels=tf.argmax(logits_y, axis=-1), logits=logits_y 55 | ) 56 | # Masks denoting which data points have high-confidence predictions 57 | greater_than_thresh = tf.reduce_any( 58 | tf.greater(tf.nn.softmax(logits_y), threshold), 59 | axis=-1, 60 | keepdims=True, 61 | ) 62 | greater_than_thresh = tf.cast(greater_than_thresh, loss_pl.dtype) 63 | # Only enforce the loss when the model is confident 64 | loss_pl *= greater_than_thresh 65 | # Note that we also average over examples without confident outputs; 66 | # this is consistent with the realistic evaluation codebase 67 | loss_pl = tf.reduce_mean(loss_pl) 68 | 69 | loss = tf.nn.softmax_cross_entropy_with_logits_v2(labels=l, logits=logits_x) 70 | loss = tf.reduce_mean(loss) 71 | tf.summary.scalar('losses/xe', loss) 72 | tf.summary.scalar('losses/pl', loss_pl) 73 | 74 | # L2 regularization 75 | loss_wd = sum(tf.nn.l2_loss(v) for v in utils.model_vars('classify') if 'kernel' in v.name) 76 | tf.summary.scalar('losses/wd', loss_wd) 77 | 78 | ema = tf.train.ExponentialMovingAverage(decay=ema) 79 | ema_op = ema.apply(utils.model_vars()) 80 | ema_getter = functools.partial(utils.getter_ema, ema) 81 | post_ops.append(ema_op) 82 | 83 | train_op = tf.train.MomentumOptimizer(lr, 0.9, use_nesterov=True).minimize( 84 | loss + loss_pl * warmup * consistency_weight + wd * loss_wd, colocate_gradients_with_ops=True) 85 | with tf.control_dependencies([train_op]): 86 | train_op = tf.group(*post_ops) 87 | 88 | return EasyDict( 89 | xt=xt_in, x=x_in, y=y_in, label=l_in, train_op=train_op, 90 | classify_raw=tf.nn.softmax(classifier(x_in, training=False)), # No EMA, for debugging. 91 | classify_op=tf.nn.softmax(classifier(x_in, getter=ema_getter, training=False))) 92 | 93 | 94 | def main(argv): 95 | utils.setup_main() 96 | del argv # Unused. 97 | dataset = data.DATASETS()[FLAGS.dataset]() 98 | log_width = utils.ilog2(dataset.width) 99 | model = PseudoLabel( 100 | os.path.join(FLAGS.train_dir, dataset.name), 101 | dataset, 102 | lr=FLAGS.lr, 103 | wd=FLAGS.wd, 104 | arch=FLAGS.arch, 105 | warmup_pos=FLAGS.warmup_pos, 106 | batch=FLAGS.batch, 107 | nclass=dataset.nclass, 108 | ema=FLAGS.ema, 109 | smoothing=FLAGS.smoothing, 110 | consistency_weight=FLAGS.consistency_weight, 111 | threshold=FLAGS.threshold, 112 | 113 | scales=FLAGS.scales or (log_width - 2), 114 | filters=FLAGS.filters, 115 | repeat=FLAGS.repeat) 116 | model.train(FLAGS.train_kimg << 10, FLAGS.report_kimg << 10) 117 | 118 | 119 | if __name__ == '__main__': 120 | utils.setup_tf() 121 | flags.DEFINE_float('wd', 0.0005, 'Weight decay.') 122 | flags.DEFINE_float('consistency_weight', 1., 'Consistency weight.') 123 | flags.DEFINE_float('threshold', 0.95, 'Pseudo-label threshold.') 124 | flags.DEFINE_float('warmup_pos', 0.4, 'Relative position at which constraint loss warmup ends.') 125 | flags.DEFINE_float('ema', 0.999, 'Exponential moving average of params.') 126 | flags.DEFINE_float('smoothing', 0.1, 'Label smoothing.') 127 | flags.DEFINE_integer('scales', 0, 'Number of 2x2 downscalings in the classifier.') 128 | flags.DEFINE_integer('filters', 32, 'Filter size of convolutions.') 129 | flags.DEFINE_integer('repeat', 4, 'Number of residual layers per stage.') 130 | FLAGS.set_default('dataset', 'cifar10.3@250-5000') 131 | FLAGS.set_default('batch', 64) 132 | FLAGS.set_default('lr', 0.03) 133 | FLAGS.set_default('train_kimg', 1 << 16) 134 | app.run(main) 135 | -------------------------------------------------------------------------------- /TF-BOSS/requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py 2 | easydict 3 | cython 4 | numpy 5 | #tensorflow-gpu==1.14.0 6 | tqdm 7 | -------------------------------------------------------------------------------- /TF-BOSS/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export ML_DATA="./data" 3 | export PYTHONPATH=$PYTHONPATH:$PWD 4 | 5 | bal=4 #set balance method, 1,2,3 or 4 6 | seed=6 #set seed choose number 1 to 7 7 | ds=cifar10 #set dataset; either cifar10 or svhn 8 | 9 | iterSize=400 # set to the number of pseudolabeled used for the self-training iteration. 10 | 11 | dataset=${ds}p; size=10; valid=1; time=48; arch=resnet 12 | kimg=32768; aug="d.d.d"; mom=0.88; wu=1; #hyperparameters 13 | 14 | datastring='B'$bal'S'$seed'.npy' 15 | datastring2=${dataset}.${seed}@${size}.npy 16 | 17 | if [ $bal -eq 0 ]; 18 | then 19 | delt=0; con=0.95;wd=5e-4;lr=0.03; ratio=7; batch=64 20 | elif [ $bal -eq 1 ]; 21 | then 22 | delt=0.25; con=0.95;wd=8e-4;lr=0.06; ratio=9; batch=30 23 | elif [ $bal -eq 2 ]; 24 | then 25 | delt=0; con=0.9;wd=8e-4;lr=0.06; ratio=9; batch=30 26 | elif [ $bal -eq 3 ]; 27 | then 28 | delt=0; con=0.9;wd=8e-4;lr=0.04; ratio=9; batch=30 29 | elif [ $bal -eq 4 ]; 30 | then 31 | delt=0.25; con=0.95;wd=8e-4;lr=0.06; ratio=9; batch=30 32 | else 33 | delt=0; con=0.95;wd=5e-4;lr=0.03; ratio=7; batch=64 34 | fi 35 | 36 | echo "Running BOSS for balance = " $bal 37 | CUDA_VISIBLE_DEVICES=0 python fixmatch.py --train_kimg $kimg --uratio $ratio --confidence $con --wd $wd --wu $wu --batch $batch --lr $lr --arch $arch --filters 32 --scales 3 --repeat 4 --dataset=${dataset}.${seed}@${size}-${valid} --delT $delt --train_dir experiments/BOSS/ --augment $aug --mom $mom --balance $bal 38 | 39 | mv top_probs$datastring ./data/pseudolabeled/top/Probs$datastring2 40 | mv top_labels$datastring ./data/pseudolabeled/top/Labels$datastring2 41 | mv true_labels$datastring ./data/pseudolabeled/top/TrueLabels$datastring2 42 | 43 | 44 | export ML_DATA="./data/pseudolabeled" #change to pseudolabeled data 45 | 46 | echo "Running " ${ds}_iteration.py 47 | python scripts/${ds}_iteration.py --seed=$seed --size=$iterSize --pseudo_file data/pseudolabeled/top/Probs$datastring2 $ML_DATA/SSL2/$dataset $ML_DATA/${dataset}-train.tfrecord 48 | 49 | dataset2=${dataset}; 50 | iterSize2=$(($size + $iterSize)) 51 | bal=0; con=0.95;delt=0; mom=0.9; ratio=7; wd=5e-4; batch=64; lr=0.03; #these hyperparameters change 52 | datastring4=${dataset2}.${seed}@${iterSize2} 53 | 54 | echo "Running self-training " 55 | CUDA_VISIBLE_DEVICES=0 python fixmatch.py --train_kimg $kimg --uratio $ratio --confidence $con --wd $wd --wu $wu --batch $batch --lr $lr --arch $arch --filters 32 --scales 3 --repeat 4 --dataset=${datastring4}-${valid} --delT $delt --train_dir experiments/BOSS/iter --augment $aug --mom $mom --balance $bal 56 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/accuracy_table.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import json 18 | import sys 19 | 20 | with open(sys.argv[1], 'r') as f: 21 | d = json.load(f) 22 | 23 | d = {x: y.get('last020', y.get('last20')) for x, y in d.items()} 24 | l = list(d.items()) 25 | for x in sorted([('/'.join(x[0].split('/')[-6:-2]), x[1]) for x in l]): 26 | print('%.2f %s' % (x[1], x[0])) 27 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/aggregate_accuracy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Report all 'stats/accuracy.json' into a json file on stdout. 16 | 17 | All the accuracies are summarized. 18 | """ 19 | import json 20 | import sys 21 | import threading 22 | 23 | import tensorflow as tf 24 | import tqdm 25 | from absl import app 26 | from absl import flags 27 | 28 | FLAGS = flags.FLAGS 29 | N_THREADS = 100 30 | 31 | 32 | def add_contents_to_dict(filename: str, target): 33 | with tf.gfile.Open(filename, 'r') as f: 34 | target[filename] = json.load(f) 35 | 36 | 37 | def main(argv): 38 | files = [] 39 | for path in argv[1:]: 40 | files.extend(tf.io.gfile.glob(path)) 41 | assert files, 'No files found' 42 | print('Found %d files.' % len(files), file=sys.stderr) 43 | summary = {} 44 | threads = [] 45 | for x in tqdm.tqdm(files, leave=False, desc='Collating'): 46 | t = threading.Thread( 47 | target=add_contents_to_dict, kwargs=dict(filename=x, target=summary)) 48 | threads.append(t) 49 | t.start() 50 | while len(threads) >= N_THREADS: 51 | dead = [p for p, t in enumerate(threads) if not t.is_alive()] 52 | while dead: 53 | p = dead.pop() 54 | del threads[p] 55 | if x == files[-1]: 56 | for t in threads: 57 | t.join() 58 | 59 | assert len(summary) == len(files) 60 | print(json.dumps(summary, sort_keys=True, indent=4)) 61 | 62 | 63 | if __name__ == '__main__': 64 | app.run(main) 65 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/check_split.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Script to measure the overlap between data splits. 18 | 19 | There should not be any overlap unless the original dataset has duplicates. 20 | """ 21 | 22 | import hashlib 23 | import os 24 | 25 | import tensorflow as tf 26 | from absl import app 27 | from absl import flags 28 | from tqdm import trange 29 | 30 | from libml import data, utils 31 | 32 | flags.DEFINE_integer('batch', 1024, 'Batch size.') 33 | flags.DEFINE_integer('samples', 1 << 20, 'Number of samples to load.') 34 | 35 | FLAGS = flags.FLAGS 36 | 37 | 38 | def to_byte(d: dict): 39 | return tf.to_int32(tf.round(127.5 * (d['image'] + 1))) 40 | 41 | 42 | def collect_hashes(sess, group, data): 43 | data = data.parse().batch(FLAGS.batch).prefetch(1).make_one_shot_iterator().get_next() 44 | hashes = set() 45 | hasher = hashlib.sha512 46 | for _ in trange(0, FLAGS.samples, FLAGS.batch, desc='Building hashes for %s' % group, leave=False): 47 | try: 48 | batch = sess.run(data) 49 | except tf.errors.OutOfRangeError: 50 | break 51 | for img in batch: 52 | hashes.add(hasher(img).digest()) 53 | return hashes 54 | 55 | 56 | def main(argv): 57 | utils.setup_main() 58 | del argv 59 | utils.setup_tf() 60 | dataset = data.DATASETS()[FLAGS.dataset]() 61 | with tf.Session(config=utils.get_config()) as sess: 62 | hashes = (collect_hashes(sess, 'labeled', dataset.eval_labeled), 63 | collect_hashes(sess, 'unlabeled', dataset.eval_unlabeled), 64 | collect_hashes(sess, 'validation', dataset.valid), 65 | collect_hashes(sess, 'test', dataset.test)) 66 | print('Overlap matrix (should be an almost perfect diagonal matrix with counts).') 67 | groups = 'labeled unlabeled validation test'.split() 68 | fmt = '%-10s %10s %10s %10s %10s' 69 | print(fmt % tuple([''] + groups)) 70 | for p, x in enumerate(hashes): 71 | overlaps = [len(x & y) for y in hashes] 72 | print(fmt % tuple([groups[p]] + overlaps)) 73 | 74 | 75 | if __name__ == '__main__': 76 | os.environ['CUDA_VISIBLE_DEVICES'] = '' 77 | app.run(main) 78 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/cifar10_iteration.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Self-train iteration for Cifar-10 3 | 4 | 5 | # Copyright 2019 Google LLC 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | """Script to create SSL splits from a dataset. 20 | """ 21 | 22 | import json 23 | import os 24 | from collections import defaultdict 25 | 26 | import numpy as np 27 | import tensorflow as tf 28 | from absl import app 29 | from absl import flags 30 | from tqdm import trange, tqdm 31 | 32 | from libml import data as libml_data 33 | from libml import utils 34 | 35 | flags.DEFINE_integer('seed', 0, 'Random seed to use, 0 for no shuffling.') 36 | flags.DEFINE_integer('size', 0, 'Size of labelled set.') 37 | flags.DEFINE_string('pseudo_file', None, 38 | 'Directory of top pseudolabeled examples. ') 39 | 40 | 41 | FLAGS = flags.FLAGS 42 | 43 | 44 | def get_class(serialized_example): 45 | return tf.parse_single_example(serialized_example, features={'label': tf.FixedLenFeature([], tf.int64)})['label'] 46 | 47 | 48 | def main(argv): 49 | assert FLAGS.size 50 | argv.pop(0) 51 | if any(not tf.gfile.Exists(f) for f in argv[1:]): 52 | raise FileNotFoundError(argv[1:]) 53 | target = '%s.%d@%d' % (argv[0], FLAGS.seed, FLAGS.size+10) 54 | if tf.gfile.Exists(target): 55 | raise FileExistsError('For safety overwriting is not allowed', target) 56 | input_files = argv[1:] 57 | print("=> input_files= ", input_files) 58 | print("=> target_file= ", target) 59 | print("=> pseudo_file= ", FLAGS.pseudo_file) 60 | count = 0 61 | id_class = [] 62 | class_id = defaultdict(list) 63 | print('Computing class distribution') 64 | dataset = tf.data.TFRecordDataset(input_files).map(get_class, 4).batch(1 << 10) 65 | it = dataset.make_one_shot_iterator().get_next() 66 | #### Prototype code 67 | if (FLAGS.seed ==0): 68 | label= [35, 4, 18, 9, 20, 128, 19, 87, 69, 14] 69 | elif (FLAGS.seed == 1): 70 | label= [165, 61, 108, 91, 58, 156, 104, 163, 106, 1] 71 | elif (FLAGS.seed == 2): 72 | label= [199, 105, 120, 101, 149, 182, 124, 152, 111, 53] 73 | elif (FLAGS.seed == 3): 74 | label= [213, 160, 138, 169, 34, 177, 210, 172, 190, 109] 75 | elif (FLAGS.seed == 4): 76 | # 1 2 3 4 5 6 7 8 9 77 | label= [233, 176, 194, 33, 28, 198, 245, 289, 192, 225] 78 | elif (FLAGS.seed == 5): 79 | label= [199, 226, 138, 266, 343, 215, 326, 318, 216, 219] 80 | elif (FLAGS.seed == 6): # Based on prototype refining for seed=2 81 | #From seeds 3 2 2 2 2 2 2 2 2 1 82 | label= [213, 105, 120, 101, 149, 182, 124, 152, 111, 1] 83 | elif (FLAGS.seed == 7): # Based on prototype refining for seed=4 84 | #From seeds 4 4 4 1 4 2 4 4 4 4 85 | label= [233, 176, 194, 91, 28, 182, 245, 289, 192, 225] 86 | elif (FLAGS.seed == 8): 87 | label= [42112, 46875, 27460, 15846, 35142, 45802, 39501, 8628, 48571, 49791] 88 | else: 89 | print(FLAGS.seed, " seed not defined") 90 | exit(1) 91 | print("label ",label) 92 | #### End Prototype code 93 | try: 94 | with tf.Session() as session, tqdm(leave=False) as t: 95 | while 1: 96 | old_count = count 97 | for i in session.run(it): 98 | id_class.append(i) # i is the class label 99 | class_id[i].append(count) # count is sample number 100 | if count in label: 101 | print("class, count ",i,count) 102 | count += 1 103 | t.update(count - old_count) 104 | except tf.errors.OutOfRangeError: 105 | pass 106 | print('%d records found' % count) 107 | #### Code for self train iterations 108 | print("=> id_class.shape,class_id.shape= ", len(id_class) ,len(class_id) ) 109 | nclass = len(class_id) 110 | if FLAGS.pseudo_file: 111 | ID = np.load(FLAGS.pseudo_file) 112 | lab_name = FLAGS.pseudo_file.replace("Probs","Labels") 113 | id_labels = np.load(lab_name) 114 | lab_name = FLAGS.pseudo_file.replace("Probs","TrueLabels") 115 | true_labels = np.load(lab_name) 116 | # id_labels = true_labels 117 | # for j in range( len(ID)): 118 | # print("Sample number ",ID[j]," pseudo-label, true-label, id_class ", 119 | # id_labels[j],true_labels[j],id_class[ID[j]]) 120 | # exit(1) 121 | 122 | # print("size ID ", len(ID)) 123 | # print("size id_labels ", len(id_labels)) 124 | # print("ID ", ID[0:20]) 125 | print("id_labels ", id_labels[0:20]) 126 | print("true_labels ", true_labels[0:20]) 127 | sz = round(FLAGS.size / nclass) 128 | sortByClass = -np.ones([nclass,2000],dtype=int) 129 | sortIndx = [0]*nclass 130 | for j in range( len(ID)): 131 | sortByClass[id_labels[j]][sortIndx[id_labels[j]]] = ID[j] 132 | sortIndx[id_labels[j]] += 1 133 | print("sortIndx ", sortIndx) 134 | # print(" sortByClass ", sortByClass[0:10]) 135 | er = 0 136 | for j in range(sz): 137 | for i in range(nclass): 138 | assert sortIndx[i] >= sz 139 | if id_class[sortByClass[i][j]] != i: 140 | # print("Error for ",sortByClass[i][j] ," : Pseudo label ",i, 141 | # " actual label ",id_class[sortByClass[i][j]]," +-1 ",id_class[sortByClass[i][j]-1],id_class[sortByClass[i][j]+1]) 142 | er += 1 143 | label.append(sortByClass[i][j]) 144 | print(" size, number of errors, label ", len(label),er, label) 145 | del ID 146 | # exit(1) 147 | #### End Prototype code 148 | assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1) 149 | train_stats = np.array([len(class_id[i]) for i in range(nclass)], np.float64) 150 | train_stats /= train_stats.max() 151 | if 'stl10' in argv[1]: 152 | # All of the unlabeled data is given label 0, but we know that 153 | # STL has equally distributed data among the 10 classes. 154 | train_stats[:] = 1 155 | 156 | print(' Stats', ' '.join(['%.2f' % (100 * x) for x in train_stats])) 157 | assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1) 158 | class_id = [np.array(class_id[i], dtype=np.int64) for i in range(nclass)] 159 | del class_id 160 | # print("===> label= ", label) 161 | label = frozenset([int(x) for x in label]) 162 | print("===> label= ", label) 163 | if 'stl10' in argv[1] and FLAGS.size == 1000: 164 | data = tf.gfile.Open(os.path.join(libml_data.DATA_DIR, 'stl10_fold_indices.txt'), 'r').read() 165 | label = frozenset(list(map(int, data.split('\n')[FLAGS.seed].split()))) 166 | 167 | print('Creating split in %s' % target) 168 | tf.gfile.MakeDirs(os.path.dirname(target)) 169 | with tf.python_io.TFRecordWriter(target + '-label.tfrecord') as writer_label: 170 | pos, loop = 0, trange(count, desc='Writing records') 171 | for input_file in input_files: 172 | for record in tf.python_io.tf_record_iterator(input_file): 173 | if pos in label: 174 | writer_label.write(record) 175 | pos += 1 176 | loop.update() 177 | loop.close() 178 | with tf.gfile.Open(target + '-label.json', 'w') as writer: 179 | writer.write(json.dumps(dict(distribution=train_stats.tolist(), label=sorted(label)), indent=2, sort_keys=True)) 180 | 181 | 182 | if __name__ == '__main__': 183 | utils.setup_tf() 184 | app.run(main) 185 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/cifar10_prototypes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Script to create SSL splits from a dataset. 18 | """ 19 | 20 | import json 21 | import os 22 | from collections import defaultdict 23 | 24 | import numpy as np 25 | import tensorflow as tf 26 | from absl import app 27 | from absl import flags 28 | from tqdm import trange, tqdm 29 | 30 | from libml import data as libml_data 31 | from libml import utils 32 | 33 | flags.DEFINE_integer('seed', 0, 'Random seed to use, 0 for no shuffling.') 34 | flags.DEFINE_integer('size', 0, 'Size of labelled set.') 35 | 36 | FLAGS = flags.FLAGS 37 | 38 | 39 | def get_class(serialized_example): 40 | return tf.parse_single_example(serialized_example, features={'label': tf.FixedLenFeature([], tf.int64)})['label'] 41 | 42 | 43 | def main(argv): 44 | assert FLAGS.size 45 | argv.pop(0) 46 | if any(not tf.gfile.Exists(f) for f in argv[1:]): 47 | raise FileNotFoundError(argv[1:]) 48 | target = '%s.%d@%d' % (argv[0], FLAGS.seed, FLAGS.size) 49 | if tf.gfile.Exists(target): 50 | raise FileExistsError('For safety overwriting is not allowed', target) 51 | input_files = argv[1:] 52 | print("=> input_files= ", input_files) 53 | count = 0 54 | id_class = [] 55 | class_id = defaultdict(list) 56 | print('Computing class distribution') 57 | dataset = tf.data.TFRecordDataset(input_files).map(get_class, 4).batch(1 << 10) 58 | it = dataset.make_one_shot_iterator().get_next() 59 | #### Prototype code 60 | if (FLAGS.seed ==0): 61 | label= [35, 4, 18, 9, 20, 128, 19, 87, 69, 14] 62 | elif (FLAGS.seed == 1): 63 | label= [165, 61, 108, 91, 58, 156, 104, 163, 106, 1] 64 | elif (FLAGS.seed == 2): 65 | label= [199, 105, 120, 101, 149, 182, 124, 152, 111, 53] 66 | elif (FLAGS.seed == 3): 67 | label= [213, 160, 138, 169, 34, 177, 210, 172, 190, 109] 68 | elif (FLAGS.seed == 4): 69 | # 1 2 3 4 5 6 7 8 9 70 | label= [233, 176, 194, 33, 28, 198, 245, 289, 192, 225] 71 | elif (FLAGS.seed == 5): 72 | label= [199, 226, 138, 266, 343, 215, 326, 318, 216, 219] 73 | elif (FLAGS.seed == 6): # Based on prototype refining for seed=2 74 | #From seeds 3 2 2 2 2 2 2 2 2 1 75 | label= [213, 105, 120, 101, 149, 182, 124, 152, 111, 1] 76 | elif (FLAGS.seed == 7): # Based on prototype refining for seed=4 77 | #From seeds 4 4 4 1 4 2 4 4 4 4 78 | label= [233, 176, 194, 91, 28, 182, 245, 289, 192, 225] 79 | elif (FLAGS.seed == 8): 80 | label= [42112, 46875, 27460, 15846, 35142, 45802, 39501, 8628, 48571, 49791] 81 | else: 82 | print(FLAGS.seed, " seed not defined") 83 | exit(1) 84 | print("label ",label) 85 | #### End Prototype code 86 | try: 87 | with tf.Session() as session, tqdm(leave=False) as t: 88 | while 1: 89 | old_count = count 90 | for i in session.run(it): 91 | id_class.append(i) 92 | class_id[i].append(count) 93 | if count in label: 94 | print("class, count ",i,count) 95 | count += 1 96 | t.update(count - old_count) 97 | except tf.errors.OutOfRangeError: 98 | pass 99 | print('%d records found' % count) 100 | print("=> id_class.shape,class_id.shape= ", len(id_class) ,len(class_id) ) 101 | nclass = len(class_id) 102 | for i in range(nclass): 103 | print("i,class_id[i][0:32]= ",i,class_id[i][0:32]) 104 | assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1) 105 | train_stats = np.array([len(class_id[i]) for i in range(nclass)], np.float64) 106 | train_stats /= train_stats.max() 107 | if 'stl10' in argv[1]: 108 | # All of the unlabeled data is given label 0, but we know that 109 | # STL has equally distributed data among the 10 classes. 110 | train_stats[:] = 1 111 | 112 | print(' Stats', ' '.join(['%.2f' % (100 * x) for x in train_stats])) 113 | assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1) 114 | class_id = [np.array(class_id[i], dtype=np.int64) for i in range(nclass)] 115 | del class_id 116 | print("===> label= ", label) 117 | label = frozenset([int(x) for x in label]) 118 | print("===> label= ", label) 119 | if 'stl10' in argv[1] and FLAGS.size == 1000: 120 | data = tf.gfile.Open(os.path.join(libml_data.DATA_DIR, 'stl10_fold_indices.txt'), 'r').read() 121 | label = frozenset(list(map(int, data.split('\n')[FLAGS.seed].split()))) 122 | 123 | print('Creating split in %s' % target) 124 | tf.gfile.MakeDirs(os.path.dirname(target)) 125 | with tf.python_io.TFRecordWriter(target + '-label.tfrecord') as writer_label: 126 | pos, loop = 0, trange(count, desc='Writing records') 127 | for input_file in input_files: 128 | for record in tf.python_io.tf_record_iterator(input_file): 129 | if pos in label: 130 | writer_label.write(record) 131 | pos += 1 132 | loop.update() 133 | loop.close() 134 | with tf.gfile.Open(target + '-label.json', 'w') as writer: 135 | writer.write(json.dumps(dict(distribution=train_stats.tolist(), label=sorted(label)), indent=2, sort_keys=True)) 136 | 137 | 138 | if __name__ == '__main__': 139 | utils.setup_tf() 140 | app.run(main) 141 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/createSummary.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import numpy as np 4 | import glob 5 | #import math 6 | 7 | 8 | ''' 9 | Reads all the files in the directory and extracts a summary of the results. 10 | Expects 4 runs of each scenario 11 | Writes the summary to a file Summary. 12 | The highlights are the Class accuracies for Test and best_test 13 | 14 | Typical input at the end of the results files looks like: 15 | 16 | Class accuracies for Test: {'test': [68.0, 98.6, 19.3, 0.0, 92.7, 96.8, 94.39999999999999, 74.2, 95.5, 92.9]} 17 | Class accuracies for the top 5 and top 20 of Test: {'test': [[80.0, 100.0, 20.0, 0.0, 100.0, 100.0, 100.0, 60.0, 100.0, 100.0], [95.0, 100.0, 25.0, 0.0, 100.0, 100.0, 10\ 18 | 0.0, 65.0, 100.0, 100.0]]} 19 | kimg 32448 accuracy train/valid/test/best_test 100.00 75.58 74.47 76.42 20 | kimg 32512 accuracy train/valid/test/best_test 100.00 75.42 74.52 76.42 21 | kimg 32576 accuracy train/valid/test/best_test 100.00 75.42 74.58 76.42 22 | kimg 32640 accuracy train/valid/test/best_test 100.00 75.54 74.67 76.42 23 | Class accuracies for Test: {'test': [66.4, 97.6, 19.7, 0.0, 96.1, 97.5, 97.1, 76.7, 98.3, 96.89999999999999]} 24 | Class accuracies for the top 5 and top 20 of Test: {'test': [[80.0, 100.0, 40.0, 0.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0], [80.0, 100.0, 30.0, 0.0, 100.0, 100.0, 1\ 25 | 00.0, 75.0, 100.0, 100.0]]} 26 | Number of training pseudo-labels in each class: [ 377 49608 9 5] for classes: [0 1 8 9] 27 | kimg 32704 accuracy train/valid/test/best_test 100.00 75.56 74.63 76.42 28 | 29 | 30 | ''' 31 | 32 | #numFiles = int(sys.argv[2]) 33 | numFiles = 4 34 | 35 | bestAcc = [0]*numFiles 36 | 37 | print("=> Writing out files ....") 38 | filename = 'ResultsSummary' 39 | print(filename) 40 | fileOut = open(filename,'w') 41 | 42 | files = os.listdir('.') 43 | listing = glob.glob('./*0') 44 | 45 | listing.sort() 46 | for j in range(len(listing)): 47 | classAcc = ['', '', '', ''] 48 | bestAcc = ['', '', '', ''] 49 | numTrainPerClass = ['', '', '', ''] 50 | # print(listing[j]) 51 | for i in range(0,numFiles): 52 | name = listing[j] 53 | name = name[:-1] + str(i) 54 | # print(name," exits ",os.path.isfile(name)) 55 | if os.path.isfile(name): 56 | with open(name,"r") as f: 57 | for line in f: 58 | if (line.find("lass accuracies for Test") > 0): 59 | pref, post = line.split("[") 60 | classAcc[i] = post[:-3] 61 | #Number of training pseudo-labels in each class: [ 4488 6828 1324 4870 13553 5727 6403 3551 1552 1703] for classes: [0 1 2 3 4 5 6 7 8 9] 62 | # if (line.find("pseudo-labels") > 0): 63 | # pref, mid, post = line.split("[") 64 | # numbers, post = mid.split("]") 65 | # numTrainPerClass[i] = numbers 66 | 67 | if (line.find("accuracy train/valid/test/best_test") > 0): 68 | pref, post = line.split("best_test") 69 | accs = post.split(" ") 70 | bestAcc[i] = accs[4][:-1] 71 | 72 | print(name," ",bestAcc) 73 | for i in range(0,numFiles): 74 | print(classAcc[i]) 75 | # print(numTrainPerClass[i]) 76 | 77 | # fileOut.write('{:f}'.format(bestAcc) 78 | fileOut.write(name+" ") 79 | for i in range(0,numFiles): 80 | fileOut.write(bestAcc[i]+" ") 81 | fileOut.write("\n") 82 | for i in range(0,numFiles): 83 | # fileOut.write(classAcc[i]) 84 | accs = classAcc[i].split(",") 85 | try: 86 | for x in accs: 87 | fileOut.write('{0:0.2f}, '.format(float(x))) 88 | except: 89 | pass 90 | fileOut.write("\n") 91 | # for i in range(0,numFiles): 92 | # fileOut.write(numTrainPerClass[i]) 93 | # fileOut.write("\n") 94 | fileOut.close() 95 | exit(1) 96 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/create_prototypicals.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Script to create SSL splits from a dataset. 18 | """ 19 | 20 | import json 21 | import os 22 | from collections import defaultdict 23 | 24 | import numpy as np 25 | import tensorflow as tf 26 | from absl import app 27 | from absl import flags 28 | from tqdm import trange, tqdm 29 | 30 | from libml import data as libml_data 31 | from libml import utils 32 | 33 | flags.DEFINE_integer('seed', 0, 'Random seed to use, 0 for no shuffling.') 34 | flags.DEFINE_integer('size', 0, 'Size of labelled set.') 35 | 36 | FLAGS = flags.FLAGS 37 | 38 | 39 | def get_class(serialized_example): 40 | return tf.parse_single_example(serialized_example, features={'label': tf.FixedLenFeature([], tf.int64)})['label'] 41 | 42 | 43 | def main(argv): 44 | assert FLAGS.size 45 | argv.pop(0) 46 | if any(not tf.gfile.Exists(f) for f in argv[1:]): 47 | raise FileNotFoundError(argv[1:]) 48 | target = '%s.%d@%d' % (argv[0], FLAGS.seed, FLAGS.size) 49 | if tf.gfile.Exists(target): 50 | raise FileExistsError('For safety overwriting is not allowed', target) 51 | input_files = argv[1:] 52 | print("=> input_files= ", input_files) 53 | count = 0 54 | id_class = [] 55 | class_id = defaultdict(list) 56 | print('Computing class distribution') 57 | dataset = tf.data.TFRecordDataset(input_files).map(get_class, 4).batch(1 << 10) 58 | it = dataset.make_one_shot_iterator().get_next() 59 | #### Prototype code 60 | if (FLAGS.seed ==0): 61 | label= [35, 4, 18, 9, 20, 128, 19, 87, 69, 14] 62 | elif (FLAGS.seed == 1): 63 | label= [165, 61, 108, 91, 58, 156, 104, 163, 106, 1] 64 | elif (FLAGS.seed == 2): 65 | label= [199, 105, 120, 101, 149, 182, 124, 152, 111, 53] 66 | elif (FLAGS.seed == 3): 67 | label= [213, 160, 138, 169, 34, 177, 210, 172, 190, 109] 68 | elif (FLAGS.seed == 4): 69 | # 1 2 3 4 5 6 7 8 9 70 | label= [233, 176, 194, 33, 28, 198, 245, 289, 192, 225] 71 | elif (FLAGS.seed == 5): 72 | label= [199, 226, 138, 266, 343, 215, 326, 318, 216, 219] 73 | elif (FLAGS.seed == 6): # Based on prototype refining for seed=2 74 | #From seeds 3 2 2 2 2 2 2 2 2 1 75 | label= [213, 105, 120, 101, 149, 182, 124, 152, 111, 1] 76 | # elif (FLAGS.seed == 6): # Based on prototype refining for balance=0 77 | # label= [213, 105, 108, 91, 149, 182, 210, 163, 190, 1] 78 | # label= [20483, 8867, 47593, 23531, 14445, 35952, 28881, 8210, 30034, 40253] 79 | # elif (FLAGS.seed == 7): # Based on prototype refining for balance=2 80 | # label= [213, 226, 138, 169, 343, 177, 245, 318, 106, 225] 81 | # label= [46819, 19300, 13322, 29007, 44816, 1427, 14931, 344, 31772, 27357] 82 | elif (FLAGS.seed == 7): # Based on prototype refining for seed=0 83 | #From seeds 4 4 4 1 4 2 4 4 4 4 84 | label= [233, 176, 194, 91, 28, 124, 245, 289, 192, 225] 85 | #From seeds 3 0 3 0 2 2 0 0 0 0 86 | # label= [213, 4, 138, 9, 149, 182, 19, 87, 69, 14] 87 | #From seeds 3 0 0 0 5 0 0 0 0 0 88 | # label= [213, 4, 18, 9, 343, 128, 19, 87, 69, 14] 89 | elif (FLAGS.seed == 8): 90 | label= [42112, 46875, 27460, 15846, 35142, 45802, 39501, 8628, 48571, 49791] 91 | elif (FLAGS.seed == 9): 92 | #From seeds 0 0 3 0 0 2 0 0 0 0 93 | label= [35, 4, 138, 9, 20, 182, 19, 87, 69, 14] 94 | #From seeds 3 3 3 1 3 3 3 3 3 3 95 | # label= [213, 160, 138, 91, 34, 177, 210, 172, 190, 109] 96 | elif (FLAGS.seed == 10): 97 | #From seeds 3 0 3 0 2 2 0 0 0 0 98 | label= [213, 4, 138, 9, 149, 182, 19, 87, 69, 14] 99 | #From seeds 3 3 1 3 3 3 3 3 3 3 100 | # label= [213, 160, 108, 169, 34, 177, 210, 172, 190, 109] 101 | elif (FLAGS.seed == 11): 102 | #From seeds 3 3 3 3 2 3 3 3 3 3 103 | label= [213, 160, 138, 169, 149, 177, 210, 172, 190, 109] 104 | elif (FLAGS.seed == 12): 105 | #From seeds 3 3 3 3 3 2 3 3 3 3 106 | label= [213, 160, 138, 169, 34, 182, 210, 172, 190, 109] 107 | else: 108 | print(FLAGS.seed, " seed not defined") 109 | exit(1) 110 | print("label ",label) 111 | #### End Prototype code 112 | try: 113 | with tf.Session() as session, tqdm(leave=False) as t: 114 | while 1: 115 | old_count = count 116 | for i in session.run(it): 117 | id_class.append(i) 118 | class_id[i].append(count) 119 | if count in label: 120 | print("class, count ",i,count) 121 | count += 1 122 | t.update(count - old_count) 123 | except tf.errors.OutOfRangeError: 124 | pass 125 | print('%d records found' % count) 126 | print("=> id_class.shape,class_id.shape= ", len(id_class) ,len(class_id) ) 127 | nclass = len(class_id) 128 | for i in range(nclass): 129 | print("i,class_id[i][0:32]= ",i,class_id[i][0:32]) 130 | assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1) 131 | train_stats = np.array([len(class_id[i]) for i in range(nclass)], np.float64) 132 | train_stats /= train_stats.max() 133 | if 'stl10' in argv[1]: 134 | # All of the unlabeled data is given label 0, but we know that 135 | # STL has equally distributed data among the 10 classes. 136 | train_stats[:] = 1 137 | 138 | print(' Stats', ' '.join(['%.2f' % (100 * x) for x in train_stats])) 139 | assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1) 140 | class_id = [np.array(class_id[i], dtype=np.int64) for i in range(nclass)] 141 | del class_id 142 | print("===> label= ", label) 143 | label = frozenset([int(x) for x in label]) 144 | print("===> label= ", label) 145 | if 'stl10' in argv[1] and FLAGS.size == 1000: 146 | data = tf.gfile.Open(os.path.join(libml_data.DATA_DIR, 'stl10_fold_indices.txt'), 'r').read() 147 | label = frozenset(list(map(int, data.split('\n')[FLAGS.seed].split()))) 148 | 149 | print('Creating split in %s' % target) 150 | tf.gfile.MakeDirs(os.path.dirname(target)) 151 | with tf.python_io.TFRecordWriter(target + '-label.tfrecord') as writer_label: 152 | pos, loop = 0, trange(count, desc='Writing records') 153 | for input_file in input_files: 154 | for record in tf.python_io.tf_record_iterator(input_file): 155 | if pos in label: 156 | writer_label.write(record) 157 | pos += 1 158 | loop.update() 159 | loop.close() 160 | with tf.gfile.Open(target + '-label.json', 'w') as writer: 161 | writer.write(json.dumps(dict(distribution=train_stats.tolist(), label=sorted(label)), indent=2, sort_keys=True)) 162 | 163 | 164 | if __name__ == '__main__': 165 | utils.setup_tf() 166 | app.run(main) 167 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/create_prototypicals.py~: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Script to create SSL splits from a dataset. 18 | """ 19 | 20 | import json 21 | import os 22 | from collections import defaultdict 23 | 24 | import numpy as np 25 | import tensorflow as tf 26 | from absl import app 27 | from absl import flags 28 | from tqdm import trange, tqdm 29 | 30 | from libml import data as libml_data 31 | from libml import utils 32 | 33 | flags.DEFINE_integer('seed', 0, 'Random seed to use, 0 for no shuffling.') 34 | flags.DEFINE_integer('size', 0, 'Size of labelled set.') 35 | 36 | FLAGS = flags.FLAGS 37 | 38 | 39 | def get_class(serialized_example): 40 | return tf.parse_single_example(serialized_example, features={'label': tf.FixedLenFeature([], tf.int64)})['label'] 41 | 42 | 43 | def main(argv): 44 | assert FLAGS.size 45 | argv.pop(0) 46 | if any(not tf.gfile.Exists(f) for f in argv[1:]): 47 | raise FileNotFoundError(argv[1:]) 48 | target = '%s.%d@%d' % (argv[0], FLAGS.seed, FLAGS.size) 49 | if tf.gfile.Exists(target): 50 | raise FileExistsError('For safety overwriting is not allowed', target) 51 | input_files = argv[1:] 52 | print("=> input_files= ", input_files) 53 | count = 0 54 | id_class = [] 55 | class_id = defaultdict(list) 56 | print('Computing class distribution') 57 | dataset = tf.data.TFRecordDataset(input_files).map(get_class, 4).batch(1 << 10) 58 | it = dataset.make_one_shot_iterator().get_next() 59 | try: 60 | with tf.Session() as session, tqdm(leave=False) as t: 61 | while 1: 62 | old_count = count 63 | for i in session.run(it): 64 | id_class.append(i) 65 | class_id[i].append(count) 66 | count += 1 67 | t.update(count - old_count) 68 | except tf.errors.OutOfRangeError: 69 | pass 70 | print('%d records found' % count) 71 | print("=> id_class.shape,class_id.shape= ", len(id_class) ,len(class_id) ) 72 | nclass = len(class_id) 73 | for i in range(nclass): 74 | print("i,class_id[i][0:9]= ",i,class_id[i][0:9]) 75 | assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1) 76 | train_stats = np.array([len(class_id[i]) for i in range(nclass)], np.float64) 77 | train_stats /= train_stats.max() 78 | if 'stl10' in argv[1]: 79 | # All of the unlabeled data is given label 0, but we know that 80 | # STL has equally distributed data among the 10 classes. 81 | train_stats[:] = 1 82 | 83 | print(' Stats', ' '.join(['%.2f' % (100 * x) for x in train_stats])) 84 | assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1) 85 | class_id = [np.array(class_id[i], dtype=np.int64) for i in range(nclass)] 86 | if FLAGS.seed ==0: 87 | label= [36, 5, 19, 10, 21, 129, 20, 88, 70, 15] 88 | else if FLAGS.seed == 1: 89 | label= [166, 62, 109, 92, 59, 157, 105, 164, 107, 2] 90 | else if FLAGS.seed == 2: 91 | label= [200, 106, 121, 102, 150, 183, 125, 153, 112, 54] 92 | else if FLAGS.seed == 3: 93 | label= [214, 161, 139, 170, 35, 178, 211, 173, 191, 110] 94 | else if FLAGS.seed == 4: 95 | label= [234, 177, 195, 34, 29, 199, 246, 290, 193, 226] 96 | else if FLAGS.seed == 5: 97 | label= [200, 227, 139, 267, 344, 216, 327, 319, 217, 220] 98 | 99 | del class_id 100 | print("===> label= ", label) 101 | label = frozenset([int(x) for x in label]) 102 | print("===> label= ", label) 103 | if 'stl10' in argv[1] and FLAGS.size == 1000: 104 | data = tf.gfile.Open(os.path.join(libml_data.DATA_DIR, 'stl10_fold_indices.txt'), 'r').read() 105 | label = frozenset(list(map(int, data.split('\n')[FLAGS.seed].split()))) 106 | 107 | print('Creating split in %s' % target) 108 | tf.gfile.MakeDirs(os.path.dirname(target)) 109 | with tf.python_io.TFRecordWriter(target + '-typical-label.tfrecord') as writer_label: 110 | pos, loop = 0, trange(count, desc='Writing records') 111 | for input_file in input_files: 112 | for record in tf.python_io.tf_record_iterator(input_file): 113 | if pos in label: 114 | writer_label.write(record) 115 | pos += 1 116 | loop.update() 117 | loop.close() 118 | with tf.gfile.Open(target + '-typical-label.json', 'w') as writer: 119 | writer.write(json.dumps(dict(distribution=train_stats.tolist(), label=sorted(label)), indent=2, sort_keys=True)) 120 | 121 | 122 | if __name__ == '__main__': 123 | utils.setup_tf() 124 | app.run(main) 125 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/create_split.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Script to create SSL splits from a dataset. 18 | """ 19 | 20 | import json 21 | import os 22 | from collections import defaultdict 23 | 24 | import numpy as np 25 | import tensorflow as tf 26 | from absl import app 27 | from absl import flags 28 | from tqdm import trange, tqdm 29 | 30 | from libml import data as libml_data 31 | from libml import utils 32 | 33 | flags.DEFINE_integer('seed', 0, 'Random seed to use, 0 for no shuffling.') 34 | flags.DEFINE_integer('size', 0, 'Size of labelled set.') 35 | 36 | FLAGS = flags.FLAGS 37 | 38 | 39 | def get_class(serialized_example): 40 | return tf.parse_single_example(serialized_example, features={'label': tf.FixedLenFeature([], tf.int64)})['label'] 41 | 42 | 43 | def main(argv): 44 | assert FLAGS.size 45 | argv.pop(0) 46 | if any(not tf.gfile.Exists(f) for f in argv[1:]): 47 | raise FileNotFoundError(argv[1:]) 48 | target = '%s.%d@%d' % (argv[0], FLAGS.seed, FLAGS.size) 49 | if tf.gfile.Exists(target): 50 | raise FileExistsError('For safety overwriting is not allowed', target) 51 | input_files = argv[1:] 52 | count = 0 53 | id_class = [] 54 | class_id = defaultdict(list) 55 | print('Computing class distribution') 56 | dataset = tf.data.TFRecordDataset(input_files).map(get_class, 4).batch(1 << 10) 57 | it = dataset.make_one_shot_iterator().get_next() 58 | try: 59 | with tf.Session() as session, tqdm(leave=False) as t: 60 | while 1: 61 | old_count = count 62 | for i in session.run(it): 63 | id_class.append(i) 64 | class_id[i].append(count) 65 | count += 1 66 | t.update(count - old_count) 67 | except tf.errors.OutOfRangeError: 68 | pass 69 | print('%d records found' % count) 70 | nclass = len(class_id) 71 | assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1) 72 | train_stats = np.array([len(class_id[i]) for i in range(nclass)], np.float64) 73 | train_stats /= train_stats.max() 74 | if 'stl10' in argv[1]: 75 | # All of the unlabeled data is given label 0, but we know that 76 | # STL has equally distributed data among the 10 classes. 77 | train_stats[:] = 1 78 | 79 | print(' Stats', ' '.join(['%.2f' % (100 * x) for x in train_stats])) 80 | assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1) 81 | class_id = [np.array(class_id[i], dtype=np.int64) for i in range(nclass)] 82 | if FLAGS.seed: 83 | np.random.seed(FLAGS.seed) 84 | for i in range(nclass): 85 | np.random.shuffle(class_id[i]) 86 | 87 | # Distribute labels to match the input distribution. 88 | npos = np.zeros(nclass, np.int64) 89 | label = [] 90 | for i in range(FLAGS.size): 91 | c = np.argmax(train_stats - npos / max(npos.max(), 1)) 92 | label.append(class_id[c][npos[c]]) 93 | npos[c] += 1 94 | 95 | del npos, class_id 96 | print(' Classes', ' '.join(['%d' % id_class[x] for x in label])) 97 | label = frozenset([int(x) for x in label]) 98 | print(label) 99 | exit(1) 100 | if 'stl10' in argv[1] and FLAGS.size == 1000: 101 | data = tf.gfile.Open(os.path.join(libml_data.DATA_DIR, 'stl10_fold_indices.txt'), 'r').read() 102 | label = frozenset(list(map(int, data.split('\n')[FLAGS.seed].split()))) 103 | 104 | print('Creating split in %s' % target) 105 | tf.gfile.MakeDirs(os.path.dirname(target)) 106 | with tf.python_io.TFRecordWriter(target + '-label.tfrecord') as writer_label: 107 | pos, loop = 0, trange(count, desc='Writing records') 108 | for input_file in input_files: 109 | for record in tf.python_io.tf_record_iterator(input_file): 110 | if pos in label: 111 | writer_label.write(record) 112 | pos += 1 113 | loop.update() 114 | loop.close() 115 | with tf.gfile.Open(target + '-label.json', 'w') as writer: 116 | writer.write(json.dumps(dict(distribution=train_stats.tolist(), label=sorted(label)), indent=2, sort_keys=True)) 117 | 118 | 119 | if __name__ == '__main__': 120 | utils.setup_tf() 121 | app.run(main) 122 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/create_unlabeled.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Script to create SSL splits from a dataset. 18 | """ 19 | 20 | import json 21 | import os 22 | from collections import defaultdict 23 | 24 | import numpy as np 25 | import tensorflow as tf 26 | from absl import app 27 | from tqdm import trange, tqdm 28 | 29 | from libml import utils 30 | 31 | 32 | def get_class(serialized_example): 33 | return tf.parse_single_example(serialized_example, features={'label': tf.FixedLenFeature([], tf.int64)})['label'] 34 | 35 | 36 | def main(argv): 37 | argv.pop(0) 38 | if any(not tf.gfile.Exists(f) for f in argv[1:]): 39 | raise FileNotFoundError(argv[1:]) 40 | target = argv[0] 41 | input_files = argv[1:] 42 | print("=> input_files= ", input_files) 43 | count = 0 44 | id_class = [] 45 | class_id = defaultdict(list) 46 | print('Computing class distribution') 47 | dataset = tf.data.TFRecordDataset(input_files).map(get_class, 4).batch(1 << 10) 48 | it = dataset.make_one_shot_iterator().get_next() 49 | try: 50 | with tf.Session() as session, tqdm(leave=False) as t: 51 | while 1: 52 | old_count = count 53 | for i in session.run(it): 54 | id_class.append(i) 55 | class_id[i].append(count) 56 | count += 1 57 | t.update(count - old_count) 58 | except tf.errors.OutOfRangeError: 59 | pass 60 | print('%d records found' % count) 61 | print("=> id_class.shape,class_id.shape= ", len(id_class) ,len(class_id) ) 62 | nclass = len(class_id) 63 | assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1) 64 | train_stats = np.array([len(class_id[i]) for i in range(nclass)], np.float64) 65 | train_stats /= train_stats.max() 66 | if 'stl10' in argv[1]: 67 | # All of the unlabeled data is given label 0, but we know that 68 | # STL has equally distributed data among the 10 classes. 69 | train_stats[:] = 1 70 | 71 | print(' Stats', ' '.join(['%.2f' % (100 * x) for x in train_stats])) 72 | del class_id 73 | 74 | print('Creating unlabeled dataset for in %s' % target) 75 | npos = np.zeros(nclass, np.int64) 76 | class_data = [[] for _ in range(nclass)] 77 | unlabel = [] 78 | tf.gfile.MakeDirs(os.path.dirname(target)) 79 | with tf.python_io.TFRecordWriter(target + '-unlabel.tfrecord') as writer_unlabel: 80 | pos, loop = 0, trange(count, desc='Writing records') 81 | for input_file in input_files: 82 | for record in tf.python_io.tf_record_iterator(input_file): 83 | class_data[id_class[pos]].append((pos, record)) 84 | while True: 85 | c = np.argmax(train_stats - npos / max(npos.max(), 1)) 86 | if class_data[c]: 87 | p, v = class_data[c].pop(0) 88 | unlabel.append(p) 89 | writer_unlabel.write(v) 90 | npos[c] += 1 91 | else: 92 | break 93 | pos += 1 94 | loop.update() 95 | for remain in class_data: 96 | for p, v in remain: 97 | unlabel.append(p) 98 | writer_unlabel.write(v) 99 | loop.close() 100 | with tf.gfile.Open(target + '-unlabel.json', 'w') as writer: 101 | writer.write(json.dumps(dict(distribution=train_stats.tolist(), indexes=unlabel), indent=2, sort_keys=True)) 102 | 103 | 104 | if __name__ == '__main__': 105 | utils.setup_tf() 106 | app.run(main) 107 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/extract_accuracy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Extract and save accuracy to 'stats/accuracy.json'. 18 | 19 | The accuracy is extracted from the most recent eventfile. 20 | """ 21 | 22 | import json 23 | import os.path 24 | 25 | import numpy as np 26 | import tensorflow as tf 27 | from absl import app 28 | from absl import flags 29 | 30 | FLAGS = flags.FLAGS 31 | TAG = 'accuracy' 32 | 33 | 34 | def summary_dict(accuracies): 35 | return { 36 | 'last%02d' % x: np.median(accuracies[-x:]) for x in [1, 10, 20, 50] 37 | } 38 | 39 | 40 | def main(argv): 41 | if len(argv) > 2: 42 | raise app.UsageError('Too many command-line arguments.') 43 | folder = argv[1] 44 | matches = sorted(tf.gfile.Glob(os.path.join(folder, 'tf/events.out.tfevents.*'))) 45 | assert matches, 'No events files found' 46 | tags = set() 47 | accuracies = [] 48 | for event_file in matches: 49 | try: 50 | for e in tf.train.summary_iterator(event_file): 51 | for v in e.summary.value: 52 | if v.tag == TAG: 53 | accuracies.append(v.simple_value) 54 | break 55 | elif not accuracies: 56 | tags.add(v.tag) 57 | except tf.errors.DataLossError: 58 | continue 59 | 60 | assert accuracies, 'No "accuracy" tag found. Found tags = %s' % tags 61 | target_dir = os.path.join(folder, 'stats') 62 | target_file = os.path.join(target_dir, 'accuracy.json') 63 | tf.gfile.MakeDirs(target_dir) 64 | 65 | with tf.gfile.Open(target_file, 'w') as f: 66 | json.dump(summary_dict(accuracies), f, sort_keys=True, indent=4) 67 | print('Saved: %s' % target_file) 68 | 69 | 70 | if __name__ == '__main__': 71 | app.run(main) 72 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/inspect_dataset.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Script to inspect a dataset, in particular label distribution. 18 | """ 19 | 20 | import numpy as np 21 | import tensorflow as tf 22 | from absl import app 23 | from absl import flags 24 | from tqdm import trange 25 | 26 | from libml import data, utils 27 | 28 | flags.DEFINE_integer('batch', 64, 'Batch size.') 29 | flags.DEFINE_integer('samples', 1 << 16, 'Number of samples to load.') 30 | 31 | FLAGS = flags.FLAGS 32 | 33 | 34 | def main(argv): 35 | utils.setup_main() 36 | del argv 37 | utils.setup_tf() 38 | nbatch = FLAGS.samples // FLAGS.batch 39 | dataset = data.DATASETS()[FLAGS.dataset]() 40 | groups = [('labeled', dataset.train_labeled), 41 | ('unlabeled', dataset.train_unlabeled), 42 | ('test', dataset.test.repeat())] 43 | groups = [(name, ds.batch(FLAGS.batch).prefetch(16).make_one_shot_iterator().get_next()) 44 | for name, ds in groups] 45 | with tf.train.MonitoredSession() as sess: 46 | for group, train_data in groups: 47 | stats = np.zeros(dataset.nclass, np.int32) 48 | minmax = [], [] 49 | for _ in trange(nbatch, leave=False, unit='img', unit_scale=FLAGS.batch, desc=group): 50 | v = sess.run(train_data)['label'] 51 | for u in v: 52 | stats[u] += 1 53 | minmax[0].append(v.min()) 54 | minmax[1].append(v.max()) 55 | print(group) 56 | print(' Label range', min(minmax[0]), max(minmax[1])) 57 | print(' Stats', ' '.join(['%.2f' % (100 * x) for x in (stats / stats.max())])) 58 | 59 | 60 | if __name__ == '__main__': 61 | app.run(main) 62 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/iteration1.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Script to create SSL splits from a dataset. 18 | """ 19 | 20 | import json 21 | import os 22 | from collections import defaultdict 23 | 24 | import numpy as np 25 | import tensorflow as tf 26 | from absl import app 27 | from absl import flags 28 | from tqdm import trange, tqdm 29 | 30 | from libml import data as libml_data 31 | from libml import utils 32 | 33 | flags.DEFINE_integer('seed', 0, 'Random seed to use, 0 for no shuffling.') 34 | flags.DEFINE_integer('size', 0, 'Size of labelled set.') 35 | flags.DEFINE_integer('balance', 0, 'Size of labelled set.') 36 | flags.DEFINE_string('pseudo_file', None, 37 | 'Directory of top pseudolabeled examples. ') 38 | 39 | 40 | FLAGS = flags.FLAGS 41 | 42 | 43 | def get_class(serialized_example): 44 | return tf.parse_single_example(serialized_example, features={'label': tf.FixedLenFeature([], tf.int64)})['label'] 45 | 46 | 47 | def main(argv): 48 | assert FLAGS.size 49 | argv.pop(0) 50 | if any(not tf.gfile.Exists(f) for f in argv[1:]): 51 | raise FileNotFoundError(argv[1:]) 52 | target = '%sB%d.%d@%d' % (argv[0], FLAGS.balance, FLAGS.seed, FLAGS.size+10) 53 | if tf.gfile.Exists(target): 54 | raise FileExistsError('For safety overwriting is not allowed', target) 55 | input_files = argv[1:] 56 | print("=> input_files= ", input_files) 57 | print("=> target_file= ", target) 58 | print("=> pseudo_file= ", FLAGS.pseudo_file) 59 | count = 0 60 | id_class = [] 61 | class_id = defaultdict(list) 62 | print('Computing class distribution') 63 | dataset = tf.data.TFRecordDataset(input_files).map(get_class, 4).batch(1 << 10) 64 | it = dataset.make_one_shot_iterator().get_next() 65 | #### Prototype code 66 | if (FLAGS.seed ==0): 67 | label= [35, 4, 18, 9, 20, 128, 19, 87, 69, 14] 68 | elif (FLAGS.seed == 1): 69 | label= [165, 61, 108, 91, 58, 156, 104, 163, 106, 1] 70 | elif (FLAGS.seed == 2): 71 | label= [199, 105, 120, 101, 149, 182, 124, 152, 111, 53] 72 | elif (FLAGS.seed == 3): 73 | label= [213, 160, 138, 169, 34, 177, 210, 172, 190, 109] 74 | elif (FLAGS.seed == 4): 75 | # 1 2 3 4 5 6 7 8 9 76 | label= [233, 176, 194, 33, 28, 198, 245, 289, 192, 225] 77 | elif (FLAGS.seed == 5): 78 | label= [199, 226, 138, 266, 343, 215, 326, 318, 216, 219] 79 | elif (FLAGS.seed == 6): 80 | label= [20483, 8867, 47593, 23531, 14445, 35952, 28881, 8210, 30034, 40253] 81 | elif (FLAGS.seed == 7): 82 | label= [46819, 19300, 13322, 29007, 44816, 1427, 14931, 344, 31772, 27357] 83 | elif (FLAGS.seed == 8): 84 | label= [42112, 46875, 27460, 15846, 35142, 45802, 39501, 8628, 48571, 49791] 85 | elif (FLAGS.seed == 9): 86 | label= [234, 177, 27460, 15846, 14445, 216, 105, 173, 191, 110] 87 | elif (FLAGS.seed == 10): 88 | # 1 2 3 4 5 6 7 8 9 89 | label= [234, 177, 121, 29007 , 29, 216, 327, 173, 70, 110] 90 | else: 91 | print(FLAGS.seed, " seed not defined") 92 | exit(1) 93 | print("label ",label) 94 | #### End Prototype code 95 | try: 96 | with tf.Session() as session, tqdm(leave=False) as t: 97 | while 1: 98 | old_count = count 99 | for i in session.run(it): 100 | id_class.append(i) # i is the class label 101 | class_id[i].append(count) # count is sample number 102 | if count in label: 103 | print("class, count ",i,count) 104 | count += 1 105 | t.update(count - old_count) 106 | except tf.errors.OutOfRangeError: 107 | pass 108 | print('%d records found' % count) 109 | #### Code for self train iterations 110 | print("=> id_class.shape,class_id.shape= ", len(id_class) ,len(class_id) ) 111 | nclass = len(class_id) 112 | if FLAGS.pseudo_file: 113 | ID = np.load(FLAGS.pseudo_file) 114 | lab_name = FLAGS.pseudo_file.replace("Probs","Labels") 115 | id_labels = np.load(lab_name) 116 | lab_name = FLAGS.pseudo_file.replace("Probs","TrueLabels") 117 | true_labels = np.load(lab_name) 118 | # id_labels = true_labels 119 | # for j in range( len(ID)): 120 | # print("Sample number ",ID[j]," pseudo-label, true-label, id_class ", 121 | # id_labels[j],true_labels[j],id_class[ID[j]]) 122 | # exit(1) 123 | 124 | # print("size ID ", len(ID)) 125 | # print("size id_labels ", len(id_labels)) 126 | # print("ID ", ID[0:20]) 127 | print("id_labels ", id_labels[0:20]) 128 | print("true_labels ", true_labels[0:20]) 129 | sz = round(FLAGS.size / nclass) 130 | sortByClass = -np.ones([nclass,2000],dtype=int) 131 | sortIndx = [0]*nclass 132 | for j in range( len(ID)): 133 | sortByClass[id_labels[j]][sortIndx[id_labels[j]]] = ID[j] 134 | sortIndx[id_labels[j]] += 1 135 | print("sortIndx ", sortIndx) 136 | # print(" sortByClass ", sortByClass[0:10]) 137 | er = 0 138 | for j in range(sz): 139 | for i in range(nclass): 140 | assert sortIndx[i] >= sz 141 | if id_class[sortByClass[i][j]] != i: 142 | # print("Error for ",sortByClass[i][j] ," : Pseudo label ",i, 143 | # " actual label ",id_class[sortByClass[i][j]]," +-1 ",id_class[sortByClass[i][j]-1],id_class[sortByClass[i][j]+1]) 144 | er += 1 145 | label.append(sortByClass[i][j]) 146 | print(" size, number of errors, label ", len(label),er, label) 147 | del ID 148 | # exit(1) 149 | #### End Prototype code 150 | assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1) 151 | train_stats = np.array([len(class_id[i]) for i in range(nclass)], np.float64) 152 | train_stats /= train_stats.max() 153 | if 'stl10' in argv[1]: 154 | # All of the unlabeled data is given label 0, but we know that 155 | # STL has equally distributed data among the 10 classes. 156 | train_stats[:] = 1 157 | 158 | print(' Stats', ' '.join(['%.2f' % (100 * x) for x in train_stats])) 159 | assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1) 160 | class_id = [np.array(class_id[i], dtype=np.int64) for i in range(nclass)] 161 | del class_id 162 | # print("===> label= ", label) 163 | label = frozenset([int(x) for x in label]) 164 | print("===> label= ", label) 165 | if 'stl10' in argv[1] and FLAGS.size == 1000: 166 | data = tf.gfile.Open(os.path.join(libml_data.DATA_DIR, 'stl10_fold_indices.txt'), 'r').read() 167 | label = frozenset(list(map(int, data.split('\n')[FLAGS.seed].split()))) 168 | 169 | print('Creating split in %s' % target) 170 | tf.gfile.MakeDirs(os.path.dirname(target)) 171 | with tf.python_io.TFRecordWriter(target + '-label.tfrecord') as writer_label: 172 | pos, loop = 0, trange(count, desc='Writing records') 173 | for input_file in input_files: 174 | for record in tf.python_io.tf_record_iterator(input_file): 175 | if pos in label: 176 | writer_label.write(record) 177 | pos += 1 178 | loop.update() 179 | loop.close() 180 | with tf.gfile.Open(target + '-label.json', 'w') as writer: 181 | writer.write(json.dumps(dict(distribution=train_stats.tolist(), label=sorted(label)), indent=2, sort_keys=True)) 182 | 183 | 184 | if __name__ == '__main__': 185 | utils.setup_tf() 186 | app.run(main) 187 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/submit4.sh: -------------------------------------------------------------------------------- 1 | dataset=cifar10 2 | size=10 3 | seed=1 4 | valid=5000 5 | 6 | kimg=32768 # 65536 7 | ratio=7 8 | con=0.95 9 | wd=5e-4 10 | batch=64 11 | lr=0.04 12 | i=3 13 | aug="d.d.d" 14 | #seed=(0 1 2 3 4 5) 15 | seed=1 16 | 17 | echo " " >> history 18 | for seed in 1 5; do 19 | #i2=`expr $i + 1` 20 | #echo $i2 21 | #for con in 0.99 0.98; do 22 | #for ratio in 7 8 16; do 23 | #for batch in 64 128 256; do 24 | #for lr in 0.1 0.06 0.01; do 25 | #for wd in 7e-4 5e-4 3e-4 1e-4; do 26 | #for kimg in 8192 32768; do 27 | #for aug in "rac.m.rac" "d.m.rac" "aac.m.aac" "d.m.aac"; do 28 | #for i in 1 2 3; do 29 | filename="${dataset}.${seed}@${size}-${valid}Iter${kimg}U${ratio}C${con}WD${wd}BS${batch}LR${lr}Aug${aug}_0" 30 | echo $filename 31 | echo $filename >> history 32 | sed -e "s/xSeed0/${seed}/g" -e "s/xSize0/${size}/g" -e "s/xValid0/${valid}/g" -e "s/xTime/120/g" -e "s/xKimg0/$kimg/g" -e "s/xUratio0/$ratio/g" -e "s/xConf0/$con/g" -e "s/xWd0/$wd/g" -e "s/xBatch0/$batch/g" -e "s/xLr0/$lr/g" -e "s/xRep0/0/g" -e "s/xRep1/1/g" -e "s/xAug0/$aug/g" z2.pbs > Q/$filename 33 | # exit 34 | qsub Q/$filename 35 | sleep 1 36 | sed -e "s/xSeed0/${seed}/g" -e "s/xSize0/${size}/g" -e "s/xValid0/${valid}/g" -e "s/xTime/120/g" -e "s/xKimg0/$kimg/g" -e "s/xUratio0/$ratio/g" -e "s/xConf0/$con/g" -e "s/xWd0/$wd/g" -e "s/xBatch0/$batch/g" -e "s/xLr0/$lr/g" -e "s/xRep0/2/g" -e "s/xRep1/3/g" -e "s/xAug0/$aug/g" z2.pbs > Q/$filename 37 | # exit 38 | qsub Q/$filename 39 | sleep 1 40 | done 41 | #done 42 | 43 | exit 44 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/svhn_iteration.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Self-train iteration for SVHN 3 | 4 | # Copyright 2019 Google LLC 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | """Script to create SSL splits from a dataset. 19 | """ 20 | 21 | import json 22 | import os 23 | from collections import defaultdict 24 | 25 | import numpy as np 26 | import tensorflow as tf 27 | from absl import app 28 | from absl import flags 29 | from tqdm import trange, tqdm 30 | 31 | from libml import data as libml_data 32 | from libml import utils 33 | 34 | flags.DEFINE_integer('seed', 0, 'Random seed to use, 0 for no shuffling.') 35 | flags.DEFINE_integer('size', 0, 'Size of labelled set.') 36 | #flags.DEFINE_integer('balance', 0, 'Size of labelled set.') 37 | flags.DEFINE_string('pseudo_file', None, 38 | 'Directory of top pseudolabeled examples. ') 39 | 40 | 41 | FLAGS = flags.FLAGS 42 | 43 | 44 | def get_class(serialized_example): 45 | return tf.parse_single_example(serialized_example, features={'label': tf.FixedLenFeature([], tf.int64)})['label'] 46 | 47 | 48 | def main(argv): 49 | assert FLAGS.size 50 | argv.pop(0) 51 | if any(not tf.gfile.Exists(f) for f in argv[1:]): 52 | raise FileNotFoundError(argv[1:]) 53 | target = '%s.%d@%d' % (argv[0], FLAGS.seed, FLAGS.size+10) 54 | if tf.gfile.Exists(target): 55 | raise FileExistsError('For safety overwriting is not allowed', target) 56 | input_files = argv[1:] 57 | print("=> input_files= ", input_files) 58 | print("=> target_file= ", target) 59 | print("=> pseudo_file= ", FLAGS.pseudo_file) 60 | count = 0 61 | id_class = [] 62 | class_id = defaultdict(list) 63 | print('Computing class distribution') 64 | dataset = tf.data.TFRecordDataset(input_files).map(get_class, 4).batch(1 << 10) 65 | it = dataset.make_one_shot_iterator().get_next() 66 | #### Prototype code 67 | if (FLAGS.seed ==1): 68 | label= [215, 230, 98, 97, 288, 157, 21, 80, 266, 306] 69 | elif (FLAGS.seed == 2): 70 | # 0 1 2 3 4 5 6 7 8 9 71 | label= [317, 209, 216, 235, 287, 180, 351, 99, 210, 110] 72 | elif (FLAGS.seed == 3): 73 | label= [108, 228, 27, 112, 26, 261, 333, 34, 342, 832] 74 | elif (FLAGS.seed == 4): 75 | label= [318, 127, 87, 159, 179, 200, 194, 224, 192, 167] 76 | else: 77 | print(FLAGS.seed, " seed not defined") 78 | exit(1) 79 | print("label ",label) 80 | #### End Prototype code 81 | try: 82 | with tf.Session() as session, tqdm(leave=False) as t: 83 | while 1: 84 | old_count = count 85 | for i in session.run(it): 86 | id_class.append(i) # i is the class label 87 | class_id[i].append(count) # count is sample number 88 | if count in label: 89 | print("class, count ",i,count) 90 | count += 1 91 | t.update(count - old_count) 92 | except tf.errors.OutOfRangeError: 93 | pass 94 | print('%d records found' % count) 95 | #### Code for self train iterations 96 | print("=> id_class.shape,class_id.shape= ", len(id_class) ,len(class_id) ) 97 | nclass = len(class_id) 98 | if FLAGS.pseudo_file: 99 | ID = np.load(FLAGS.pseudo_file) 100 | lab_name = FLAGS.pseudo_file.replace("Probs","Labels") 101 | id_labels = np.load(lab_name) 102 | lab_name = FLAGS.pseudo_file.replace("Probs","TrueLabels") 103 | true_labels = np.load(lab_name) 104 | id_labels = true_labels 105 | unique_train_pseudo_labels, unique_train_counts = np.unique(id_labels[0:2000], return_counts=True) 106 | print("Number of training pseudo-labels in each class: ", unique_train_counts," for classes: ", unique_train_pseudo_labels) 107 | 108 | # for j in range( len(ID)): 109 | # print("Sample number ",ID[j]," pseudo-label, true-label, id_class ", 110 | # id_labels[j],true_labels[j],id_class[ID[j]]) 111 | 112 | # print("size ID ", len(ID)) 113 | # print("size id_labels ", len(id_labels)) 114 | # print("ID ", ID[0:20]) 115 | print("id_labels ", id_labels[0:20]) 116 | print("true_labels ", true_labels[0:20]) 117 | sz = round(FLAGS.size / nclass) 118 | sortByClass = -np.ones([nclass,2000],dtype=int) 119 | sortIndx = [0]*nclass 120 | for j in range( len(ID)): 121 | sortByClass[id_labels[j]][sortIndx[id_labels[j]]] = ID[j] 122 | sortIndx[id_labels[j]] += 1 123 | print("sortIndx ", sortIndx) 124 | exit(1) 125 | # print(" sortByClass ", sortByClass[0:10]) 126 | er = 0 127 | for j in range(sz): 128 | for i in range(nclass): 129 | assert sortIndx[i] >= sz 130 | if id_class[sortByClass[i][j]] != i: 131 | # print("Error for ",sortByClass[i][j] ," : Pseudo label ",i, 132 | # " actual label ",id_class[sortByClass[i][j]]," +-1 ",id_class[sortByClass[i][j]-1],id_class[sortByClass[i][j]+1]) 133 | er += 1 134 | label.append(sortByClass[i][j]) 135 | print(" size, number of errors, label ", len(label),er, label) 136 | del ID 137 | # exit(1) 138 | #### End Prototype code 139 | assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1) 140 | train_stats = np.array([len(class_id[i]) for i in range(nclass)], np.float64) 141 | train_stats /= train_stats.max() 142 | if 'stl10' in argv[1]: 143 | # All of the unlabeled data is given label 0, but we know that 144 | # STL has equally distributed data among the 10 classes. 145 | train_stats[:] = 1 146 | 147 | print(' Stats', ' '.join(['%.2f' % (100 * x) for x in train_stats])) 148 | assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1) 149 | class_id = [np.array(class_id[i], dtype=np.int64) for i in range(nclass)] 150 | del class_id 151 | # print("===> label= ", label) 152 | label = frozenset([int(x) for x in label]) 153 | print("===> label= ", label) 154 | if 'stl10' in argv[1] and FLAGS.size == 1000: 155 | data = tf.gfile.Open(os.path.join(libml_data.DATA_DIR, 'stl10_fold_indices.txt'), 'r').read() 156 | label = frozenset(list(map(int, data.split('\n')[FLAGS.seed].split()))) 157 | 158 | print('Creating split in %s' % target) 159 | tf.gfile.MakeDirs(os.path.dirname(target)) 160 | with tf.python_io.TFRecordWriter(target + '-label.tfrecord') as writer_label: 161 | pos, loop = 0, trange(count, desc='Writing records') 162 | for input_file in input_files: 163 | for record in tf.python_io.tf_record_iterator(input_file): 164 | if pos in label: 165 | writer_label.write(record) 166 | pos += 1 167 | loop.update() 168 | loop.close() 169 | with tf.gfile.Open(target + '-label.json', 'w') as writer: 170 | writer.write(json.dumps(dict(distribution=train_stats.tolist(), label=sorted(label)), indent=2, sort_keys=True)) 171 | 172 | 173 | if __name__ == '__main__': 174 | utils.setup_tf() 175 | app.run(main) 176 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/svhn_prototypes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Script to create SSL splits from a dataset. 18 | """ 19 | 20 | import json 21 | import os 22 | from collections import defaultdict 23 | 24 | import numpy as np 25 | import tensorflow as tf 26 | from absl import app 27 | from absl import flags 28 | from tqdm import trange, tqdm 29 | 30 | from libml import data as libml_data 31 | from libml import utils 32 | 33 | flags.DEFINE_integer('seed', 0, 'Random seed to use, 0 for no shuffling.') 34 | flags.DEFINE_integer('size', 0, 'Size of labelled set.') 35 | 36 | FLAGS = flags.FLAGS 37 | 38 | 39 | def get_class(serialized_example): 40 | return tf.parse_single_example(serialized_example, features={'label': tf.FixedLenFeature([], tf.int64)})['label'] 41 | 42 | 43 | def main(argv): 44 | assert FLAGS.size 45 | argv.pop(0) 46 | if any(not tf.gfile.Exists(f) for f in argv[1:]): 47 | raise FileNotFoundError(argv[1:]) 48 | target = '%s.%d@%d' % (argv[0], FLAGS.seed, FLAGS.size) 49 | if tf.gfile.Exists(target): 50 | raise FileExistsError('For safety overwriting is not allowed', target) 51 | input_files = argv[1:] 52 | print("=> input_files= ", input_files) 53 | count = 0 54 | id_class = [] 55 | class_id = defaultdict(list) 56 | print('Computing class distribution') 57 | dataset = tf.data.TFRecordDataset(input_files).map(get_class, 4).batch(1 << 10) 58 | it = dataset.make_one_shot_iterator().get_next() 59 | #### Prototype code 60 | if (FLAGS.seed ==1): 61 | label= [215, 230, 98, 97, 288, 157, 21, 80, 266, 306] 62 | elif (FLAGS.seed == 2): 63 | # 0 1 2 3 4 5 6 7 8 9 64 | label= [317, 209, 216, 235, 287, 180, 351, 99, 210, 110] 65 | elif (FLAGS.seed == 3): 66 | label= [108, 228, 27, 112, 26, 261, 333, 34, 342, 832] 67 | elif (FLAGS.seed == 4): 68 | label= [318, 127, 87, 159, 179, 200, 194, 224, 192, 167] 69 | else: 70 | print(FLAGS.seed, " seed not defined") 71 | exit(1) 72 | print("label ",label) 73 | #### End Prototype code 74 | try: 75 | with tf.Session() as session, tqdm(leave=False) as t: 76 | while 1: 77 | old_count = count 78 | for i in session.run(it): 79 | id_class.append(i) 80 | class_id[i].append(count) 81 | if count in label: 82 | print("class, count ",i,count) 83 | count += 1 84 | t.update(count - old_count) 85 | except tf.errors.OutOfRangeError: 86 | pass 87 | print('%d records found' % count) 88 | print("=> id_class.shape,class_id.shape= ", len(id_class) ,len(class_id) ) 89 | nclass = len(class_id) 90 | for i in range(nclass): 91 | print("i,class_id[i][0:32]= ",i,class_id[i][0:32]) 92 | assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1) 93 | train_stats = np.array([len(class_id[i]) for i in range(nclass)], np.float64) 94 | train_stats /= train_stats.max() 95 | if 'stl10' in argv[1]: 96 | # All of the unlabeled data is given label 0, but we know that 97 | # STL has equally distributed data among the 10 classes. 98 | train_stats[:] = 1 99 | 100 | print(' Stats', ' '.join(['%.2f' % (100 * x) for x in train_stats])) 101 | assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1) 102 | class_id = [np.array(class_id[i], dtype=np.int64) for i in range(nclass)] 103 | del class_id 104 | print("===> label= ", label) 105 | label = frozenset([int(x) for x in label]) 106 | print("===> label= ", label) 107 | if 'stl10' in argv[1] and FLAGS.size == 1000: 108 | data = tf.gfile.Open(os.path.join(libml_data.DATA_DIR, 'stl10_fold_indices.txt'), 'r').read() 109 | label = frozenset(list(map(int, data.split('\n')[FLAGS.seed].split()))) 110 | 111 | print('Creating split in %s' % target) 112 | tf.gfile.MakeDirs(os.path.dirname(target)) 113 | with tf.python_io.TFRecordWriter(target + '-label.tfrecord') as writer_label: 114 | pos, loop = 0, trange(count, desc='Writing records') 115 | for input_file in input_files: 116 | for record in tf.python_io.tf_record_iterator(input_file): 117 | if pos in label: 118 | writer_label.write(record) 119 | pos += 1 120 | loop.update() 121 | loop.close() 122 | with tf.gfile.Open(target + '-label.json', 'w') as writer: 123 | writer.write(json.dumps(dict(distribution=train_stats.tolist(), label=sorted(label)), indent=2, sort_keys=True)) 124 | 125 | 126 | if __name__ == '__main__': 127 | utils.setup_tf() 128 | app.run(main) 129 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/z.pbs: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #PBS -l walltime=xTime:00:00 3 | #PBS -l select=1:ncpus=12:mpiprocs=12:ngpus=2 4 | #PBS -l place=scatter:excl 5 | #PBS -N TFBOSS2SxSeed0_xSeed1 6 | #PBS -j oe 7 | #PBS -V 8 | #PBS -q standard 9 | #PBS -A ERDCS97260VU3 10 | 11 | module load python36 12 | module load openmpi/cuda/64/3.1.4 13 | module load tensorflow-py36-cuda10.1-gcc/1.14.0 14 | 15 | cd $PBS_O_WORKDIR 16 | cd .. 17 | pwd 18 | cp -r BOSS /tmp/ 19 | cd /tmp/BOSS 20 | export ML_DATA="/tmp/BOSS/data" 21 | export PYTHONPATH=$PYTHONPATH:$PWD 22 | 23 | date 24 | 25 | echo "FixMatch cifar10.xSeed0@xSize0-xValid0IterxKimg0UxUratio0CxConf0WDxWd0BSxBatch0LRxLr0AxAug0_xRep0" 26 | CUDA_VISIBLE_DEVICES=0 python fixmatch.py --train_kimg xKimg0 --uratio xUratio0 --confidence xConf0 --wd xWd0 --wu 1 --batch xBatch0 --lr xLr0 --filters=32 --scales 3 --repeat 4 --dataset=cifar10.xSeed0@xSize0-xValid0 --train_dir experiments/fixmatch/SxSeed0@xSize0-xValid0IterxKimg0UxUratio0CxConf0WDxWd0BSxBatch0LRxLr0AxAug0_xRep0 --augment xAug0 > $PBS_O_WORKDIR/results/cifar10.xSeed0@xSize0-xValid0IterxKimg0UxUratio0CxConf0WDxWd0BSxBatch0LRxLr0AxAug0_xRep0 & 27 | 28 | echo "FixMatch cifar10.xSeed@xSize-xValidIterxKimgUxUratioCxConfWDxWdBSxBatchLRxLrAxAug_xRep" 29 | CUDA_VISIBLE_DEVICES=1 python fixmatch.py --train_kimg xKimg1 --uratio xUratio1 --confidence xConf1 --wd xWd1 --wu 1 --batch xBatch1 --lr xLr1 --filters=32 --scales 3 --repeat 4 --dataset=cifar10.xSeed1@xSize1-xValid1 --train_dir experiments/fixmatch/SxSeed1@xSize1-xValid1IterxKimg1UxUratio1CxConf1WDxWd1BSxBatch1LRxLr1AxAug1_xRep1 --augment xAug1 > $PBS_O_WORKDIR/results/cifar10.xSeed1@xSize1-xValid1IterxKimg1UxUratio1CxConf1WDxWd1BSxBatch1LRxLr1AxAug1_xRep1 30 | 31 | wait 32 | 33 | date 34 | 35 | exit 36 | -------------------------------------------------------------------------------- /TF-BOSS/scripts/z2.pbs: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #PBS -l walltime=xTime:00:00 3 | #PBS -l select=1:ncpus=12:mpiprocs=12:ngpus=2 4 | #PBS -l place=scatter:excl 5 | #PBS -N TFBOSS2SxSeed0 6 | #PBS -j oe 7 | #PBS -V 8 | #PBS -q standard 9 | #PBS -A ERDCS97260VU3 10 | 11 | module load python36 12 | module load openmpi/cuda/64/3.1.4 13 | module load tensorflow-py36-cuda10.1-gcc/1.14.0 14 | 15 | cd $PBS_O_WORKDIR 16 | cd .. 17 | pwd 18 | cp -r BOSS /tmp/ 19 | cd /tmp/BOSS 20 | export ML_DATA="/tmp/BOSS/data" 21 | export PYTHONPATH=$PYTHONPATH:$PWD 22 | 23 | date 24 | 25 | echo "FixMatch cifar10.xSeed0@xSize0-xValid0IterxKimg0UxUratio0CxConf0WDxWd0BSxBatch0LRxLr0AxAug0_xRep0" 26 | CUDA_VISIBLE_DEVICES=xRep0 python fixmatch.py --train_kimg xKimg0 --uratio xUratio0 --confidence xConf0 --wd xWd0 --wu 1 --batch xBatch0 --lr xLr0 --filters=32 --scales 3 --repeat 4 --dataset=cifar10.xSeed0@xSize0-xValid0 --train_dir experiments/fixmatch/SxSeed0@xSize0-xValid0IterxKimg0UxUratio0CxConf0WDxWd0BSxBatch0LRxLr0AxAug0_xRep0 --augment xAug0 > $PBS_O_WORKDIR/results/cifar10.xSeed0@xSize0-xValid0IterxKimg0UxUratio0CxConf0WDxWd0BSxBatch0LRxLr0AxAug0_xRep0 & 27 | 28 | echo "FixMatch cifar10.xSeed0@xSize0-xValid0IterxKimg0UxUratio0CxConf0WDxWd0BSxBatch0LRxLr0AxAug0_xRep1" 29 | CUDA_VISIBLE_DEVICES=xRep1 python fixmatch.py --train_kimg xKimg0 --uratio xUratio0 --confidence xConf0 --wd xWd0 --wu 1 --batch xBatch0 --lr xLr0 --filters=32 --scales 3 --repeat 4 --dataset=cifar10.xSeed0@xSize0-xValid0 --train_dir experiments/fixmatch/SxSeed0@xSize0-xValid0IterxKimg0UxUratio0CxConf0WDxWd0BSxBatch0LRxLr0AxAug0_xRep1 --augment xAug0 > $PBS_O_WORKDIR/results/cifar10.xSeed0@xSize0-xValid0IterxKimg0UxUratio0CxConf0WDxWd0BSxBatch0LRxLr0AxAug0_xRep1 30 | 31 | wait 32 | 33 | date 34 | 35 | exit 36 | -------------------------------------------------------------------------------- /TF-BOSS/setup.sh: -------------------------------------------------------------------------------- 1 | mkdir ./data 2 | export ML_DATA="./data" 3 | export PYTHONPATH=$PYTHONPATH:$PWD 4 | 5 | python ./scripts/create_datasets.py 6 | #cp $ML_DATA/svhn-test.tfrecord $ML_DATA/svhn_noextra-test.tfrecord 7 | 8 | # Create unlabeled datasets 9 | #python scripts/create_unlabeled.py $ML_DATA/SSL2/svhn $ML_DATA/svhn-train.tfrecord $ML_DATA/svhn-extra.tfrecord & 10 | #python scripts/create_unlabeled.py $ML_DATA/SSL2/svhn_noextra $ML_DATA/svhn-train.tfrecord & 11 | python scripts/create_unlabeled.py $ML_DATA/SSL2/cifar10 $ML_DATA/cifar10-train.tfrecord & 12 | 13 | 14 | wait 15 | 16 | # Create semi-supervised subsets 17 | for seed in 0 1 2 3 4 5; do 18 | for size in 10 20 30 40 100 250 1000 4000; do 19 | # python scripts/create_split.py --seed=$seed --size=$size $ML_DATA/SSL2/svhn $ML_DATA/svhn-train.tfrecord $ML_DATA/svhn-extra.tfrecord & 20 | # python scripts/create_split.py --seed=$seed --size=$size $ML_DATA/SSL2/svhn_noextra $ML_DATA/svhn-train.tfrecord & 21 | python scripts/create_split.py --seed=$seed --size=$size $ML_DATA/SSL2/cifar10 $ML_DATA/cifar10-train.tfrecord & 22 | done 23 | done 24 | 25 | -------------------------------------------------------------------------------- /TF-BOSS/submit.sh: -------------------------------------------------------------------------------- 1 | dataset=cifar10p 2 | size=10 3 | seed=1 4 | valid=5000 5 | 6 | kimg=65536 7 | ratio=7 8 | con=0.95 9 | wd=5e-4 10 | batch=64 11 | lr=0.03 12 | i=0 13 | aug="d.d.d" 14 | 15 | #for con in 0.95 0.9; do 16 | #for ratio in 7 8 16; do 17 | #for batch in 32 64 128; do 18 | #for lr in 0.03 0.06 0.01; do 19 | #for wd in 5e-4 3e-4 2e-4; do 20 | #for kimg in 32768 65536; do 21 | #for aug in "rac.m.rac" "d.m.rac" "aac.m.aac" "d.m.aac" "d.d.aac" "d.d.rac"; do 22 | #for i in 1 2 3; do 23 | for seed in 0 1 2 3 4 5; do 24 | filename="cifar10${dataset}.${seed}@${size}-${valid}Iter${kimg}U${ratio}C${con}WD${wd}BS${batch}LR${lr}Aug${aug}_$i" 25 | echo $filename 26 | echo $filename >> history 27 | sed -e "s/xSeed/${seed}/g" -e "s/xSize/${size}/g" -e "s/xValid/${valid}/g" -e "s/xTime/120/g" -e "s/xKimg/$kimg/g" -e "s/xUratio/$ratio/g" -e "s/xConf/$con/g" -e "s/xWd/$wd/g" -e "s/xBatch/$batch/g" -e "s/xLr/$lr/g" -e "s/xRep/$i/g" -e "s/xAug/$aug/g" z.pbs > Q/$filename 28 | # exit 29 | qsub Q/$filename 30 | sleep 1 31 | done 32 | #done 33 | 34 | exit 35 | 36 | for seed in 0 1 2 3 4 5; do 37 | for valid in 1 5000; do 38 | filename="cifar10${dataset}.${seed}@${size}-${valid}" 39 | echo "${dataset}.${seed}@${size}-${valid}" 40 | sed -e "s/xSeed/${seed}/g" -e "s/xSize/${size}/g" -e "s/xValid/${valid}/g" -e "s/xTime/72/g" z.pbs > Q/$filename 41 | # exit 42 | qsub Q/$filename 43 | sleep 1 44 | done; done 45 | -------------------------------------------------------------------------------- /TF-BOSS/svhnData.sh: -------------------------------------------------------------------------------- 1 | 2 | export ML_DATA="./data" 3 | export PYTHONPATH=$PYTHONPATH:$PWD 4 | 5 | date 6 | size=10 7 | 8 | cp $ML_DATA/svhn_noextra-test.tfrecord $ML_DATA/svhnp_noextra-test.tfrecord 9 | cp $ML_DATA/svhn-extra.tfrecord $ML_DATA/svhnp-extra.tfrecord 10 | cp $ML_DATA/svhn-test.tfrecord $ML_DATA/svhnp-test.tfrecord 11 | cp $ML_DATA/svhn-train.tfrecord $ML_DATA/svhnp-train.tfrecord 12 | cp $ML_DATA/svhn-test.tfrecord $ML_DATA/svhnp_noextra-test.tfrecord 13 | cp $ML_DATA/svhn-train.tfrecord $ML_DATA/svhnp_noextra-train.tfrecord 14 | 15 | mkdir data/pseudolabeled 16 | mkdir data/pseudolabeled/top 17 | cp data/svhnp-train.tfrecord data/pseudolabeled/ 18 | 19 | cp $ML_DATA/SSL2/svhn-unlabel.json $ML_DATA/SSL2/svhnp-unlabel.json 20 | cp $ML_DATA/SSL2/svhn-unlabel.tfrecord $ML_DATA/SSL2/svhnp-unlabel.tfrecord 21 | cp $ML_DATA/SSL2/svhn_noextra-unlabel.json $ML_DATA/SSL2/svhnp_noextra-unlabel.json 22 | cp $ML_DATA/SSL2/svhn_noextra-unlabel.tfrecord $ML_DATA/SSL2/svhnp_noextra-unlabel.tfrecord 23 | 24 | 25 | size=10 26 | 27 | for seed in 1 2 3 4; do 28 | CUDA_VISIBLE_DEVICES= scripts/svhn_prototypes.py --seed=$seed --size=$size $ML_DATA/SSL2/svhnp $ML_DATA/svhnp-train.tfrecord $ML_DATA/svhnp-extra.tfrecord 29 | # CUDA_VISIBLE_DEVICES= scripts/svhn_prototypes.py --seed=$seed --size=$size $ML_DATA/SSL2/svhnp_noextra $ML_DATA/svhnp-train.tfrecord & 30 | done 31 | wait 32 | 33 | 34 | 35 | date 36 | exit 37 | 38 | -------------------------------------------------------------------------------- /TF-BOSS/third_party/auto_augment/custom_ops.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2019 The Google UDA Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Contains convenience wrappers for typical Neural Network TensorFlow layers. 16 | 17 | Ops that have different behavior during training or eval have an is_training 18 | parameter. 19 | 20 | Copied from AutoAugment: https://github.com/tensorflow/models/blob/master/research/autoaugment/ 21 | """ 22 | from __future__ import absolute_import 23 | from __future__ import division 24 | from __future__ import print_function 25 | 26 | import numpy as np 27 | import tensorflow as tf 28 | 29 | arg_scope = tf.contrib.framework.arg_scope 30 | FLAGS = tf.flags.FLAGS 31 | 32 | 33 | def variable(name, shape, dtype, initializer, trainable): 34 | """Returns a TF variable with the passed in specifications.""" 35 | var = tf.get_variable( 36 | name, 37 | shape=shape, 38 | dtype=dtype, 39 | initializer=initializer, 40 | trainable=trainable) 41 | return var 42 | 43 | 44 | def global_avg_pool(x, scope=None): 45 | """Average pools away spatial height and width dimension of 4D tensor.""" 46 | assert x.get_shape().ndims == 4 47 | with tf.name_scope(scope, 'global_avg_pool', [x]): 48 | kernel_size = (1, int(x.shape[1]), int(x.shape[2]), 1) 49 | squeeze_dims = (1, 2) 50 | result = tf.nn.avg_pool( 51 | x, 52 | ksize=kernel_size, 53 | strides=(1, 1, 1, 1), 54 | padding='VALID', 55 | data_format='NHWC') 56 | return tf.squeeze(result, squeeze_dims) 57 | 58 | 59 | def zero_pad(inputs, in_filter, out_filter): 60 | """Zero pads `input` tensor to have `out_filter` number of filters.""" 61 | outputs = tf.pad(inputs, [[0, 0], [0, 0], [0, 0], 62 | [(out_filter - in_filter) // 2, 63 | (out_filter - in_filter) // 2]]) 64 | return outputs 65 | 66 | 67 | @tf.contrib.framework.add_arg_scope 68 | def batch_norm(inputs, 69 | update_stats=True, 70 | decay=0.999, 71 | center=True, 72 | scale=False, 73 | epsilon=0.001, 74 | is_training=True, 75 | reuse=None, 76 | scope=None, 77 | ): 78 | """Small wrapper around tf.contrib.layers.batch_norm.""" 79 | batch_norm_op = tf.layers.batch_normalization( 80 | inputs, 81 | axis=-1, 82 | momentum=decay, 83 | epsilon=epsilon, 84 | center=center, 85 | scale=scale, 86 | training=is_training, 87 | fused=True, 88 | trainable=True, 89 | ) 90 | return batch_norm_op 91 | 92 | 93 | def stride_arr(stride_h, stride_w): 94 | return [1, stride_h, stride_w, 1] 95 | 96 | 97 | @tf.contrib.framework.add_arg_scope 98 | def conv2d(inputs, 99 | num_filters_out, 100 | kernel_size, 101 | stride=1, 102 | scope=None, 103 | reuse=None): 104 | """Adds a 2D convolution. 105 | 106 | conv2d creates a variable called 'weights', representing the convolutional 107 | kernel, that is convolved with the input. 108 | 109 | Args: 110 | inputs: a 4D tensor in NHWC format. 111 | num_filters_out: the number of output filters. 112 | kernel_size: an int specifying the kernel height and width size. 113 | stride: an int specifying the height and width stride. 114 | scope: Optional scope for variable_scope. 115 | reuse: whether or not the layer and its variables should be reused. 116 | Returns: 117 | a tensor that is the result of a convolution being applied to `inputs`. 118 | """ 119 | with tf.variable_scope(scope, 'Conv', [inputs], reuse=reuse): 120 | num_filters_in = int(inputs.shape[3]) 121 | weights_shape = [kernel_size, kernel_size, num_filters_in, num_filters_out] 122 | 123 | # Initialization 124 | n = int(weights_shape[0] * weights_shape[1] * weights_shape[3]) 125 | weights_initializer = tf.random_normal_initializer( 126 | stddev=np.sqrt(2.0 / n)) 127 | 128 | weights = variable( 129 | name='weights', 130 | shape=weights_shape, 131 | dtype=tf.float32, 132 | initializer=weights_initializer, 133 | trainable=True) 134 | strides = stride_arr(stride, stride) 135 | outputs = tf.nn.conv2d( 136 | inputs, weights, strides, padding='SAME', data_format='NHWC') 137 | return outputs 138 | 139 | 140 | @tf.contrib.framework.add_arg_scope 141 | def fc(inputs, 142 | num_units_out, 143 | scope=None, 144 | reuse=None): 145 | """Creates a fully connected layer applied to `inputs`. 146 | 147 | Args: 148 | inputs: a tensor that the fully connected layer will be applied to. It 149 | will be reshaped if it is not 2D. 150 | num_units_out: the number of output units in the layer. 151 | scope: Optional scope for variable_scope. 152 | reuse: whether or not the layer and its variables should be reused. 153 | 154 | Returns: 155 | a tensor that is the result of applying a linear matrix to `inputs`. 156 | """ 157 | if len(inputs.shape) > 2: 158 | inputs = tf.reshape(inputs, [int(inputs.shape[0]), -1]) 159 | 160 | with tf.variable_scope(scope, 'FC', [inputs], reuse=reuse): 161 | num_units_in = inputs.shape[1] 162 | weights_shape = [num_units_in, num_units_out] 163 | unif_init_range = 1.0 / (num_units_out) ** (0.5) 164 | weights_initializer = tf.random_uniform_initializer( 165 | -unif_init_range, unif_init_range) 166 | weights = variable( 167 | name='weights', 168 | shape=weights_shape, 169 | dtype=tf.float32, 170 | initializer=weights_initializer, 171 | trainable=True) 172 | bias_initializer = tf.constant_initializer(0.0) 173 | biases = variable( 174 | name='biases', 175 | shape=[num_units_out, ], 176 | dtype=tf.float32, 177 | initializer=bias_initializer, 178 | trainable=True) 179 | outputs = tf.nn.xw_plus_b(inputs, weights, biases) 180 | return outputs 181 | 182 | 183 | @tf.contrib.framework.add_arg_scope 184 | def avg_pool(inputs, kernel_size, stride=2, padding='VALID', scope=None): 185 | """Wrapper around tf.nn.avg_pool.""" 186 | with tf.name_scope(scope, 'AvgPool', [inputs]): 187 | kernel = stride_arr(kernel_size, kernel_size) 188 | strides = stride_arr(stride, stride) 189 | return tf.nn.avg_pool( 190 | inputs, 191 | ksize=kernel, 192 | strides=strides, 193 | padding=padding, 194 | data_format='NHWC') 195 | -------------------------------------------------------------------------------- /TF-BOSS/third_party/auto_augment/shake_drop.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2019 The Google UDA Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Builds the Shake-Drop Model. 16 | Copied from AutoAugment: https://github.com/tensorflow/models/blob/master/research/autoaugment/ 17 | """ 18 | 19 | from __future__ import absolute_import 20 | from __future__ import division 21 | from __future__ import print_function 22 | 23 | import math 24 | 25 | import tensorflow as tf 26 | 27 | import third_party.auto_augment.custom_ops as ops 28 | 29 | 30 | def round_int(x): 31 | """Rounds `x` and then converts to an int.""" 32 | return int(math.floor(x + 0.5)) 33 | 34 | 35 | def shortcut(x, output_filters, stride): 36 | """Applies strided avg pool or zero padding to make output_filters match x.""" 37 | num_filters = int(x.shape[3]) 38 | if stride == 2: 39 | x = ops.avg_pool(x, 2, stride=stride, padding='SAME') 40 | if num_filters != output_filters: 41 | diff = output_filters - num_filters 42 | assert diff > 0 43 | # Zero padd diff zeros 44 | padding = [[0, 0], [0, 0], [0, 0], [0, diff]] 45 | x = tf.pad(x, padding) 46 | return x 47 | 48 | 49 | def calc_prob(curr_layer, total_layers, p_l): 50 | """Calculates drop prob depending on the current layer.""" 51 | return 1 - (float(curr_layer) / total_layers) * p_l 52 | 53 | 54 | def bottleneck_layer(x, n, stride, prob, is_training, alpha, beta): 55 | """Bottleneck layer for shake drop model.""" 56 | assert alpha[1] > alpha[0] 57 | assert beta[1] > beta[0] 58 | with tf.variable_scope('bottleneck_{}'.format(prob)): 59 | input_layer = x 60 | x = ops.batch_norm(x, scope='bn_1_pre') 61 | x = ops.conv2d(x, n, 1, scope='1x1_conv_contract') 62 | x = ops.batch_norm(x, scope='bn_1_post') 63 | x = tf.nn.relu(x) 64 | x = ops.conv2d(x, n, 3, stride=stride, scope='3x3') 65 | x = ops.batch_norm(x, scope='bn_2') 66 | x = tf.nn.relu(x) 67 | x = ops.conv2d(x, n * 4, 1, scope='1x1_conv_expand') 68 | x = ops.batch_norm(x, scope='bn_3') 69 | 70 | # Apply regularization here 71 | # Sample bernoulli with prob 72 | if is_training: 73 | batch_size = tf.shape(x)[0] 74 | bern_shape = [batch_size, 1, 1, 1] 75 | random_tensor = prob 76 | random_tensor += tf.random_uniform(bern_shape, dtype=tf.float32) 77 | binary_tensor = tf.floor(random_tensor) 78 | 79 | alpha_values = tf.random_uniform( 80 | [batch_size, 1, 1, 1], minval=alpha[0], maxval=alpha[1], 81 | dtype=tf.float32) 82 | beta_values = tf.random_uniform( 83 | [batch_size, 1, 1, 1], minval=beta[0], maxval=beta[1], 84 | dtype=tf.float32) 85 | rand_forward = ( 86 | binary_tensor + alpha_values - binary_tensor * alpha_values) 87 | rand_backward = ( 88 | binary_tensor + beta_values - binary_tensor * beta_values) 89 | x = x * rand_backward + tf.stop_gradient(x * rand_forward - 90 | x * rand_backward) 91 | else: 92 | expected_alpha = (alpha[1] + alpha[0]) / 2 93 | # prob is the expectation of the bernoulli variable 94 | x = (prob + expected_alpha - prob * expected_alpha) * x 95 | 96 | res = shortcut(input_layer, n * 4, stride) 97 | return x + res 98 | 99 | 100 | def build_shake_drop_model(images, num_classes, is_training): 101 | """Builds the PyramidNet Shake-Drop model. 102 | 103 | Build the PyramidNet Shake-Drop model from https://arxiv.org/abs/1802.02375. 104 | 105 | Args: 106 | images: Tensor of images that will be fed into the Wide ResNet Model. 107 | num_classes: Number of classed that the model needs to predict. 108 | is_training: Is the model training or not. 109 | 110 | Returns: 111 | The logits of the PyramidNet Shake-Drop model. 112 | """ 113 | 114 | is_training = is_training 115 | # ShakeDrop Hparams 116 | p_l = 0.5 117 | alpha_shake = [-1, 1] 118 | beta_shake = [0, 1] 119 | 120 | # PyramidNet Hparams 121 | alpha = 200 122 | depth = 272 123 | # This is for the bottleneck architecture specifically 124 | n = int((depth - 2) / 9) 125 | start_channel = 16 126 | add_channel = alpha / (3 * n) 127 | 128 | # Building the models 129 | x = images 130 | x = ops.conv2d(x, 16, 3, scope='init_conv') 131 | x = ops.batch_norm(x, scope='init_bn') 132 | 133 | layer_num = 1 134 | total_layers = n * 3 135 | start_channel += add_channel 136 | prob = calc_prob(layer_num, total_layers, p_l) 137 | x = bottleneck_layer( 138 | x, round_int(start_channel), 1, prob, is_training, alpha_shake, 139 | beta_shake) 140 | layer_num += 1 141 | for _ in range(1, n): 142 | start_channel += add_channel 143 | prob = calc_prob(layer_num, total_layers, p_l) 144 | x = bottleneck_layer( 145 | x, round_int(start_channel), 1, prob, is_training, alpha_shake, 146 | beta_shake) 147 | layer_num += 1 148 | 149 | start_channel += add_channel 150 | prob = calc_prob(layer_num, total_layers, p_l) 151 | x = bottleneck_layer( 152 | x, round_int(start_channel), 2, prob, is_training, alpha_shake, 153 | beta_shake) 154 | layer_num += 1 155 | for _ in range(1, n): 156 | start_channel += add_channel 157 | prob = calc_prob(layer_num, total_layers, p_l) 158 | x = bottleneck_layer( 159 | x, round_int(start_channel), 1, prob, is_training, alpha_shake, 160 | beta_shake) 161 | layer_num += 1 162 | 163 | start_channel += add_channel 164 | prob = calc_prob(layer_num, total_layers, p_l) 165 | x = bottleneck_layer( 166 | x, round_int(start_channel), 2, prob, is_training, alpha_shake, 167 | beta_shake) 168 | layer_num += 1 169 | for _ in range(1, n): 170 | start_channel += add_channel 171 | prob = calc_prob(layer_num, total_layers, p_l) 172 | x = bottleneck_layer( 173 | x, round_int(start_channel), 1, prob, is_training, alpha_shake, 174 | beta_shake) 175 | layer_num += 1 176 | 177 | assert layer_num - 1 == total_layers 178 | x = ops.batch_norm(x, scope='final_bn') 179 | x = tf.nn.relu(x) 180 | x = ops.global_avg_pool(x) 181 | # Fully connected 182 | logits = ops.fc(x, num_classes) 183 | return logits 184 | -------------------------------------------------------------------------------- /TF-BOSS/third_party/auto_augment/shake_shake.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2019 The Google UDA Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Builds the Shake-Shake Model. 16 | Copied from AutoAugment: https://github.com/tensorflow/models/blob/master/research/autoaugment/ 17 | """ 18 | 19 | from __future__ import absolute_import 20 | from __future__ import division 21 | from __future__ import print_function 22 | 23 | import tensorflow as tf 24 | 25 | import third_party.auto_augment.custom_ops as ops 26 | 27 | 28 | def _shake_shake_skip_connection(x, output_filters, stride): 29 | """Adds a residual connection to the filter x for the shake-shake model.""" 30 | curr_filters = int(x.shape[3]) 31 | if curr_filters == output_filters: 32 | return x 33 | stride_spec = ops.stride_arr(stride, stride) 34 | # Skip path 1 35 | path1 = tf.nn.avg_pool( 36 | x, [1, 1, 1, 1], stride_spec, 'VALID', data_format='NHWC') 37 | path1 = ops.conv2d(path1, int(output_filters / 2), 1, scope='path1_conv') 38 | 39 | # Skip path 2 40 | # First pad with 0's then crop 41 | pad_arr = [[0, 0], [0, 1], [0, 1], [0, 0]] 42 | path2 = tf.pad(x, pad_arr)[:, 1:, 1:, :] 43 | concat_axis = 3 44 | 45 | path2 = tf.nn.avg_pool( 46 | path2, [1, 1, 1, 1], stride_spec, 'VALID', data_format='NHWC') 47 | path2 = ops.conv2d(path2, int(output_filters / 2), 1, scope='path2_conv') 48 | 49 | # Concat and apply BN 50 | final_path = tf.concat(values=[path1, path2], axis=concat_axis) 51 | final_path = ops.batch_norm(final_path, scope='final_path_bn') 52 | return final_path 53 | 54 | 55 | def _shake_shake_branch(x, output_filters, stride, rand_forward, rand_backward, 56 | is_training): 57 | """Building a 2 branching convnet.""" 58 | x = tf.nn.relu(x) 59 | x = ops.conv2d(x, output_filters, 3, stride=stride, scope='conv1') 60 | x = ops.batch_norm(x, scope='bn1') 61 | x = tf.nn.relu(x) 62 | x = ops.conv2d(x, output_filters, 3, scope='conv2') 63 | x = ops.batch_norm(x, scope='bn2') 64 | if is_training: 65 | x = x * rand_backward + tf.stop_gradient(x * rand_forward - 66 | x * rand_backward) 67 | else: 68 | x *= 1.0 / 2 69 | return x 70 | 71 | 72 | def _shake_shake_block(x, output_filters, stride, is_training): 73 | """Builds a full shake-shake sub layer.""" 74 | batch_size = tf.shape(x)[0] 75 | 76 | # Generate random numbers for scaling the branches 77 | rand_forward = [ 78 | tf.random_uniform( 79 | [batch_size, 1, 1, 1], minval=0, maxval=1, dtype=tf.float32) 80 | for _ in range(2) 81 | ] 82 | rand_backward = [ 83 | tf.random_uniform( 84 | [batch_size, 1, 1, 1], minval=0, maxval=1, dtype=tf.float32) 85 | for _ in range(2) 86 | ] 87 | # Normalize so that all sum to 1 88 | total_forward = tf.add_n(rand_forward) 89 | total_backward = tf.add_n(rand_backward) 90 | rand_forward = [samp / total_forward for samp in rand_forward] 91 | rand_backward = [samp / total_backward for samp in rand_backward] 92 | zipped_rand = zip(rand_forward, rand_backward) 93 | 94 | branches = [] 95 | for branch, (r_forward, r_backward) in enumerate(zipped_rand): 96 | with tf.variable_scope('branch_{}'.format(branch)): 97 | b = _shake_shake_branch(x, output_filters, stride, r_forward, r_backward, 98 | is_training) 99 | branches.append(b) 100 | res = _shake_shake_skip_connection(x, output_filters, stride) 101 | return res + tf.add_n(branches) 102 | 103 | 104 | def _shake_shake_layer(x, output_filters, num_blocks, stride, 105 | is_training): 106 | """Builds many sub layers into one full layer.""" 107 | for block_num in range(num_blocks): 108 | curr_stride = stride if (block_num == 0) else 1 109 | with tf.variable_scope('layer_{}'.format(block_num)): 110 | x = _shake_shake_block(x, output_filters, curr_stride, 111 | is_training) 112 | return x 113 | 114 | 115 | def build_shake_shake_model(images, num_classes, hparams, is_training): 116 | """Builds the Shake-Shake model. 117 | 118 | Build the Shake-Shake model from https://arxiv.org/abs/1705.07485. 119 | 120 | Args: 121 | images: Tensor of images that will be fed into the Wide ResNet Model. 122 | num_classes: Number of classed that the model needs to predict. 123 | hparams: tf.HParams object that contains additional hparams needed to 124 | construct the model. In this case it is the `shake_shake_widen_factor` 125 | that is used to determine how many filters the model has. 126 | is_training: Is the model training or not. 127 | 128 | Returns: 129 | The logits of the Shake-Shake model. 130 | """ 131 | depth = 26 132 | k = hparams.shake_shake_widen_factor # The widen factor 133 | n = int((depth - 2) / 6) 134 | x = images 135 | 136 | x = ops.conv2d(x, 16, 3, scope='init_conv') 137 | x = ops.batch_norm(x, scope='init_bn') 138 | with tf.variable_scope('L1'): 139 | x = _shake_shake_layer(x, 16 * k, n, 1, is_training) 140 | with tf.variable_scope('L2'): 141 | x = _shake_shake_layer(x, 32 * k, n, 2, is_training) 142 | with tf.variable_scope('L3'): 143 | x = _shake_shake_layer(x, 64 * k, n, 2, is_training) 144 | x = tf.nn.relu(x) 145 | x = ops.global_avg_pool(x) 146 | 147 | # Fully connected 148 | logits = ops.fc(x, num_classes) 149 | return logits 150 | -------------------------------------------------------------------------------- /TF-BOSS/third_party/auto_augment/wrn.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2019 The Google UDA Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Builds the WideResNet Model.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import numpy as np 22 | import tensorflow as tf 23 | 24 | import third_party.auto_augment.custom_ops as ops 25 | 26 | 27 | def residual_block( 28 | x, in_filter, out_filter, stride, update_bn=True, 29 | activate_before_residual=False): 30 | """Adds residual connection to `x` in addition to applying BN->ReLU->3x3 Conv. 31 | 32 | Args: 33 | x: Tensor that is the output of the previous layer in the model. 34 | in_filter: Number of filters `x` has. 35 | out_filter: Number of filters that the output of this layer will have. 36 | stride: Integer that specified what stride should be applied to `x`. 37 | activate_before_residual: Boolean on whether a BN->ReLU should be applied 38 | to x before the convolution is applied. 39 | 40 | Returns: 41 | A Tensor that is the result of applying two sequences of BN->ReLU->3x3 Conv 42 | and then adding that Tensor to `x`. 43 | """ 44 | 45 | if activate_before_residual: # Pass up RELU and BN activation for resnet 46 | with tf.variable_scope('shared_activation'): 47 | x = ops.batch_norm(x, update_stats=update_bn, scope='init_bn') 48 | x = tf.nn.relu(x) 49 | orig_x = x 50 | else: 51 | orig_x = x 52 | 53 | block_x = x 54 | if not activate_before_residual: 55 | with tf.variable_scope('residual_only_activation'): 56 | block_x = ops.batch_norm(block_x, update_stats=update_bn, 57 | scope='init_bn') 58 | block_x = tf.nn.relu(block_x) 59 | 60 | with tf.variable_scope('sub1'): 61 | block_x = ops.conv2d( 62 | block_x, out_filter, 3, stride=stride, scope='conv1') 63 | 64 | with tf.variable_scope('sub2'): 65 | block_x = ops.batch_norm(block_x, update_stats=update_bn, scope='bn2') 66 | block_x = tf.nn.relu(block_x) 67 | block_x = ops.conv2d( 68 | block_x, out_filter, 3, stride=1, scope='conv2') 69 | 70 | with tf.variable_scope( 71 | 'sub_add'): # If number of filters do not agree then zero pad them 72 | if in_filter != out_filter: 73 | orig_x = ops.avg_pool(orig_x, stride, stride) 74 | orig_x = ops.zero_pad(orig_x, in_filter, out_filter) 75 | x = orig_x + block_x 76 | return x 77 | 78 | 79 | def _res_add(in_filter, out_filter, stride, x, orig_x): 80 | """Adds `x` with `orig_x`, both of which are layers in the model. 81 | 82 | Args: 83 | in_filter: Number of filters in `orig_x`. 84 | out_filter: Number of filters in `x`. 85 | stride: Integer specifying the stide that should be applied `orig_x`. 86 | x: Tensor that is the output of the previous layer. 87 | orig_x: Tensor that is the output of an earlier layer in the network. 88 | 89 | Returns: 90 | A Tensor that is the result of `x` and `orig_x` being added after 91 | zero padding and striding are applied to `orig_x` to get the shapes 92 | to match. 93 | """ 94 | if in_filter != out_filter: 95 | orig_x = ops.avg_pool(orig_x, stride, stride) 96 | orig_x = ops.zero_pad(orig_x, in_filter, out_filter) 97 | x = x + orig_x 98 | orig_x = x 99 | return x, orig_x 100 | 101 | 102 | def build_wrn_model(images, num_classes, wrn_size, update_bn=True): 103 | """Builds the WRN model. 104 | 105 | Build the Wide ResNet model from https://arxiv.org/abs/1605.07146. 106 | 107 | Args: 108 | images: Tensor of images that will be fed into the Wide ResNet Model. 109 | num_classes: Number of classed that the model needs to predict. 110 | wrn_size: Parameter that scales the number of filters in the Wide ResNet 111 | model. 112 | 113 | Returns: 114 | The logits of the Wide ResNet model. 115 | """ 116 | # wrn_size = 16 * widening factor k 117 | kernel_size = wrn_size 118 | filter_size = 3 119 | # depth = num_blocks_per_resnet * 6 + 4 = 28 120 | num_blocks_per_resnet = 4 121 | filters = [ 122 | min(kernel_size, 16), kernel_size, kernel_size * 2, kernel_size * 4 123 | ] 124 | strides = [1, 2, 2] # stride for each resblock 125 | 126 | # Run the first conv 127 | with tf.variable_scope('init'): 128 | x = images 129 | output_filters = filters[0] 130 | x = ops.conv2d(x, output_filters, filter_size, scope='init_conv') 131 | 132 | first_x = x # Res from the beginning 133 | orig_x = x # Res from previous block 134 | 135 | for block_num in range(1, 4): 136 | with tf.variable_scope('unit_{}_0'.format(block_num)): 137 | activate_before_residual = True if block_num == 1 else False 138 | x = residual_block( 139 | x, 140 | filters[block_num - 1], 141 | filters[block_num], 142 | strides[block_num - 1], 143 | update_bn=update_bn, 144 | activate_before_residual=activate_before_residual) 145 | for i in range(1, num_blocks_per_resnet): 146 | with tf.variable_scope('unit_{}_{}'.format(block_num, i)): 147 | x = residual_block( 148 | x, 149 | filters[block_num], 150 | filters[block_num], 151 | 1, 152 | update_bn=update_bn, 153 | activate_before_residual=False) 154 | x, orig_x = _res_add(filters[block_num - 1], filters[block_num], 155 | strides[block_num - 1], x, orig_x) 156 | final_stride_val = np.prod(strides) 157 | x, _ = _res_add(filters[0], filters[3], final_stride_val, x, first_x) 158 | with tf.variable_scope('unit_last'): 159 | x = ops.batch_norm(x, scope='final_bn') 160 | x = tf.nn.relu(x) 161 | x = ops.global_avg_pool(x) 162 | logits = ops.fc(x, num_classes) 163 | return logits 164 | --------------------------------------------------------------------------------