├── model ├── __init__.py ├── gan_utils.py ├── res_utils.py ├── testModel.py ├── DCGAN.py ├── WGAN.py ├── ACGAN.py ├── modelFactory.py ├── cDCGAN.py ├── densenet.py └── resnet32.py ├── dataHandler ├── __init__.py ├── datasetFactory.py ├── dataset.py ├── incrementalLoader.py └── selectedCIFARIndicesForTrainingDataK1.txt ├── experiment ├── __init__.py └── experiment.py ├── plotter ├── __init__.py └── plotter.py ├── trainer ├── __init__.py ├── gans │ ├── ganFactory.py │ ├── gan.py │ ├── wgan.py │ ├── dcgan.py │ ├── acgan.py │ ├── gutils.py │ └── cdcgan.py ├── nmcTrainer.py ├── classifierFactory.py ├── classifierTrainer.py └── ganTrainer.py ├── utils ├── __init__.py └── utils.py ├── requirements.txt ├── eQueueRunner.sh ├── .gitignore ├── README.md └── runExperiment.py /model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dataHandler/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /experiment/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plotter/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /trainer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | from utils.utils import * -------------------------------------------------------------------------------- /model/gan_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | def normal_init(m, mean, std, has_bias=True): 5 | #TODO Refactor this and init_weights 6 | if isinstance(m, nn.ConvTranspose2d) or isinstance(m, nn.Conv2d): 7 | m.weight.data.normal_(mean, std) 8 | if has_bias: 9 | m.bias.data.zero_() 10 | elif isinstance(m, nn.BatchNorm2d): 11 | m.weight.data.normal_(1.0, 0.02) 12 | m.bias.data.fill_(0) 13 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | certifi==2018.1.18 2 | chardet==3.0.4 3 | cycler==0.10.0 4 | idna==2.6 5 | imageio==2.2.0 6 | matplotlib==2.1.2 7 | numpy==1.14.0 8 | opencv-python==3.4.0.12 9 | Pillow==5.0.0 10 | pyparsing==2.2.0 11 | python-dateutil==2.6.1 12 | pytz==2018.3 13 | PyYAML==3.12 14 | pyzmq==17.0.0 15 | requests==2.18.4 16 | six==1.11.0 17 | torch==0.3.0 18 | torchfile==0.1.0 19 | torchnet==0.0.1 20 | torchvision==0.2.0 21 | tornado==4.5.3 22 | urllib3==1.22 23 | visdom==0.1.7 24 | -------------------------------------------------------------------------------- /dataHandler/datasetFactory.py: -------------------------------------------------------------------------------- 1 | import dataHandler.dataset as data 2 | 3 | 4 | class DatasetFactory(): 5 | def __init__(self): 6 | pass 7 | 8 | @staticmethod 9 | def get_dataset(name): 10 | if name == "MNIST": 11 | return data.MNIST() 12 | elif name == "CIFAR100": 13 | return data.CIFAR100() 14 | elif name == "CIFAR10": 15 | return data.CIFAR10() 16 | else: 17 | print("Unsupported dataset") 18 | assert (False) 19 | -------------------------------------------------------------------------------- /eQueueRunner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Reads all files matching experimentQueue/*.t and executes 3 | # their contents. The file is then deleted and output is logged 4 | # in .log file with same name. 5 | # Script keeps restarting every 10 seconds to look for new tasks. 6 | 7 | if [ "-e experimentQueue/*.t" ]; then 8 | for k in experimentQueue/*.t; do 9 | echo "Executing" $k 10 | chmod a+x $k 11 | cat $k >> $k.log 12 | $k >> $k.log 13 | rm -f $k 14 | if hash notify-send 2>/dev/null; then 15 | notify-send eQueueRunner $k" execution complete" 16 | fi 17 | done 18 | fi 19 | 20 | echo "Tasks complete, waiting for new experiments..." 21 | sleep 10 22 | script=$(readlink -f "$0") 23 | exec "$script" 24 | 25 | -------------------------------------------------------------------------------- /trainer/gans/ganFactory.py: -------------------------------------------------------------------------------- 1 | from trainer.gans.dcgan import DCGAN 2 | from trainer.gans.cdcgan import CDCGAN 3 | from trainer.gans.wgan import WGAN 4 | from trainer.gans.acgan import ACGAN 5 | 6 | class GANFactory(): 7 | @staticmethod 8 | def get_trainer(gan_type, args, total_classes, train_iterator, fixed_classifier, experiment): 9 | if gan_type=="dcgan": 10 | return DCGAN(args, total_classes, train_iterator, fixed_classifier, experiment) 11 | elif gan_type=="wgan": 12 | return WGAN(args, total_classes, train_iterator, fixed_classifier, experiment) 13 | elif gan_type=="cdcgan": 14 | return CDCGAN(args, total_classes, train_iterator, fixed_classifier, experiment) 15 | elif gan_type=="acgan": 16 | return ACGAN(args, total_classes, train_iterator, fixed_classifier, experiment) 17 | 18 | else: 19 | print ("Unsupported model, trainer not found.") 20 | assert(False) 21 | -------------------------------------------------------------------------------- /trainer/gans/gan.py: -------------------------------------------------------------------------------- 1 | from abc import abstractmethod 2 | 3 | import torch 4 | from torch.autograd import Variable 5 | 6 | class GAN(): 7 | ''' 8 | Base class to inherit the GAN trainers from 9 | ''' 10 | def __init__(self, args, total_classes, train_iterator, fixed_classifier, experiment): 11 | ''' 12 | args: arguments 13 | total_classes: total classes ._. 14 | train_iterator: Iterator over examples for GAN 15 | fixed_classifier: Classifier frozen at current increment 16 | experiment: Instance of Experiment class 17 | ''' 18 | self.args = args 19 | self.experiment = experiment 20 | self.total_classes = total_classes 21 | self.train_iterator = train_iterator 22 | self.fixed_classifier = fixed_classifier 23 | # Noise for generating examples 24 | self.fixed_noise = torch.randn(100,100,1,1) 25 | 26 | @abstractmethod 27 | def train(self, G, D, active_classes, increment): 28 | ''' 29 | Must implement in derived trainers 30 | ''' 31 | pass 32 | -------------------------------------------------------------------------------- /model/res_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | class DownsampleA(nn.Module): 5 | 6 | def __init__(self, n_in, n_out, stride): 7 | super(DownsampleA, self).__init__() 8 | assert stride == 2 9 | self.avg = nn.AvgPool2d(kernel_size=1, stride=stride) 10 | 11 | def forward(self, x): 12 | x = self.avg(x) 13 | return torch.cat((x, x.mul(0)), 1) 14 | 15 | class DownsampleC(nn.Module): 16 | 17 | def __init__(self, n_in, n_out, stride): 18 | super(DownsampleC, self).__init__() 19 | assert stride != 1 or n_in != n_out 20 | self.conv = nn.Conv2d(n_in, n_out, kernel_size=1, stride=stride, padding=0, bias=False) 21 | 22 | def forward(self, x): 23 | x = self.conv(x) 24 | return x 25 | 26 | class DownsampleD(nn.Module): 27 | 28 | def __init__(self, n_in, n_out, stride): 29 | super(DownsampleD, self).__init__() 30 | assert stride == 2 31 | self.conv = nn.Conv2d(n_in, n_out, kernel_size=2, stride=stride, padding=0, bias=False) 32 | self.bn = nn.BatchNorm2d(n_out) 33 | 34 | def forward(self, x): 35 | x = self.conv(x) 36 | x = self.bn(x) 37 | return x 38 | -------------------------------------------------------------------------------- /experiment/experiment.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | class Experiment: 5 | ''' 6 | Class to store results of any Experiment 7 | ''' 8 | def __init__(self, name, args, output_dir="../"): 9 | if not args is None: 10 | self.name = name 11 | self.params = vars(args) 12 | self.results = {} 13 | self.dir = output_dir 14 | 15 | ver = 0 16 | 17 | while os.path.exists("../" + self.name + "_" + str(ver)): 18 | ver += 1 19 | 20 | os.makedirs("../" + self.name + "_" + str(ver)) 21 | os.makedirs("../" + self.name + "_" + str(ver) + "/results") 22 | os.makedirs("../" + self.name + "_" + str(ver) + "/checkpoints") 23 | self.path = "../" + self.name + "_" + str(ver) + "/" 24 | 25 | self.results["Temp Results"]= [[1,2,3,4], [5,6,2,6]] 26 | 27 | def store_json(self): 28 | with open(self.path +"JSONDump", 'w') as outfile: 29 | json.dump(json.dumps(self.__dict__), outfile) 30 | 31 | 32 | import argparse 33 | if __name__ == "__main__": 34 | parser = argparse.ArgumentParser(description='iCarl2.0') 35 | args = parser.parse_args() 36 | e = Experiment("TestExperiment", args) 37 | e.store_json() 38 | -------------------------------------------------------------------------------- /model/testModel.py: -------------------------------------------------------------------------------- 1 | 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | import torch 5 | 6 | class Net(nn.Module): 7 | def __init__(self, no_classes): 8 | super(Net, self).__init__() 9 | self.conv1 = nn.Conv2d(3, 10, kernel_size=5,padding=(2,2)) 10 | self.conv2 = nn.Conv2d(10, 20, kernel_size=5,padding=(2,2)) 11 | self.conv2_bn1 = nn.BatchNorm2d(20) 12 | self.conv3 = nn.Conv2d(20, 30, kernel_size=5,padding=(2,2)) 13 | self.conv2_bn2 = nn.BatchNorm2d(30) 14 | self.conv4 = nn.Conv2d(30, 40, kernel_size=5,padding=(2,2)) 15 | self.conv2_bn3 = nn.BatchNorm2d(40) 16 | self.conv2_drop = nn.Dropout2d() 17 | self.fc1 = nn.Linear(640, 100) 18 | self.fc2 = nn.Linear(100, no_classes) 19 | 20 | def forward(self, x, feature=False): 21 | x = F.relu(F.max_pool2d(self.conv1(x), 2)) 22 | x = self.conv2_bn1(self.conv2(x)) 23 | x = F.relu(F.max_pool2d(self.conv2_bn2(self.conv3(x)), 2)) 24 | x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2_bn3(self.conv4(x))), 2)) 25 | # print ("X after conv", x.shape) 26 | x = x.view(-1, 640) 27 | if feature: 28 | # print ("Size = ",torch.norm(x, 2, 1).unsqueeze(1).shape) 29 | return x / torch.norm(x, 2, 1).unsqueeze(1) 30 | x = F.relu(self.fc1(x)) 31 | x = F.dropout(x, training=self.training) 32 | x = self.fc2(x) 33 | return F.sigmoid(x) 34 | # return F.softmax(x) -------------------------------------------------------------------------------- /model/DCGAN.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from .gan_utils import normal_init 5 | 6 | class Generator(nn.Module): 7 | def __init__(self, d=128, c=1): 8 | super(Generator, self).__init__() 9 | #ConvTranspose2d(in_channels, out_channels, kernel_size, stride=1, padding=0) 10 | print(">>> NOTICE: Adjust the network to match CDCGAN one") 11 | self.ct1_noise = nn.ConvTranspose2d(100, d*4, 4, 1, 0) 12 | self.ct1_noise_bn = nn.BatchNorm2d(d*4) 13 | self.ct2 = nn.ConvTranspose2d(d*4, d*2, 4, 2, 1) 14 | self.ct2_bn = nn.BatchNorm2d(d*2) 15 | self.ct3 = nn.ConvTranspose2d(d*2, d, 4, 2, 1) 16 | self.ct3_bn = nn.BatchNorm2d(d) 17 | self.ct4 = nn.ConvTranspose2d(d, c, 4, 2, 1) 18 | 19 | def forward(self, noise): 20 | x = F.relu(self.ct1_noise_bn(self.ct1_noise(noise))) 21 | x = F.relu(self.ct2_bn(self.ct2(x))) 22 | x = F.relu(self.ct3_bn(self.ct3(x))) 23 | x = F.tanh(self.ct4(x)) 24 | return x 25 | 26 | def init_weights(self, mean, std): 27 | for m in self._modules: 28 | normal_init(self._modules[m], mean, std) 29 | 30 | class Discriminator(nn.Module): 31 | def __init__(self, d=128, c=1): 32 | super(Discriminator, self).__init__() 33 | self.conv1_img = nn.Conv2d(c, d//2, 4, 2, 1) 34 | self.conv1_bn = nn.BatchNorm2d(d//2) 35 | self.conv2 = nn.Conv2d(d//2, d*2, 4, 2, 1) 36 | self.conv2_bn = nn.BatchNorm2d(d*2) 37 | self.conv3 = nn.Conv2d(d*2, d*4, 4, 2, 1) 38 | self.conv3_bn = nn.BatchNorm2d(d*4) 39 | self.conv4 = nn.Conv2d(d * 4, 1, 4, 1, 0) 40 | 41 | def forward(self, img): 42 | x = F.leaky_relu(self.conv1_bn(self.conv1_img(img)), 0.2) 43 | x = F.leaky_relu(self.conv2_bn(self.conv2(x)), 0.2) 44 | x = F.leaky_relu(self.conv3_bn(self.conv3(x)), 0.2) 45 | x = F.sigmoid(self.conv4(x)) 46 | return x 47 | 48 | def init_weights(self, mean, std): 49 | for m in self._modules: 50 | normal_init(self._modules[m], mean, std) 51 | -------------------------------------------------------------------------------- /model/WGAN.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from .gan_utils import normal_init 5 | 6 | # "Directly applying batchnorm to all layers however, 7 | # resulted in sample oscillation and model instability. 8 | # This was avoided by not applying batchnorm to the 9 | # generator output layer and the discriminator input layer." 10 | # DCGAN paper https://arxiv.org/pdf/1511.06434.pdf 11 | 12 | #TODO Add the jump incase of d=128 as in original WGAN 13 | class Generator(nn.Module): 14 | def __init__(self, d=64, c=1): 15 | super(Generator, self).__init__() 16 | #ConvTranspose2d(in_channels, out_channels, kernel_size, stride=1, padding=0) 17 | self.ct1_noise = nn.ConvTranspose2d(100, d*4, 4, 1, 0) 18 | self.ct1_noise_bn = nn.BatchNorm2d(d*4) 19 | self.ct2 = nn.ConvTranspose2d(d*4, d*2, 4, 2, 1) 20 | self.ct2_bn = nn.BatchNorm2d(d*2) 21 | self.ct3 = nn.ConvTranspose2d(d*2, d, 4, 2, 1) 22 | self.ct3_bn = nn.BatchNorm2d(d) 23 | self.ct4 = nn.ConvTranspose2d(d, c, 4, 2, 1) 24 | print(self) 25 | 26 | def forward(self, noise): 27 | x = F.relu(self.ct1_noise_bn(self.ct1_noise(noise))) 28 | x = F.relu(self.ct2_bn(self.ct2(x))) 29 | x = F.relu(self.ct3_bn(self.ct3(x))) 30 | x = F.tanh(self.ct4(x)) 31 | return x 32 | 33 | def init_weights(self, mean, std): 34 | for m in self._modules: 35 | normal_init(self._modules[m], mean, std) 36 | 37 | class Discriminator(nn.Module): 38 | def __init__(self, d=64, c=1): 39 | super(Discriminator, self).__init__() 40 | self.conv1_img = nn.Conv2d(c, d, 4, 2, 1) 41 | self.conv2 = nn.Conv2d(d, d*2, 4, 2, 1) 42 | self.conv2_bn = nn.BatchNorm2d(d*2) 43 | self.conv3 = nn.Conv2d(d*2, d*4, 4, 2, 1) 44 | self.conv3_bn = nn.BatchNorm2d(d*4) 45 | self.conv4 = nn.Conv2d(d*4, 1, 4, 1, 0) 46 | #self.conv4_bn = nn.BatchNorm2d(d*8) 47 | #self.conv5 = nn.Conv2d(d*8, 1, 4, 1, 0) 48 | print(self) 49 | 50 | def forward(self, img): 51 | #Note: no sigmoid for wgan 52 | x = F.leaky_relu(self.conv1_img(img), 0.2) 53 | x = F.leaky_relu(self.conv2_bn(self.conv2(x)), 0.2) 54 | x = F.leaky_relu(self.conv3_bn(self.conv3(x)), 0.2) 55 | #x = F.leaky_relu(self.conv4_bn(self.conv4(x)), 0.2) 56 | x = self.conv4(x) 57 | x = x.mean(0) 58 | return x.view(1) 59 | 60 | def init_weights(self, mean, std): 61 | for m in self._modules: 62 | normal_init(self._modules[m], mean, std) 63 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | 104 | # Pycharm 105 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 106 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 107 | 108 | # User-specific stuff: 109 | .idea/**/workspace.xml 110 | .idea/**/tasks.xml 111 | .idea/dictionaries 112 | 113 | # Sensitive or high-churn files: 114 | .idea/**/dataSources/ 115 | .idea/**/dataSources.ids 116 | .idea/**/dataSources.xml 117 | .idea/**/dataSources.local.xml 118 | .idea/**/sqlDataSources.xml 119 | .idea/**/dynamic.xml 120 | .idea/**/uiDesigner.xml 121 | 122 | # Gradle: 123 | .idea/**/gradle.xml 124 | .idea/**/libraries 125 | 126 | # CMake 127 | cmake-build-debug/ 128 | cmake-build-release/ 129 | 130 | # Mongo Explorer plugin: 131 | .idea/**/mongoSettings.xml 132 | 133 | ## File-based project format: 134 | *.iws 135 | 136 | ## Plugin-specific files: 137 | 138 | # IntelliJ 139 | out/ 140 | 141 | # mpeltonen/sbt-idea plugin 142 | .idea_modules/ 143 | 144 | # JIRA plugin 145 | atlassian-ide-plugin.xml 146 | 147 | # Cursive Clojure plugin 148 | .idea/replstate.xml 149 | 150 | # Crashlytics plugin (for Android Studio and IntelliJ) 151 | com_crashlytics_export_strings.xml 152 | crashlytics.properties 153 | crashlytics-build.properties 154 | fabric.properties 155 | 156 | # Data files 157 | *.data 158 | *.jpg 159 | -------------------------------------------------------------------------------- /trainer/nmcTrainer.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.optim as optim 3 | import torch.utils.data as td 4 | import trainer.classifierTrainer as t 5 | import utils.utils as ut 6 | import trainer.classifierFactory as tF 7 | 8 | class Trainer(): 9 | def __init__(self, args, dataset, classifier_trainer, model, train_iterator, 10 | test_iterator, train_dataset_loader, experiment): 11 | self.args = args 12 | self.dataset = dataset 13 | self.classifier_trainer = classifier_trainer 14 | self.model = model 15 | self.train_iterator = train_iterator 16 | self.test_iterator = test_iterator 17 | self.train_dataset_loader = train_dataset_loader 18 | self.experiment = experiment 19 | 20 | def train(self): 21 | x = [] 22 | y = [] 23 | y1 = [] 24 | train_y = [] 25 | left_over = [] 26 | my_test_factory = tF.ClassifierFactory() 27 | nmc = my_test_factory.get_tester("nmc", self.args.cuda) 28 | 29 | if not self.args.sortby == "none": 30 | print("Sorting by", self.args.sortby) 31 | self.train_dataset_loader.sort_by_importance(self.args.sortby) 32 | 33 | for class_group in range(0, self.dataset.classes, self.args.step_size): 34 | self.classifier_trainer.setup_training() 35 | self.classifier_trainer.increment_classes(class_group) 36 | 37 | epoch = 0 38 | for epoch in range(0, self.args.epochs_class): 39 | self.classifier_trainer.update_lr(epoch) 40 | self.classifier_trainer.train() 41 | if epoch % self.args.log_interval == 0: 42 | print("Train Classifier", 43 | self.classifier_trainer.evaluate(self.train_iterator)) 44 | print("Test Classifier", 45 | self.classifier_trainer.evaluate(self.test_iterator)) 46 | self.classifier_trainer.update_frozen_model() 47 | 48 | nmc.update_means(self.model, self.train_iterator, self.args.cuda, 49 | self.dataset.classes) 50 | 51 | temp_train = nmc.classify(self.model, self.train_iterator, 52 | self.args.cuda, True) 53 | train_y.append(temp_train) 54 | 55 | # Saving confusion matrix 56 | ut.saveConfusionMatrix(int(class_group / self.args.step_size) * 57 | self.args.epochs_class + epoch, 58 | self.experiment.path + "CONFUSION", 59 | self.model, self.args, self.dataset, 60 | self.test_iterator) 61 | 62 | # Computing test error for graphing 63 | test_y = nmc.classify(self.model, self.test_iterator, 64 | self.args.cuda, True) 65 | y.append(test_y) 66 | 67 | print("Train NMC", temp_train) 68 | print("Test NMC", test_y) 69 | 70 | y1.append(self.classifier_trainer.evaluate(self.test_iterator)) 71 | x.append(class_group + self.args.step_size) 72 | 73 | ut.plotAccuracy(self.experiment, x, 74 | [("NMC", y), ("Trained Classifier", y1)], 75 | self.dataset.classes + 1, self.args.name) 76 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Distillation Techniques for Pseudo-rehearsal Based Incremental Learning 2 | This repository contains the Pytorch code for [Distillation Techniques for Pseudo-rehearsal Based Incremental Learning](https://arxiv.org/abs/1807.02799). 3 | 4 | **Abstract:** 5 | The ability to learn from incrementally arriving data is essential for any life-long learning system. However, standard deep neural networks forget the knowledge about the old tasks, a phenomenon called catastrophic forgetting, when trained on incrementally arriving data. We discuss the biases in current Generative Adversarial Networks (GAN) based approaches that learn the classifier by knowledge distillation from previously trained classifiers. These biases cause the trained classifier to perform poorly. We propose an approach to remove these biases by distilling knowledge from the classifier of AC-GAN. Experiments on MNIST and CIFAR10 show that this method is comparable to current state of the art rehearsal based approaches. 6 | 7 | ## Dependencies 8 | The requirements.txt file contains all the dependencies along with their specific versions. These dependencies can be fulfilled by using `pip install -r requirements.txt` 9 | 10 | ## Usage 11 | Use the following interface to run the experiments. The description for each argument is given in the runExperiment.py file. 12 | 13 | ``` 14 | python runExperiment.py [-h] [--batch-size N] [--test-batch-size N] 15 | [--epochs N] [--lr LR] 16 | [--schedule SCHEDULE [SCHEDULE ...]] 17 | [--gammas GAMMAS [GAMMAS ...]] [--momentum M] 18 | [--no-cuda] [--no-distill] [--no-random] 19 | [--no-herding] [--seed S] [--log-interval N] 20 | [--model-type MODEL_TYPE] [--name NAME] 21 | [--sortby SORTBY] [--decay DECAY] 22 | [--step-size STEP_SIZE] 23 | [--memory-budget MEMORY_BUDGET] 24 | [--epochs-class EPOCHS_CLASS] [--dataset DATASET] 25 | [--no-upsampling] [--process PROCESS] 26 | [--gan-epochs GAN_EPOCHS [GAN_EPOCHS ...]] 27 | [--gan-d GAN_D] [--gan-lr GAN_LR] 28 | [--gan-batch-size GAN_BATCH_SIZE] 29 | [--gan-num-examples GAN_NUM_EXAMPLES] 30 | [--gan-schedule GAN_SCHEDULE [GAN_SCHEDULE ...]] 31 | [--gan-gammas GAN_GAMMAS [GAN_GAMMAS ...]] 32 | [--persist-gan] 33 | [--gan-img-save-interval GAN_IMG_SAVE_INTERVAL] 34 | [--d-iter D_ITER] [--ckpt-interval CKPT_INTERVAL] 35 | [--load-g-ckpt LOAD_G_CKPT] [--T T] [--save-g-ckpt] 36 | [--gan-save-classes GAN_SAVE_CLASSES] 37 | [--label-smoothing] [--minibatch-discrimination] 38 | [--ideal-nmc] [--optimize-features] 39 | [--optimize-feat-epochs OPTIMIZE_FEAT_EPOCHS] 40 | [--optimize-feat-lr OPTIMIZE_FEAT_LR] 41 | [--joint-gan-obj] [--joint-gan-alpha JOINT_GAN_ALPHA] 42 | [--ac-distill] [--filter-using-disc] 43 | [--filter-val FILTER_VAL] [--disc-eval] 44 | ``` 45 | 46 | ## Paper citation 47 | If you used this code for your experiments or found it helpful, consider citing the following paper: 48 | ``` 49 | @article{2018arXiv180702799S, 50 | author = {{Shah}, H. and {Javed}, K. and {Shafait}, F.}, 51 | title = "{Distillation Techniques for Pseudo-rehearsal Based Incremental Learning}", 52 | journal = {ArXiv e-prints}, 53 | year = 2018, 54 | } 55 | ``` 56 | -------------------------------------------------------------------------------- /model/ACGAN.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from .gan_utils import normal_init 5 | 6 | class Generator(nn.Module): 7 | def __init__(self, d=384, c=1, num_classes=10, nz=100): 8 | super(Generator, self).__init__() 9 | self.d = d 10 | self.nz = nz 11 | self.num_classes = num_classes 12 | self.fc1 = nn.Linear(nz+num_classes, d) 13 | 14 | self.ct2 = nn.ConvTranspose2d(d, d//2, 4, 1, 0, bias=False) 15 | self.ct2_bn = nn.BatchNorm2d(d//2) 16 | 17 | self.ct3 = nn.ConvTranspose2d(d//2, d//4, 4, 2, 1, bias=False) 18 | self.ct3_bn = nn.BatchNorm2d(d//4) 19 | 20 | self.ct4 = nn.ConvTranspose2d(d//4, d//8, 4, 2, 1, bias=False) 21 | self.ct4_bn = nn.BatchNorm2d(d//8) 22 | 23 | self.ct5 = nn.ConvTranspose2d(d//8, c, 4, 2, 1, bias=False) 24 | 25 | print(self) 26 | 27 | def forward(self, input): 28 | x = input.view(-1, self.nz + self.num_classes) 29 | x = self.fc1(x) 30 | x = x.view(-1, self.d, 1, 1) 31 | x = F.relu(self.ct2_bn(self.ct2(x))) 32 | x = F.relu(self.ct3_bn(self.ct3(x))) 33 | x = F.relu(self.ct4_bn(self.ct4(x))) 34 | x = F.tanh(self.ct5(x)) 35 | return x 36 | 37 | def init_weights(self, mean, std): 38 | for m in self._modules: 39 | normal_init(self._modules[m], mean, std, False) 40 | 41 | class Discriminator(nn.Module): 42 | def __init__(self, d=16, c=1, num_classes=10): 43 | super(Discriminator, self).__init__() 44 | self.d = d 45 | self.conv1 = nn.Conv2d(c, d, 3, 2, 1, bias=False) 46 | self.Drop1 = nn.Dropout(0.5) 47 | 48 | self.conv2 = nn.Conv2d(d, d*2, 3, 1, 1, bias=False) 49 | self.conv2_bn = nn.BatchNorm2d(d*2) 50 | self.Drop2 = nn.Dropout(0.5) 51 | 52 | self.conv3 = nn.Conv2d(d*2, d*4, 3, 2, 1, bias=False) 53 | self.conv3_bn = nn.BatchNorm2d(d*4) 54 | self.Drop3 = nn.Dropout(0.5) 55 | 56 | self.conv4 = nn.Conv2d(d*4, d*8, 3, 1, 1, bias=False) 57 | self.conv4_bn = nn.BatchNorm2d(d*8) 58 | self.Drop4 = nn.Dropout(0.5) 59 | 60 | self.conv5 = nn.Conv2d(d*8, d*16, 3, 2, 1, bias=False) 61 | self.conv5_bn = nn.BatchNorm2d(d*16) 62 | self.Drop5 = nn.Dropout(0.5) 63 | 64 | self.conv6 = nn.Conv2d(d*16, d*32, 3, 1, 1, bias=False) 65 | self.conv6_bn = nn.BatchNorm2d(d*32) 66 | self.Drop6 = nn.Dropout(0.5) 67 | 68 | self.fc_dis = nn.Linear(4*4*d*32, 1) 69 | self.fc_aux = nn.Linear(4*4*d*32, num_classes) 70 | 71 | self.softmax = nn.Softmax() 72 | self.sigmoid = nn.Sigmoid() 73 | 74 | print(self) 75 | 76 | def forward(self, img, get_features=False, T=1): 77 | x = self.Drop1(F.leaky_relu(self.conv1(img), 0.2)) 78 | x = self.Drop2(F.leaky_relu(self.conv2_bn(self.conv2(x)), 0.2)) 79 | x = self.Drop3(F.leaky_relu(self.conv3_bn(self.conv3(x)), 0.2)) 80 | x = self.Drop4(F.leaky_relu(self.conv4_bn(self.conv4(x)), 0.2)) 81 | x = self.Drop5(F.leaky_relu(self.conv5_bn(self.conv5(x)), 0.2)) 82 | x = self.Drop6(F.leaky_relu(self.conv6_bn(self.conv6(x)), 0.2)) 83 | 84 | #When d=16, d*32=512, TODO 85 | x = x.view(-1, 4*4*self.d*32) 86 | fc_aux = self.fc_aux(x) 87 | if get_features: 88 | return fc_aux 89 | fc_dis = self.fc_dis(x) 90 | liklihood_correct_class = self.softmax(fc_aux/T) 91 | liklihood_real_img = self.sigmoid(fc_dis).view(-1,1).squeeze(1) 92 | return liklihood_real_img, liklihood_correct_class 93 | 94 | def init_weights(self, mean, std): 95 | for m in self._modules: 96 | normal_init(self._modules[m], mean, std, False) 97 | -------------------------------------------------------------------------------- /trainer/classifierFactory.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | from torch.autograd import Variable 4 | from torchnet.meter import confusionmeter 5 | 6 | 7 | class ClassifierFactory(): 8 | def __init__(self): 9 | pass 10 | 11 | def get_tester(self, test_type="nmc", cuda=True): 12 | if test_type == "nmc": 13 | return NearestMeanClassifier(cuda) 14 | 15 | 16 | class NearestMeanClassifier(): 17 | def __init__(self, cuda): 18 | self.cuda = cuda 19 | self.means = np.zeros((100, 64)) 20 | self.total_features = np.zeros((100, 1)) 21 | 22 | def classify(self, model, test_loader, cuda, verbose=False): 23 | model.eval() 24 | test_loss = 0 25 | correct = 0 26 | c_matrix = confusionmeter.ConfusionMeter(self.classes, True) 27 | 28 | for data, target in test_loader: 29 | if cuda: 30 | data, target = data.cuda(), target.cuda() 31 | self.means = self.means.cuda() 32 | data, target = Variable(data, volatile=True), Variable(target) 33 | output = model(data, True).unsqueeze(1) 34 | result = (output.data - self.means.float()) 35 | result = torch.norm(result, 2, 2) 36 | 37 | _, pred = torch.min(result, 1) 38 | 39 | correct += pred.eq(target.data.view_as(pred)).cpu().sum() 40 | c_matrix.add(pred, target.data.view_as(pred)) 41 | # 0/0 42 | test_loss /= len(test_loader.dataset) 43 | 44 | return 100. * correct / len(test_loader.dataset) 45 | 46 | def update_means(self, model, train_loader, cuda, classes=100, old_classes=None, is_cond=False): 47 | # Set the mean to zero 48 | model.eval() 49 | # TODO Add this to init 50 | if classes != 100: 51 | self.means = np.zeros((classes, 64)) 52 | self.means *= 0 53 | self.classes = classes 54 | self.means = np.zeros((classes, 64)) 55 | self.total_features = np.zeros((classes, 1)) + 1 56 | print("Computing means") 57 | if (old_classes != None and is_cond == False): 58 | print("Fixing labels too") 59 | # Iterate over all train dataset 60 | for batch_id, (data, target) in enumerate(train_loader): 61 | # Get features for a minibactch 62 | if cuda: 63 | data = data.cuda() 64 | features = model.forward(Variable(data), True) 65 | 66 | #TODO make efficient and calculate this once 67 | #TODO make sure that the gradients and not stored 68 | if (old_classes != None and is_cond == False): 69 | old_targets = torch.zeros(target.shape[0]).byte() 70 | for klass in old_classes: 71 | old_targets += (target == klass) 72 | new_targets = (old_targets == 0) 73 | output = model.forward(Variable(data)) 74 | predictions = output.data.max(1, keepdim=True)[1].squeeze() 75 | target = target*new_targets.long() + predictions.cpu()*old_targets.long() 76 | 77 | # Convert result to a numpy array 78 | features_np = features.data.cpu().numpy() 79 | # Accumulate the results in the means array 80 | np.add.at(self.means, target, features_np) 81 | # Keep track of how many instances of a class have been seen. This should be an array with all elements = classSize 82 | np.add.at(self.total_features, target, 1) 83 | 84 | # Divide the means array with total number of instaces to get the average 85 | self.means = self.means / self.total_features 86 | 87 | self.means = torch.from_numpy(self.means).unsqueeze(0) 88 | print("Mean vectors computed") 89 | # Return 90 | return 91 | -------------------------------------------------------------------------------- /dataHandler/dataset.py: -------------------------------------------------------------------------------- 1 | import torchvision 2 | from torchvision import datasets, transforms 3 | 4 | 5 | # To incdude a new Dataset, inherit from Dataset and add all the Dataset specific parameters here. 6 | # Goal : Remove any data specific parameters from the rest of the code 7 | 8 | class Dataset(): 9 | ''' 10 | Base class to reprenent a Dataset 11 | ''' 12 | 13 | def __init__(self, classes, name, labels_per_class_train, labels_per_class_test): 14 | self.classes = classes 15 | self.name = name 16 | self.train_data = None 17 | self.test_data = None 18 | self.labels_per_class_train = labels_per_class_train 19 | self.labels_per_class_test = labels_per_class_test 20 | 21 | 22 | class MNIST(Dataset): 23 | ''' 24 | Class to include MNIST specific details 25 | ''' 26 | 27 | def __init__(self): 28 | super().__init__(10, "MNIST", 6000, 1000) 29 | 30 | self.train_transform = transforms.Compose( 31 | [transforms.ToTensor(), 32 | transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]) 33 | 34 | self.alt_transform = None 35 | 36 | self.test_transform = transforms.Compose([ 37 | transforms.ToTensor(), 38 | transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]) 39 | 40 | self.train_data = datasets.MNIST("data", train=True, transform=self.train_transform, download=True) 41 | 42 | self.test_data = datasets.MNIST("data", train=False, transform=self.test_transform, download=True) 43 | 44 | 45 | class CIFAR100(Dataset): 46 | def __init__(self): 47 | super().__init__(100, "CIFAR100", 500, 100) 48 | 49 | mean = [x / 255 for x in [125.3, 123.0, 113.9]] 50 | std = [x / 255 for x in [63.0, 62.1, 66.7]] 51 | 52 | self.train_transform = transforms.Compose( 53 | #[torchvision.transforms.ColorJitter(0.5, 0.5, 0.5, 0.5), 54 | #transforms.RandomCrop(32, padding=6), torchvision.transforms.RandomRotation((-10, 10)), 55 | [transforms.ToTensor(), 56 | transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]) 57 | 58 | self.test_transform = transforms.Compose( 59 | [transforms.ToTensor(), transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]) 60 | 61 | self.train_data = datasets.CIFAR100("data", train=True, transform=self.train_transform, download=True) 62 | 63 | self.test_data = datasets.CIFAR100("data", train=False, transform=self.test_transform, download=True) 64 | 65 | class CIFAR10(Dataset): 66 | def __init__(self): 67 | super().__init__(10, "CIFAR10", 5000, 1000) 68 | 69 | #self.train_transform = transforms.Compose( 70 | # #[torchvision.transforms.ColorJitter(0.5, 0.5, 0.5, 0.5), 71 | # #transforms.RandomCrop(32, padding=6), torchvision.transforms.RandomRotation((-10, 10)), 72 | # [transforms.ToTensor(), 73 | # transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]) 74 | 75 | # self.test_transform = transforms.Compose( 76 | # [transforms.ToTensor(), transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]) 77 | 78 | self.train_transform = transforms.Compose( 79 | [transforms.RandomHorizontalFlip(), 80 | transforms.RandomCrop(32, padding=4), 81 | transforms.ToTensor(), 82 | transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]) 83 | 84 | self.alt_transform = transforms.Compose( 85 | [transforms.ToTensor(), 86 | transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]) 87 | 88 | self.test_transform = transforms.Compose( 89 | [transforms.ToTensor(), 90 | transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]) 91 | 92 | 93 | self.train_data = datasets.CIFAR10("data", train=True, transform=self.train_transform, download=True) 94 | 95 | self.test_data = datasets.CIFAR10("data", train=False, transform=self.test_transform, download=True) 96 | -------------------------------------------------------------------------------- /model/modelFactory.py: -------------------------------------------------------------------------------- 1 | import model.densenet as dn 2 | import model.resnet32 as res 3 | import model.testModel as tm 4 | import model.cDCGAN as cdcgan 5 | import model.DCGAN as dcgan 6 | import model.WGAN as wgan 7 | import model.ACGAN as acgan 8 | 9 | class ModelFactory(): 10 | def __init__(self): 11 | pass 12 | def get_model(self, model_type, dataset="CIFAR100", use_mbd=False, d=64): 13 | if model_type=="densenet": 14 | if dataset=="MNIST": 15 | print ("MNIST dataset not supported in this model. Try resnet20 or 32") 16 | assert(False) 17 | return dn.DenseNet(growth_rate=12, depth=40, reduction=0.5, 18 | bottleneck=True, n_classes=100) 19 | 20 | elif model_type=="resnet32": 21 | if dataset=="MNIST": 22 | return res.resnet32mnist(10) 23 | elif dataset=="CIFAR10": 24 | return res.resnet32(10) 25 | return res.resnet32(100) 26 | 27 | 28 | elif model_type=="resnet20": 29 | if dataset=="MNIST": 30 | return res.resnet20mnist(10) 31 | return res.resnet20(100) 32 | 33 | 34 | elif model_type=="resnet44": 35 | if dataset == "MNIST": 36 | print("MNIST dataset not supported in this model. Try resnet20 or 32") 37 | assert (False) 38 | return res.resnet44(100) 39 | 40 | 41 | elif model_type=="test": 42 | if dataset=="MNIST": 43 | print ("MNIST dataset not supported in this model. Try resnet20 or 32") 44 | assert(False) 45 | return tm.Net(100) 46 | 47 | elif model_type=="cdcgan": 48 | if dataset=="CIFAR100": 49 | G = cdcgan.Generator(d, 3, 100) 50 | D = cdcgan.Discriminator(d, 3, 100, use_mbd) 51 | elif dataset=="CIFAR10": 52 | G = cdcgan.Generator(d, 3, 10) 53 | D = cdcgan.Discriminator(d, 3, 10, use_mbd) 54 | else: 55 | G = cdcgan.Generator(d) 56 | D = cdcgan.Discriminator(d, 1, 10, use_mbd) 57 | G.init_weights(mean=0.0, std=0.02) 58 | D.init_weights(mean=0.0, std=0.02) 59 | return G, D 60 | 61 | elif model_type=="dcgan": 62 | if dataset=="CIFAR100" or dataset=="CIFAR10": 63 | G = dcgan.Generator(d, 3) 64 | D = dcgan.Discriminator(d, 3) 65 | else: 66 | G = dcgan.Generator(d) 67 | D = dcgan.Discriminator(d) 68 | G.init_weights(mean=0.0, std=0.02) 69 | D.init_weights(mean=0.0, std=0.02) 70 | return G, D 71 | 72 | elif model_type=="wgan": 73 | if dataset=="CIFAR100" or dataset=="CIFAR10": 74 | G = wgan.Generator(d, 3) 75 | D = wgan.Discriminator(d, 3) 76 | else: 77 | G = wgan.Generator(d) 78 | D = wgan.Discriminator(d) 79 | G.init_weights(mean=0.0, std=0.02) 80 | D.init_weights(mean=0.0, std=0.02) 81 | return G, D 82 | 83 | elif model_type=="acgan": 84 | num_classes = 100 if dataset=="CIFAR100" else 10 85 | gen_d = 384 86 | if d < 16: 87 | print("[!!!] d<16, You sure??") 88 | assert False 89 | if d == 32: 90 | gen_d = 768 91 | if dataset=="CIFAR100" or dataset=="CIFAR10": 92 | G = acgan.Generator(gen_d, 3, num_classes) 93 | D = acgan.Discriminator(d, 3, num_classes) 94 | else: 95 | G = acgan.Generator(gen_d, 1, num_classes) 96 | D = acgan.Discriminator(d, 1, num_classes) 97 | G.init_weights(mean=0.0, std=0.02) 98 | D.init_weights(mean=0.0, std=0.02) 99 | return G, D 100 | 101 | else: 102 | print ("Unsupported model; either implement the model in model/ModelFactory or choose a different model") 103 | assert(False) 104 | -------------------------------------------------------------------------------- /model/cDCGAN.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from .gan_utils import normal_init 5 | 6 | class View(nn.Module): 7 | def __init__(self, *shape): 8 | super(View, self).__init__() 9 | self.shape = shape 10 | def forward(self, input): 11 | return input.view(*shape) 12 | 13 | class Generator(nn.Module): 14 | ''' 15 | d = base multiplier 16 | c = number of channels in the image 17 | l = number of unique classes in the dataset 18 | ''' 19 | def __init__(self, d=128, c=1, l=10): 20 | super(Generator, self).__init__() 21 | #ConvTranspose2d(in_channels, out_channels, kernel_size, stride=1, padding=0) 22 | self.ct1_noise = nn.ConvTranspose2d(100, d*2, 4, 1, 0) 23 | self.ct1_noise_bn = nn.BatchNorm2d(d*2) 24 | self.ct1_label = nn.ConvTranspose2d(l, d*2, 4, 1, 0) 25 | self.ct1_label_bn = nn.BatchNorm2d(d*2) 26 | self.ct2 = nn.ConvTranspose2d(d*4, d*2, 4, 2, 1) 27 | self.ct2_bn = nn.BatchNorm2d(d*2) 28 | self.ct3 = nn.ConvTranspose2d(d*2, d, 4, 2, 1) 29 | self.ct3_bn = nn.BatchNorm2d(d) 30 | self.ct4 = nn.ConvTranspose2d(d, c, 4, 2, 1) 31 | 32 | def forward(self, noise, label): 33 | x = F.relu(self.ct1_noise_bn(self.ct1_noise(noise))) 34 | y = F.relu(self.ct1_label_bn(self.ct1_label(label))) 35 | x = torch.cat([x, y], 1) 36 | x = F.relu(self.ct2_bn(self.ct2(x))) 37 | x = F.relu(self.ct3_bn(self.ct3(x))) 38 | x = F.tanh(self.ct4(x)) 39 | return x 40 | 41 | def init_weights(self, mean, std): 42 | for m in self._modules: 43 | normal_init(self._modules[m], mean, std) 44 | 45 | 46 | class Discriminator(nn.Module): 47 | ''' 48 | d = base multiplier 49 | c = number of channels in the image 50 | l = number of unique classes in the dataset 51 | ''' 52 | def __init__(self, d=128, c=1, l=10, use_mbd=True, mbd_num=128, mbd_dim=3): 53 | super(Discriminator, self).__init__() 54 | self.use_mbd = use_mbd 55 | self.mbd_num = mbd_num 56 | self.mbd_dim = mbd_dim 57 | 58 | self.conv1_img = nn.Conv2d(c, d//2, 4, 2, 1) 59 | self.conv1_label = nn.Conv2d(l, d//2, 4, 2, 1) 60 | self.conv2 = nn.Conv2d(d, d*2, 4, 2, 1) 61 | self.conv2_bn = nn.BatchNorm2d(d*2) 62 | self.conv3 = nn.Conv2d(d*2, d*4, 4, 2, 1) 63 | self.conv3_bn = nn.BatchNorm2d(d*4) 64 | # Use linear layer to produce a matrix of shape (B, mbd_num*mbd_dim) 65 | # The input to this layer is of shape [B, d*4, 4, 4], we reshape it 66 | if self.use_mbd: 67 | self.mbd = nn.Linear(d*4*4*4, mbd_num * mbd_dim) 68 | self.conv4 = nn.Conv2d(d * 4 + 8, 1, 4, 1, 0) 69 | else: 70 | self.conv4 = nn.Conv2d(d * 4, 1, 4, 1, 0) 71 | 72 | def forward(self, img, label): 73 | x = F.leaky_relu(self.conv1_img(img), 0.2) 74 | y = F.leaky_relu(self.conv1_label(label), 0.2) 75 | x = torch.cat([x, y], 1) 76 | x = F.leaky_relu(self.conv2_bn(self.conv2(x)), 0.2) 77 | x = F.leaky_relu(self.conv3_bn(self.conv3(x)), 0.2) 78 | if self.use_mbd: 79 | # Reshape for linear layer 80 | x = x.view(-1, 128 * 4 * 4 * 4) 81 | # Use the linear layer 82 | mbd = self.mbd(x) 83 | # Calculate minibatch features and concat them 84 | x = self.minibatch_discrimination(mbd, x) 85 | # Make it compatible for convolution layer 86 | # 520 = 128 * 4 + 8 87 | x = x.view(-1, 520, 4, 4) 88 | x = self.conv4(x) 89 | x = F.sigmoid(x) 90 | return x 91 | 92 | def init_weights(self, mean, std): 93 | for m in self._modules: 94 | normal_init(self._modules[m], mean, std) 95 | 96 | def minibatch_discrimination(self, x, input_to_layer): 97 | activation = x.view(-1, self.mbd_num, self.mbd_dim) 98 | diffs = activation.unsqueeze(3) - activation.permute(1,2,0).unsqueeze(0) 99 | abs_diff = torch.abs(diffs).sum(2) 100 | mb_feats = torch.exp(-abs_diff).sum(2) 101 | return torch.cat([input_to_layer, mb_feats], 1) 102 | -------------------------------------------------------------------------------- /model/densenet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | import torch.nn as nn 4 | import torch.optim as optim 5 | 6 | import torch.nn.functional as F 7 | from torch.autograd import Variable 8 | 9 | import torchvision.datasets as dset 10 | import torchvision.transforms as transforms 11 | from torch.utils.data import DataLoader 12 | 13 | import torchvision.models as models 14 | 15 | import sys 16 | import math 17 | 18 | class Bottleneck(nn.Module): 19 | def __init__(self, n_channels, growth_rate): 20 | super(Bottleneck, self).__init__() 21 | inter_channels = 4*growth_rate 22 | self.bn1 = nn.BatchNorm2d(n_channels) 23 | self.conv1 = nn.Conv2d(n_channels, inter_channels, kernel_size=1, 24 | bias=False) 25 | self.bn2 = nn.BatchNorm2d(inter_channels) 26 | self.conv2 = nn.Conv2d(inter_channels, growth_rate, kernel_size=3, 27 | padding=1, bias=False) 28 | 29 | def forward(self, x): 30 | out = self.conv1(F.relu(self.bn1(x))) 31 | out = self.conv2(F.relu(self.bn2(out))) 32 | out = torch.cat((x, out), 1) 33 | return out 34 | 35 | class SingleLayer(nn.Module): 36 | def __init__(self, n_channels, growth_rate): 37 | super(SingleLayer, self).__init__() 38 | self.bn1 = nn.BatchNorm2d(n_channels) 39 | self.conv1 = nn.Conv2d(n_channels, growth_rate, kernel_size=3, 40 | padding=1, bias=False) 41 | 42 | def forward(self, x): 43 | out = self.conv1(F.relu(self.bn1(x))) 44 | out = torch.cat((x, out), 1) 45 | return out 46 | 47 | class Transition(nn.Module): 48 | def __init__(self, n_channels, n_out_channels): 49 | super(Transition, self).__init__() 50 | self.bn1 = nn.BatchNorm2d(n_channels) 51 | self.conv1 = nn.Conv2d(n_channels, n_out_channels, kernel_size=1, 52 | bias=False) 53 | 54 | def forward(self, x): 55 | out = self.conv1(F.relu(self.bn1(x))) 56 | out = F.avg_pool2d(out, 2) 57 | return out 58 | 59 | 60 | class DenseNet(nn.Module): 61 | def __init__(self, growth_rate, depth, reduction, n_classes, bottleneck): 62 | super(DenseNet, self).__init__() 63 | 64 | n_dense_blocks = (depth-4) // 3 65 | if bottleneck: 66 | n_dense_blocks //= 2 67 | 68 | n_channels = 2*growth_rate 69 | self.conv1 = nn.Conv2d(3, n_channels, kernel_size=3, padding=1, 70 | bias=False) 71 | self.dense1 = self._make_dense(n_channels, growth_rate, n_dense_blocks, bottleneck) 72 | n_channels += n_dense_blocks*growth_rate 73 | n_out_channels = int(math.floor(n_channels*reduction)) 74 | self.trans1 = Transition(n_channels, n_out_channels) 75 | 76 | n_channels = n_out_channels 77 | self.dense2 = self._make_dense(n_channels, growth_rate, n_dense_blocks, bottleneck) 78 | n_channels += n_dense_blocks*growth_rate 79 | n_out_channels = int(math.floor(n_channels*reduction)) 80 | self.trans2 = Transition(n_channels, n_out_channels) 81 | 82 | n_channels = n_out_channels 83 | self.dense3 = self._make_dense(n_channels, growth_rate, n_dense_blocks, bottleneck) 84 | n_channels += n_dense_blocks*growth_rate 85 | 86 | self.bn1 = nn.BatchNorm2d(n_channels) 87 | self.fc = nn.Linear(n_channels, n_classes) 88 | 89 | for m in self.modules(): 90 | if isinstance(m, nn.Conv2d): 91 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 92 | m.weight.data.normal_(0, math.sqrt(2. / n)) 93 | elif isinstance(m, nn.BatchNorm2d): 94 | m.weight.data.fill_(1) 95 | m.bias.data.zero_() 96 | elif isinstance(m, nn.Linear): 97 | m.bias.data.zero_() 98 | 99 | def _make_dense(self, n_channels, growth_rate, n_dense_blocks, bottleneck): 100 | layers = [] 101 | for i in range(int(n_dense_blocks)): 102 | if bottleneck: 103 | layers.append(Bottleneck(n_channels, growth_rate)) 104 | else: 105 | layers.append(SingleLayer(n_channels, growth_rate)) 106 | n_channels += growth_rate 107 | return nn.Sequential(*layers) 108 | 109 | def forward(self, x, feature=False): 110 | out = self.conv1(x) 111 | out = self.trans1(self.dense1(out)) 112 | out = self.trans2(self.dense2(out)) 113 | out = self.dense3(out) 114 | out = torch.squeeze(F.avg_pool2d(F.relu(self.bn1(out)), 8)) 115 | if feature: 116 | return out / torch.norm(out, 2, 1).unsqueeze(1) 117 | out = F.log_softmax(self.fc(out)) 118 | return out -------------------------------------------------------------------------------- /plotter/plotter.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import matplotlib 3 | import numpy as np 4 | 5 | plt.switch_backend('agg') 6 | 7 | MEDIUM_SIZE = 16 8 | 9 | font = {'family' : 'sans-serif', 10 | 'weight':'bold'} 11 | 12 | matplotlib.rc('xtick', labelsize=MEDIUM_SIZE) 13 | matplotlib.rc('ytick', labelsize=MEDIUM_SIZE) 14 | plt.rc('axes', labelsize=MEDIUM_SIZE) # fontsize of the x and y labels 15 | 16 | # matplotlib.rc('font', **font) 17 | from matplotlib import rcParams 18 | rcParams.update({'figure.autolayout': True}) 19 | 20 | 21 | class Plotter(): 22 | def __init__(self): 23 | import itertools 24 | # plt.figure(figsize=(12, 9)) 25 | self.marker = itertools.cycle(('o', '+', "v", "^", "8",'.', '*')) 26 | self.handles=[] 27 | self.lines = itertools.cycle(('--', '-.', '-', ':')) 28 | def plot(self,x,y, xLabel="Number of Classes",yLabel = "Accuracy %", legend="none",title=None, error=None): 29 | self.x = x 30 | self.y = y 31 | plt.grid(color='0.89', linestyle='--', linewidth=1.0) 32 | if error is None: 33 | l, = plt.plot(x,y,linestyle=next(self.lines), marker=next(self.marker), label=legend, linewidth=2.0) 34 | else: 35 | l, = plt.plot(x,y,linestyle=next(self.lines), marker=next(self.marker), label=legend, linewidth=2.0) 36 | y, error = np.array(y), np.array(error) 37 | plt.fill_between(x, y-error, y+error, alpha=0.12) 38 | #l = plt.errorbar(x, y, yerr=error, capsize=4.0, capthick=2.0, linestyle=next(self.lines), marker=next(self.marker), label=legend, linewidth=2.0) 39 | 40 | self.handles.append(l) 41 | self.x_label = xLabel 42 | self.y_label = yLabel 43 | if title is not None: 44 | plt.title(title) 45 | 46 | def save_fig(self, path, xticks=105): 47 | plt.legend(handles=self.handles) 48 | plt.ylim((0, 100+1.2) ) 49 | plt.xlim((0,xticks+.2)) 50 | #plt.ylabel(self.y_label) 51 | plt.xlabel(self.x_label) 52 | plt.yticks(list(range(10,105,10))) 53 | plt.xticks(list(range(0, xticks+1, int(xticks/10)))) 54 | plt.savefig(path+".jpg") 55 | plt.savefig(path+".pdf", format='pdf', dpi=600) 56 | plt.gcf().clear() 57 | 58 | def save_fig2(self, path, xticks=105): 59 | plt.legend(handles=self.handles) 60 | plt.xlabel("Memory Budget") 61 | plt.ylabel("Average Incremental Accuracy") 62 | plt.savefig(path+".jpg") 63 | plt.gcf().clear() 64 | 65 | def plotMatrix(self, epoch, path, img): 66 | import matplotlib.pyplot as plt 67 | plt.imshow(img, cmap='jet', interpolation='nearest') 68 | plt.colorbar() 69 | plt.savefig(path + str(epoch) + ".svg", format='svg', dpi=1200) 70 | plt.gcf().clear() 71 | 72 | def saveImage(self, img, path, epoch): 73 | from PIL import Image 74 | im = Image.fromarray(img) 75 | im.save(path + str(epoch) + ".jpg") 76 | 77 | def plot_histogram(self, subplots, embeddings, range_val, colors): 78 | p1, p2, p3 = subplots 79 | bins = np.linspace(range_val[0], range_val[1], 100) 80 | p1.hist(embeddings[0], bins, alpha=0.5, edgecolor="k", color=colors[0]) 81 | p2.hist(embeddings[1], bins, alpha=0.5, edgecolor="k", color=colors[1]) 82 | p3.hist(embeddings[0] - embeddings[1], bins, alpha=0.5, edgecolor="k", color=colors[2]) 83 | plt.xlabel('Value') 84 | plt.ylabel('Frequency') 85 | 86 | def plot_chart(self, subplots, embeddings, range_val, colors): 87 | p1, p2, p3 = subplots 88 | p1.plot(embeddings[0], color=colors[0]) 89 | p2.plot(embeddings[1], color=colors[1]) 90 | p3.plot(np.abs(embeddings[0] - embeddings[1]), color=colors[2]) 91 | plt.xlabel('Logit') 92 | plt.ylabel('Value') 93 | 94 | def save_embedding_plot(self, subplots, labels, plot_name, experiment): 95 | p1, p2, p3 = subplots 96 | p1.legend([labels[0]], loc="upper right") 97 | p2.legend([labels[1]], loc="upper right") 98 | p3.legend(["Difference"], loc="upper right") 99 | plt.savefig(experiment.path + "embedding_" + plot_name + ".jpg") 100 | plt.gcf().clear() 101 | 102 | def plot_embeddings(self, embeddings, labels, range_val, experiment, plot_name=""): 103 | """ 104 | Plots Histogram and Chart of two embeddings and their difference 105 | Parameters: 106 | embeddings: Array containing 2x 1D vector of embeddings 107 | labels: Array containing labels of the corresponding embeddings 108 | range_val: Range of logit values for the histogram 109 | """ 110 | colors = ['b','g', 'r'] 111 | #Plot histogram 112 | f, (subplots) = plt.subplots(3, 1, sharex=True, sharey=True) 113 | self.plot_histogram(subplots, embeddings, range_val, colors) 114 | self.save_embedding_plot(subplots, labels, "hist_" + plot_name, experiment) 115 | #Plot chart 116 | f, (subplots) = plt.subplots(3, 1, sharex=True, sharey=True) 117 | self.plot_chart(subplots, embeddings, range_val, colors) 118 | self.save_embedding_plot(subplots, labels, "chart_" + plot_name, experiment) 119 | 120 | if __name__=="__main__": 121 | pl = Plotter() 122 | pl.plot([1,2,3,4], [2,3,6,2]) 123 | pl.save_fig("test.jpg") 124 | -------------------------------------------------------------------------------- /utils/utils.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import numpy as np 3 | import plotter.plotter as pl 4 | import matplotlib.pyplot as plt 5 | import torch 6 | import torch.nn.functional as F 7 | from torch.autograd import Variable 8 | from torchnet.meter import confusionmeter 9 | 10 | #TODO CONFIGURE FOR CIFAR 11 | def normalize_images(images, mean=[0.5,0.5,0.5], std=[0.5,0.5,0.5]): 12 | #images.sub_(mean[0]).div_(std[0]) 13 | #No need to return but the var is needed 14 | return images 15 | 16 | def compute_acc(preds, labels): 17 | ''' 18 | Computes the classification acc 19 | ''' 20 | correct = 0 21 | preds_ = preds.data.max(1)[1] 22 | correct = preds_.eq(labels.data).cpu().sum() 23 | acc = float(correct) / float(len(labels.data)) * 100.0 24 | return acc 25 | 26 | def get_new_iterator(cuda, train_loader, new_batch_size): 27 | kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {} 28 | train_iterator = torch.utils.data.DataLoader(train_loader, 29 | batch_size=new_batch_size, 30 | shuffle=True, **kwargs) 31 | return train_iterator 32 | 33 | def resizeImage(img, factor): 34 | ''' 35 | 36 | :param img: 37 | :param factor: 38 | :return: 39 | ''' 40 | img2 = np.zeros(np.array(img.shape) * factor) 41 | 42 | for a in range(0, img.shape[0]): 43 | for b in range(0, img.shape[1]): 44 | img2[a * factor:(a + 1) * factor, b * factor:(b + 1) * factor] = img[a, b] 45 | return img2 46 | 47 | 48 | 49 | def saveConfusionMatrix(epoch, path, model, args, dataset, test_loader): 50 | model.eval() 51 | test_loss = 0 52 | correct = 0 53 | cMatrix = confusionmeter.ConfusionMeter(dataset.classes, True) 54 | 55 | for data, target in test_loader: 56 | if args.cuda: 57 | data, target = data.cuda(), target.cuda() 58 | data, target = Variable(data, volatile=True), Variable(target) 59 | output = model(data) 60 | test_loss += F.nll_loss(output, target, size_average=False).data[0] # sum up batch loss 61 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 62 | correct += pred.eq(target.data.view_as(pred)).cpu().sum() 63 | if epoch > 0: 64 | cMatrix.add(pred.squeeze(), target.data.view_as(pred).squeeze()) 65 | 66 | test_loss /= len(test_loader.dataset) 67 | img = cMatrix.value() 68 | import matplotlib.pyplot as plt 69 | 70 | plt.imshow(img, cmap='plasma', interpolation='nearest') 71 | #plt.colorbar() 72 | plt.savefig(path + str(epoch) + ".jpg") 73 | plt.savefig(path + str(epoch) + ".eps", format='eps') 74 | plt.gcf().clear() 75 | return 100. * correct / len(test_loader.dataset) 76 | 77 | def get_confusion_matrix_nmc(path, model, loader, size, args, means, epoch): 78 | model.eval() 79 | test_loss = 0 80 | correct = 0 81 | # Get the confusion matrix object 82 | cMatrix = confusionmeter.ConfusionMeter(size, True) 83 | 84 | for data, target in loader: 85 | if args.cuda: 86 | data, target = data.cuda(), target.cuda() 87 | means = means.cuda() 88 | data, target = Variable(data, volatile=True), Variable(target) 89 | output = model(data, True).unsqueeze(1) 90 | result = (output.data - means.float()) 91 | 92 | result = torch.norm(result, 2, 2) 93 | # NMC for classification 94 | _, pred = torch.min(result, 1) 95 | # Evaluate results 96 | correct += pred.eq(target.data.view_as(pred)).cpu().sum() 97 | # Add the results in appropriate places in the matrix. 98 | cMatrix.add(pred, target.data.view_as(pred)) 99 | 100 | test_loss /= len(loader.dataset) 101 | img = cMatrix.value() 102 | import matplotlib.pyplot as plt 103 | 104 | plt.imshow(img, cmap='plasma', interpolation='nearest') 105 | #plt.colorbar() 106 | plt.savefig(path + str(epoch) + ".jpg") 107 | plt.savefig(path + str(epoch) + ".eps", format='eps') 108 | plt.gcf().clear() 109 | return 100. * correct / len(loader.dataset) 110 | 111 | def constructExperimentName(args): 112 | import os 113 | name = [args.model_type, str(args.epochs_class), str(args.step_size)] 114 | if not args.no_herding: 115 | name.append("herding") 116 | if not args.no_distill: 117 | name.append("distillation") 118 | if not os.path.exists("../" + args.name): 119 | os.makedirs("../" + args.name) 120 | 121 | return "../" + args.name + "/" + "_".join(name) 122 | 123 | # y should be of type [("Value Name", y_values), ....] 124 | def plotAccuracy(experiment, x, y, num_classes, plot_name): 125 | myPlotter = pl.Plotter() 126 | 127 | if not isinstance(y, list): 128 | print("y must be a list of tuples!") 129 | assert(False) 130 | 131 | for i in range(len(y)): 132 | experiment.results[y[i][0]] = [x, y[i][1]] 133 | myPlotter.plot(x, y[i][1], title=plot_name, legend=y[i][0]) 134 | 135 | myPlotter.save_fig(experiment.path + "Overall", num_classes) 136 | experiment.store_json() 137 | 138 | def plotEmbeddings(experiment, embedding_name_pairs, plot_name="", range_val=(-.20,.20)): 139 | """ 140 | Takes avg across all classes of embedding (axis 0) then plots them 141 | embedding_name_pairs: Should be of type [("Embedding Name 1", Embedding_vector_1), ....] 142 | 143 | """ 144 | embeddings = [] 145 | labels = [] 146 | for i in embedding_name_pairs: 147 | embeddings.append(np.mean(i[1].squeeze().cpu().numpy(), 0)) 148 | labels.append(i[0]) 149 | 150 | myPlotter = pl.Plotter() 151 | myPlotter.plot_embeddings(embeddings, labels, range_val, experiment, plot_name) 152 | -------------------------------------------------------------------------------- /model/resnet32.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | from torch.nn import init 7 | 8 | from .res_utils import DownsampleA 9 | 10 | 11 | class ResNetBasicblock(nn.Module): 12 | expansion = 1 13 | """ 14 | RexNet basicblock (https://github.com/facebook/fb.resnet.torch/blob/master/models/resnet.lua) 15 | """ 16 | 17 | def __init__(self, inplanes, planes, stride=1, downsample=None): 18 | super(ResNetBasicblock, self).__init__() 19 | 20 | self.conv_a = nn.Conv2d(inplanes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 21 | self.bn_a = nn.BatchNorm2d(planes) 22 | 23 | self.conv_b = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) 24 | self.bn_b = nn.BatchNorm2d(planes) 25 | 26 | self.downsample = downsample 27 | 28 | def forward(self, x): 29 | residual = x 30 | 31 | basicblock = self.conv_a(x) 32 | basicblock = self.bn_a(basicblock) 33 | basicblock = F.relu(basicblock, inplace=True) 34 | 35 | basicblock = self.conv_b(basicblock) 36 | basicblock = self.bn_b(basicblock) 37 | 38 | if self.downsample is not None: 39 | residual = self.downsample(x) 40 | 41 | return F.relu(residual + basicblock, inplace=True) 42 | 43 | 44 | class CifarResNet(nn.Module): 45 | """ 46 | ResNet optimized for the Cifar dataset, as specified in 47 | https://arxiv.org/abs/1512.03385.pdf 48 | """ 49 | 50 | def __init__(self, block, depth, num_classes, channels=3): 51 | """ Constructor 52 | Args: 53 | depth: number of layers. 54 | num_classes: number of classes 55 | base_width: base width 56 | """ 57 | super(CifarResNet, self).__init__() 58 | 59 | # Model type specifies number of layers for CIFAR-10 and CIFAR-100 model 60 | assert (depth - 2) % 6 == 0, 'depth should be one of 20, 32, 44, 56, 110' 61 | layer_blocks = (depth - 2) // 6 62 | print('CifarResNet : Depth : {} , Layers for each block : {}'.format(depth, layer_blocks)) 63 | 64 | self.num_classes = num_classes 65 | 66 | self.conv_1_3x3 = nn.Conv2d(channels, 16, kernel_size=3, stride=1, padding=1, bias=False) 67 | self.bn_1 = nn.BatchNorm2d(16) 68 | 69 | self.inplanes = 16 70 | self.stage_1 = self._make_layer(block, 16, layer_blocks, 1) 71 | self.stage_2 = self._make_layer(block, 32, layer_blocks, 2) 72 | self.stage_3 = self._make_layer(block, 64, layer_blocks, 2) 73 | self.avgpool = nn.AvgPool2d(8) 74 | self.classifier = nn.Linear(64 * block.expansion, num_classes) 75 | 76 | for m in self.modules(): 77 | if isinstance(m, nn.Conv2d): 78 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 79 | m.weight.data.normal_(0, math.sqrt(2. / n)) 80 | # m.bias.data.zero_() 81 | elif isinstance(m, nn.BatchNorm2d): 82 | m.weight.data.fill_(1) 83 | m.bias.data.zero_() 84 | elif isinstance(m, nn.Linear): 85 | init.kaiming_normal(m.weight) 86 | m.bias.data.zero_() 87 | 88 | def _make_layer(self, block, planes, blocks, stride=1): 89 | downsample = None 90 | if stride != 1 or self.inplanes != planes * block.expansion: 91 | downsample = DownsampleA(self.inplanes, planes * block.expansion, stride) 92 | 93 | layers = [] 94 | layers.append(block(self.inplanes, planes, stride, downsample)) 95 | self.inplanes = planes * block.expansion 96 | for i in range(1, blocks): 97 | layers.append(block(self.inplanes, planes)) 98 | 99 | return nn.Sequential(*layers) 100 | 101 | def forward(self, x, feature=False, labels=False, T=1, both=False): 102 | # print ("X shape", x.shape) 103 | x = self.conv_1_3x3(x) 104 | x = F.relu(self.bn_1(x), inplace=True) 105 | x = self.stage_1(x) 106 | x = self.stage_2(x) 107 | x = self.stage_3(x) 108 | x = self.avgpool(x) 109 | x = x.view(x.size(0), -1) 110 | #print("aaa") 111 | #print(x) 112 | if feature: 113 | return x / torch.norm(x, 2, 1).unsqueeze(1) 114 | 115 | if labels: 116 | return F.softmax(self.classifier(x)/T, dim=1) 117 | if both: 118 | return F.log_softmax(self.classifier(x), dim=1), F.log_softmax(self.classifier(x)/T, dim=1) 119 | 120 | return F.log_softmax(self.classifier(x)/T, dim=1) 121 | 122 | 123 | def forward_feature(self, x): 124 | pass 125 | 126 | 127 | def resnet20(num_classes=10): 128 | """Constructs a ResNet-20 model for CIFAR-10 (by default) 129 | Args: 130 | num_classes (uint): number of classes 131 | """ 132 | model = CifarResNet(ResNetBasicblock, 20, num_classes) 133 | return model 134 | 135 | 136 | def resnet10mnist(num_classes=10): 137 | """Constructs a ResNet-20 model for CIFAR-10 (by default) 138 | Args: 139 | num_classes (uint): number of classes 140 | """ 141 | model = CifarResNet(ResNetBasicblock, 10, num_classes, 1) 142 | return model 143 | 144 | 145 | def resnet20mnist(num_classes=10): 146 | """Constructs a ResNet-20 model for CIFAR-10 (by default) 147 | Args: 148 | num_classes (uint): number of classes 149 | """ 150 | model = CifarResNet(ResNetBasicblock, 20, num_classes, 1) 151 | return model 152 | 153 | 154 | def resnet32mnist(num_classes=10, channels=1): 155 | model = CifarResNet(ResNetBasicblock, 32, num_classes, channels) 156 | return model 157 | 158 | 159 | def resnet32(num_classes=10): 160 | """Constructs a ResNet-32 model for CIFAR-10 (by default) 161 | Args: 162 | num_classes (uint): number of classes 163 | """ 164 | model = CifarResNet(ResNetBasicblock, 32, num_classes) 165 | return model 166 | 167 | 168 | def resnet44(num_classes=10): 169 | """Constructs a ResNet-44 model for CIFAR-10 (by default) 170 | Args: 171 | num_classes (uint): number of classes 172 | """ 173 | model = CifarResNet(ResNetBasicblock, 44, num_classes) 174 | return model 175 | 176 | 177 | def resnet56(num_classes=10): 178 | """Constructs a ResNet-56 model for CIFAR-10 (by default) 179 | Args: 180 | num_classes (uint): number of classes 181 | """ 182 | model = CifarResNet(ResNetBasicblock, 56, num_classes) 183 | return model 184 | 185 | 186 | def resnet110(num_classes=10): 187 | """Constructs a ResNet-110 model for CIFAR-10 (by default) 188 | Args: 189 | num_classes (uint): number of classes 190 | """ 191 | model = CifarResNet(ResNetBasicblock, 110, num_classes) 192 | return model 193 | -------------------------------------------------------------------------------- /trainer/gans/wgan.py: -------------------------------------------------------------------------------- 1 | import time 2 | import itertools 3 | 4 | import torch 5 | import numpy as np 6 | import torch.nn as nn 7 | import torch.optim as optim 8 | 9 | from trainer.gans.gan import GAN 10 | from torch.autograd import Variable 11 | import trainer.gans.gutils as gutils 12 | 13 | class WGAN(GAN): 14 | ''' 15 | WGAN Trainer 16 | __init__ in the base class 17 | ''' 18 | 19 | def train(self, G, D, active_classes, increment): 20 | ''' 21 | G: Generator 22 | D: Discriminator 23 | active_classes: All classes trained on so far 24 | increment: Which increment number are we on (0 indexed) 25 | ''' 26 | g_losses = [] 27 | d_losses = [] 28 | print("ACTIVE CLASSES: ", active_classes) 29 | 30 | #Author of WGAN used RMSprop 31 | if self.args.gan_lr > 5e-5 or len(self.args.gan_schedule) > 1: 32 | print(">>> NOTICE: Did you mean to set GAN lr/schedule to this value?") 33 | g_opt = optim.RMSprop(G.parameters(), lr=self.args.gan_lr) 34 | d_opt = optim.RMSprop(D.parameters(), lr=self.args.gan_lr) 35 | 36 | one_sample_saved = False 37 | # Count G and D iters 38 | a = 0 39 | b = 0 40 | print("Starting GAN Training") 41 | #-------------Start Epoch-----------# 42 | for epoch in range(int(self.args.gan_epochs[increment])): 43 | G.train() 44 | d_losses_e = [] 45 | g_losses_e = [] 46 | dist_losses_e = [] 47 | start_time = time.time() 48 | gutils.update_lr(epoch, g_opt, d_opt, 49 | self.args.gan_schedule, 50 | self.args.gan_gammas) 51 | 52 | #Iterate over examples that the classifier Trainer just iterated on 53 | for batch_idx, (image, label) in enumerate(self.train_iterator): 54 | batch_size = image.shape[0] 55 | if not one_sample_saved: 56 | gutils.save_results(self.args, image, "sample_E" + str(epoch), True, 57 | np.sqrt(self.args.batch_size), 58 | self.experiment) 59 | one_sample_saved = True 60 | 61 | #-------------Train Discriminator-----------# 62 | ##Train using real images 63 | D.zero_grad() 64 | a = a + 1 65 | 66 | #Discriminator output for real images 67 | if self.args.cuda: 68 | image = Variable(image.cuda()) 69 | d_output_real = D(image).squeeze() 70 | 71 | ##Train using fake images 72 | g_random_noise = torch.randn((batch_size, 100)) 73 | g_random_noise = g_random_noise.view(-1, 100, 1, 1) 74 | if self.args.cuda: 75 | g_random_noise = Variable(g_random_noise.cuda()) 76 | 77 | #Generating fake images and passing them to discriminator 78 | g_output = G(g_random_noise) 79 | #Detach gradient from Generator 80 | g_output = g_output.detach() 81 | d_output_fake = D(g_output).squeeze() 82 | 83 | #Calculate WGAN Loss (no BCE here) 84 | d_loss = -(torch.mean(d_output_real) - torch.mean(d_output_fake)) 85 | 86 | 87 | #Perform a backward step 88 | d_losses_e.append(d_loss_total.cpu().data.numpy()) 89 | d_loss.backward() 90 | d_opt.step() 91 | 92 | #Clamp the parameters (part of WGAN) 93 | for p in D.parameters(): 94 | p.data.clamp_(-0.01, 0.01) 95 | 96 | #-------------Train Generator-----------# 97 | #Train critic (discriminator) more in case of WGAN because the 98 | #critic needs to be trained to optimality 99 | if batch_idx % self.args.d_iter != 0: 100 | continue 101 | 102 | b = b + 1 103 | G.zero_grad() 104 | g_output = G(g_random_noise) 105 | d_output = D(g_output).squeeze() 106 | 107 | #Calculate WGAN Generator loss 108 | g_loss = -torch.mean(d_output) 109 | total_loss = g_loss 110 | 111 | #Jointly optimizes the GAN Generator loss and lowers 112 | #the euclidean distance between features of generated 113 | #and real images (default: off) 114 | if self.args.joint_gan_obj: 115 | model = self.fixed_classifier 116 | euclidean_dist = nn.PairwiseDistance(2) 117 | # Generate features for real and fake images 118 | output_fake = model.forward(g_output, True) 119 | output_real = model.forward(image, True) 120 | # Calculate euclidean distance 121 | distance_loss = torch.mean(euclidean_dist(output_fake, output_real)) 122 | total_loss = g_loss + (self.args.joint_gan_alpha * distance_loss) 123 | dist_losses_e.append(distance_loss.cpu().data.numpy()) 124 | 125 | #Backward step 126 | total_loss.backward() 127 | g_losses_e.append(g_loss.cpu().data.numpy()) 128 | g_opt.step() 129 | 130 | #-------------End Epoch-----------# 131 | #Print Stats and save results 132 | mean_g = (sum(g_losses_e)/len(g_losses_e)) 133 | mean_d = (sum(d_losses_e)/len(d_losses_e)) 134 | mean_dist = None 135 | if self.args.joint_gan_obj: 136 | mean_dist = (sum(dist_losses_e)/len(dist_losses_e)) 137 | g_losses.append(mean_g) 138 | d_losses.append(mean_d) 139 | 140 | if epoch % self.args.gan_img_save_interval == 0: 141 | #Generate some examples for visualizing 142 | gutils.generate_examples(self.args, G, 100, 143 | active_classes, 144 | self.total_classes, 145 | self.fixed_noise, 146 | self.experiment, 147 | "Inc" + str(increment) + "_E" + str(epoch), 148 | True, False) 149 | #Plot GAN losses 150 | gutils.save_gan_losses(g_losses, d_losses, 151 | self.args.gan_epochs[increment], 152 | increment, 153 | self.experiment) 154 | 155 | #Save ckpt if interval is satisfied 156 | if self.args.save_g_ckpt and epoch % self.args.ckpt_interval == 0: 157 | gutils.save_checkpoint(epoch, increment, self.experiment, G) 158 | 159 | print("[GAN] Epoch:", epoch, 160 | "G_iters:", b, 161 | "D_iters:", a, 162 | "g_loss:", mean_g, 163 | "d_loss:", mean_d, 164 | "dist_Loss:", mean_dist, 165 | "Time taken:", time.time() - start_time) 166 | -------------------------------------------------------------------------------- /trainer/gans/dcgan.py: -------------------------------------------------------------------------------- 1 | import time 2 | import itertools 3 | 4 | import torch 5 | import numpy as np 6 | import torch.nn as nn 7 | import torch.optim as optim 8 | 9 | from trainer.gans.gan import GAN 10 | from torch.autograd import Variable 11 | import trainer.gans.gutils as gutils 12 | 13 | class DCGAN(GAN): 14 | ''' 15 | DCGAN Trainer 16 | __init__ in the base class 17 | ''' 18 | 19 | def train(self, G, D, active_classes, increment): 20 | ''' 21 | G: Generator 22 | D: Discriminator 23 | active_classes: All classes trained on so far 24 | increment: Which increment number are we on (0 indexed) 25 | ''' 26 | g_losses = [] 27 | d_losses = [] 28 | print("ACTIVE CLASSES: ", active_classes) 29 | 30 | criterion = nn.BCELoss() 31 | g_opt = optim.Adam(G.parameters(), lr=self.args.gan_lr, betas=(0.5, 0.999)) 32 | d_opt = optim.Adam(D.parameters(), lr=self.args.gan_lr, betas=(0.5, 0.999)) 33 | 34 | one_sample_saved = False 35 | # Count G and D iters 36 | a = 0 37 | b = 0 38 | print("Starting GAN Training") 39 | #-------------Start Epoch-----------# 40 | for epoch in range(int(self.args.gan_epochs[increment])): 41 | G.train() 42 | d_losses_e = [] 43 | g_losses_e = [] 44 | dist_losses_e = [] 45 | start_time = time.time() 46 | gutils.update_lr(epoch, g_opt, d_opt, 47 | self.args.gan_schedule, 48 | self.args.gan_gammas) 49 | 50 | #Iterate over examples that the classifier Trainer just iterated on 51 | for batch_idx, (image, label) in enumerate(self.train_iterator): 52 | batch_size = image.shape[0] 53 | if not one_sample_saved: 54 | gutils.save_results(self.args, image, "sample_E" + str(epoch), True, 55 | np.sqrt(self.args.batch_size), 56 | self.experiment) 57 | one_sample_saved = True 58 | 59 | #Make vectors of ones and zeros of same shape as output by 60 | #Discriminator so that it can be used in BCELoss 61 | smoothing_val = 0 62 | if self.args.label_smoothing: 63 | smoothing_val = 0.1 64 | d_like_real = torch.ones(batch_size) - smoothing_val 65 | d_like_fake = torch.zeros(batch_size) 66 | if self.args.cuda: 67 | d_like_real = Variable(d_like_real.cuda()) 68 | d_like_fake = Variable(d_like_fake.cuda()) 69 | #-------------Train Discriminator-----------# 70 | ##Train using real images 71 | D.zero_grad() 72 | a = a + 1 73 | 74 | #Discriminator output for real images 75 | if self.args.cuda: 76 | image = Variable(image.cuda()) 77 | d_output_real = D(image).squeeze() 78 | 79 | ##Train using fake images 80 | g_random_noise = torch.randn((batch_size, 100)) 81 | g_random_noise = g_random_noise.view(-1, 100, 1, 1) 82 | if self.args.cuda: 83 | g_random_noise = Variable(g_random_noise.cuda()) 84 | 85 | #Generating fake images and passing them to discriminator 86 | g_output = G(g_random_noise) 87 | #Detach gradient from Generator 88 | g_output = g_output.detach() 89 | d_output_fake = D(g_output).squeeze() 90 | 91 | #Calculate BCE loss 92 | d_real_loss = criterion(d_output_real, d_like_real) 93 | d_fake_loss = criterion(d_output_fake, d_like_fake) 94 | d_loss = d_real_loss + d_fake_loss 95 | 96 | #Perform a backward step 97 | d_losses_e.append(d_loss.cpu().data.numpy()) 98 | d_loss.backward() 99 | d_opt.step() 100 | 101 | #-------------Train Generator-----------# 102 | #TODO Disabled regenerating of noise, check if it still works 103 | b = b + 1 104 | G.zero_grad() 105 | g_output = G(g_random_noise) 106 | d_output = D(g_output).squeeze() 107 | 108 | #Calculate BCE loss 109 | g_loss = criterion(d_output, d_like_real) 110 | total_loss = g_loss 111 | 112 | #Jointly optimizes the GAN Generator loss and lowers 113 | #the euclidean distance between features of generated 114 | #and real images 115 | if self.args.joint_gan_obj: 116 | model = self.fixed_classifier 117 | euclidean_dist = nn.PairwiseDistance(2) 118 | # Generate features for real and fake images 119 | output_fake = model.forward(g_output, True) 120 | output_real = model.forward(image, True) 121 | # Calculate euclidean distance 122 | distance_loss = torch.mean(euclidean_dist(output_fake, output_real)) 123 | total_loss = g_loss + (self.args.joint_gan_alpha * distance_loss) 124 | dist_losses_e.append(distance_loss.cpu().data.numpy()) 125 | 126 | #Backward step 127 | total_loss.backward() 128 | g_losses_e.append(g_loss.cpu().data.numpy()) 129 | g_opt.step() 130 | 131 | #-------------End Epoch-----------# 132 | #Print Stats and save results 133 | mean_g = (sum(g_losses_e)/len(g_losses_e)) 134 | mean_d = (sum(d_losses_e)/len(d_losses_e)) 135 | mean_dist = None 136 | if self.args.joint_gan_obj: 137 | mean_dist = (sum(dist_losses_e)/len(dist_losses_e)) 138 | g_losses.append(mean_g) 139 | d_losses.append(mean_d) 140 | 141 | if epoch % self.args.gan_img_save_interval == 0: 142 | #Generate some examples for visualizing 143 | gutils.generate_examples(self.args, G, 100, 144 | active_classes, 145 | self.total_classes, 146 | self.fixed_noise, 147 | self.experiment, 148 | "Inc" + str(increment) + "_E" + str(epoch), 149 | True, False) 150 | #Plot GAN losses 151 | gutils.save_gan_losses(g_losses, d_losses, 152 | self.args.gan_epochs[increment], 153 | increment, 154 | self.experiment) 155 | 156 | #Save ckpt if interval is satisfied 157 | if self.args.save_g_ckpt and epoch % self.args.ckpt_interval == 0: 158 | gutils.save_checkpoint(epoch, increment, self.experiment, G) 159 | 160 | print("[GAN] Epoch:", epoch, 161 | "G_iters:", b, 162 | "D_iters:", a, 163 | "g_loss:", mean_g, 164 | "d_loss:", mean_d, 165 | "dist_Loss:", mean_dist, 166 | "Time taken:", time.time() - start_time) 167 | -------------------------------------------------------------------------------- /trainer/gans/acgan.py: -------------------------------------------------------------------------------- 1 | import time 2 | import itertools 3 | 4 | import torch 5 | import numpy as np 6 | import torch.nn as nn 7 | import torch.optim as optim 8 | 9 | from trainer.gans.gan import GAN 10 | from torch.autograd import Variable 11 | from utils import utils 12 | import trainer.gans.gutils as gutils 13 | 14 | class ACGAN(GAN): 15 | ''' 16 | ACGAN Trainer 17 | __init__ in the base class 18 | ''' 19 | 20 | def train(self, G, D, active_classes, increment): 21 | ''' 22 | G: Generator 23 | D: Discriminator 24 | active_classes: All classes trained on so far 25 | increment: Which increment number are we on (0 indexed) 26 | K: Total number of classes in the dataset (just for convenience) 27 | ''' 28 | print("ACTIVE CLASSES: ", active_classes) 29 | 30 | #Define variables 31 | iters = 0 32 | a_losses = [] 33 | g_losses = [] 34 | d_losses = [] 35 | one_sample_saved = False 36 | d_criterion = nn.BCELoss() 37 | a_criterion = nn.NLLLoss() 38 | nz = self.total_classes + 100 39 | nc = 1 if self.args.dataset == "MNIST" else 3 40 | real_label = 0.9 if self.args.label_smoothing else 1 41 | fake_label = 0 42 | 43 | #Define tensors, d_label=discriminator label a_label=auxilary label 44 | inp = torch.FloatTensor(self.args.batch_size, nc, 32, 32) 45 | noise = torch.FloatTensor(self.args.batch_size, nz, 1, 1) 46 | d_label = torch.FloatTensor(self.args.batch_size) 47 | a_label = torch.LongTensor(self.args.batch_size) 48 | 49 | #To cuda and Variable 50 | if self.args.cuda: 51 | d_criterion.cuda() 52 | a_criterion.cuda() 53 | inp, noise= inp.cuda(), noise.cuda() 54 | d_label, a_label = d_label.cuda(), a_label.cuda() 55 | inp, noise = Variable(inp), Variable(noise) 56 | d_label, a_label = Variable(d_label), Variable(a_label) 57 | 58 | #what is a better way to not forget setting good hyperparams? 59 | assert self.args.gan_lr == 0.0002 60 | g_opt = optim.Adam(G.parameters(), lr=self.args.gan_lr, betas=(0.5, 0.999)) 61 | d_opt = optim.Adam(D.parameters(), lr=self.args.gan_lr, betas=(0.5, 0.999)) 62 | 63 | print("Starting GAN Training") 64 | #-------------Start Epoch-----------# 65 | for epoch in range(int(self.args.gan_epochs[increment])): 66 | torch.manual_seed(self.args.seed + epoch) 67 | acc_e = [] 68 | a_losses_e = [] 69 | d_losses_e = [] 70 | g_losses_e = [] 71 | fake_prob_e = [] 72 | real_prob_e = [] 73 | G.train() 74 | D.train() 75 | gutils.update_lr(epoch, g_opt, d_opt, self.args.gan_schedule, 76 | self.args.gan_gammas) 77 | start_time = time.time() 78 | #Iterate over examples that the classifier Trainer just iterated on 79 | for batch_idx, (image, label) in enumerate(self.train_iterator): 80 | iters = iters + 1 81 | batch_size = image.shape[0] 82 | if self.args.cuda: 83 | image = image.cuda() 84 | label = label.cuda() 85 | if inp.shape[0] != batch_size: 86 | inp.data.resize_as_(image) 87 | noise.data.resize_(batch_size, nz, 1, 1) 88 | d_label.data.resize_(batch_size) 89 | a_label.data.resize_(batch_size) 90 | if not one_sample_saved: 91 | one_sample_saved = True 92 | gutils.save_results(self.args, image, "sample_E" + str(epoch), True, 93 | np.sqrt(self.args.batch_size), self.experiment) 94 | 95 | #-------------Train Discriminator-----------# 96 | # https://discuss.pytorch.org/t/how-to-use-the-backward-functions-for-multiple-losses/1826/6 Huh? 97 | ##Train using real images 98 | D.zero_grad() 99 | inp.data.copy_(image) 100 | d_label.data.fill_(real_label) 101 | a_label.data.copy_(label) 102 | 103 | d_output, a_output = D(inp) 104 | d_loss_real = d_criterion(d_output, d_label) 105 | a_loss_real = a_criterion(a_output, a_label) 106 | d_real_total = d_loss_real + a_loss_real 107 | d_real_total.backward() 108 | real_prob_e.append(d_output.data.mean()) 109 | acc_e.append(utils.compute_acc(a_output, a_label)) 110 | 111 | ##Train using fake images 112 | #Generate random noise 113 | noise.data.normal_(0, 1) 114 | #Noise in ACGAN consists of label info + noise 115 | nz_labels = np.random.choice(active_classes, batch_size) 116 | nz_noise = np.random.normal(0, 1, (batch_size, 100)) 117 | hot_labels = np.zeros((batch_size, self.total_classes)) 118 | hot_labels[np.arange(batch_size), nz_labels] = 1 119 | #Combine the two vectors 120 | combined_noise = np.append(hot_labels, nz_noise, axis=1) 121 | combined_noise = torch.from_numpy(combined_noise) 122 | #Insert into the Variables 123 | noise.data.copy_(combined_noise.view(batch_size, nz, 1, 1)) 124 | d_label.data.fill_(fake_label) 125 | a_label.data.copy_(torch.from_numpy(nz_labels)) 126 | 127 | g_output = G(noise) 128 | g_output_temp = g_output.detach() 129 | d_output, a_output = D(g_output_temp) 130 | d_loss_fake = d_criterion(d_output, d_label) 131 | a_loss_fake = a_criterion(a_output, a_label) 132 | d_fake_total = d_loss_fake + a_loss_fake 133 | d_fake_total.backward() 134 | d_loss_total = d_real_total + d_fake_total 135 | fake_prob_e.append(d_output.data.mean()) 136 | d_opt.step() 137 | d_losses_e.append(d_loss_total.cpu().data.numpy()) 138 | 139 | #-------------Train Generator-----------# 140 | G.zero_grad() 141 | d_label.data.fill_(real_label) 142 | d_output, a_output = D(g_output) 143 | d_loss_g = d_criterion(d_output, d_label) 144 | a_loss_g = a_criterion(a_output, a_label) 145 | g_loss_total = d_loss_g + a_loss_g 146 | g_loss_total.backward() 147 | g_opt.step() 148 | g_losses_e.append(g_loss_total.cpu().data.numpy()) 149 | #-------------End Epoch-----------# 150 | #Print Stats and save results 151 | mean_g = (sum(g_losses_e)/len(g_losses_e)) 152 | mean_d = (sum(d_losses_e)/len(d_losses_e)) 153 | mean_acc = (sum(acc_e)/len(acc_e)) 154 | mean_prob_real = (sum(real_prob_e)/len(real_prob_e)) 155 | mean_prob_fake = (sum(fake_prob_e)/len(fake_prob_e)) 156 | g_losses.append(mean_g) 157 | d_losses.append(mean_d) 158 | 159 | if epoch % self.args.gan_img_save_interval == 0: 160 | #Generate some examples for visualizing 161 | gutils.generate_examples(self.args, G, 100, 162 | active_classes, 163 | self.total_classes, 164 | self.fixed_noise, 165 | self.experiment, 166 | "Inc" + str(increment) + "_E" + str(epoch), 167 | True, True) 168 | #Plot GAN losses 169 | gutils.save_gan_losses(g_losses, d_losses, 170 | self.args.gan_epochs[increment], 171 | increment, 172 | self.experiment) 173 | #Save ckpt if interval is satisfied 174 | if self.args.save_g_ckpt and epoch % self.args.ckpt_interval == 0: 175 | gutils.save_checkpoint(epoch, increment, self.experiment, G, D) 176 | time_taken = time.time() - start_time 177 | print("[GAN] Epoch: %d" % epoch, 178 | "Iters: %d" % iters, 179 | "g_loss: %f" % mean_g, 180 | "d_loss: %f" % mean_d, 181 | "acc: %f" % mean_acc, 182 | "D(x): %f" % mean_prob_real, 183 | "D(G(z)): %f" % mean_prob_fake, 184 | "Time taken: %f" % time_taken) 185 | -------------------------------------------------------------------------------- /trainer/classifierTrainer.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import copy 4 | 5 | import torch 6 | import torch.nn.functional as F 7 | import torch.utils.data as td 8 | from torch.autograd import Variable 9 | import random 10 | 11 | class Trainer(): 12 | def __init__(self, train_data_iterator, test_data_iterator, dataset, model, args, optimizer, train_data_iterator_ideal=None): 13 | self.train_data_iterator = train_data_iterator 14 | self.test_data_ierator = test_data_iterator 15 | self.train_data_iterator_ideal = train_data_iterator_ideal 16 | self.model = model 17 | self.args = args 18 | self.dataset = dataset 19 | self.train_loader = self.train_data_iterator.dataset 20 | self.older_classes = [] 21 | self.optimizer = optimizer 22 | self.model_fixed = copy.deepcopy(self.model) 23 | self.active_classes = [] 24 | for param in self.model_fixed.parameters(): 25 | param.requires_grad = False 26 | 27 | self.current_lr = args.lr 28 | self.all_classes = list(range(dataset.classes)) 29 | self.all_classes.sort(reverse=True) 30 | self.left_over = [] 31 | if args.ideal_nmc: 32 | self.train_loader_ideal = self.train_data_iterator_ideal.dataset 33 | if not args.no_random: 34 | print("Randomly shuffling classes") 35 | random.seed(args.seed) 36 | random.shuffle(self.all_classes) 37 | 38 | def update_iterator(self, train_iterator): 39 | self.train_data_iterator = train_iterator 40 | self.train_loader = train_iterator.dataset 41 | 42 | def update_lr(self, epoch): 43 | for temp in range(0, len(self.args.schedule)): 44 | if self.args.schedule[temp] == epoch: 45 | for param_group in self.optimizer.param_groups: 46 | self.current_lr = param_group['lr'] 47 | param_group['lr'] = self.current_lr * self.args.gammas[temp] 48 | print("Changing learning rate from", self.current_lr, "to", self.current_lr * self.args.gammas[temp]) 49 | self.current_lr *= self.args.gammas[temp] 50 | 51 | def increment_classes(self, class_group): 52 | for temp in range(class_group, class_group + self.args.step_size): 53 | pop_val = self.all_classes.pop() 54 | self.train_data_iterator.dataset.add_classes(pop_val) 55 | self.test_data_ierator.dataset.add_classes(pop_val) 56 | if self.args.ideal_nmc: 57 | self.train_data_iterator_ideal.dataset.add_classes(pop_val) 58 | print("Train Classes", self.train_data_iterator.dataset.active_classes) 59 | self.left_over.append(pop_val) 60 | 61 | def update_leftover(self, k): 62 | self.older_classes.append(k) 63 | 64 | def limit_class(self, n, k, herding=True): 65 | if not herding: 66 | self.train_loader.limit_class(n, k) 67 | else: 68 | print("Sorting by herding") 69 | self.train_loader.limit_class_and_sort(n, k, self.model_fixed) 70 | if n not in self.older_classes: 71 | self.older_classes.append(n) 72 | 73 | def setup_training(self): 74 | for param_group in self.optimizer.param_groups: 75 | print("Setting LR to", self.args.lr) 76 | param_group['lr'] = self.args.lr 77 | self.current_lr = self.args.lr 78 | 79 | k = 0 80 | if self.args.process == "nmc" and self.left_over != []: 81 | k = int(self.args.memory_budget / len(self.left_over)) 82 | for val in self.left_over: 83 | self.limit_class(val, k, not self.args.no_herding) 84 | 85 | 86 | def update_frozen_model(self): 87 | self.model.eval() 88 | self.model_fixed = copy.deepcopy(self.model) 89 | for param in self.model_fixed.parameters(): 90 | param.requires_grad = False 91 | 92 | def insert_generated_images(self, data, target, gan_images, gan_labels, batch_size, is_cond=False): 93 | ''' 94 | data: Images from data iterator 95 | target: Labels from data iterator 96 | gan_images: Generated images by GAN 97 | gan_labels: Python list containing unique classes of generated 98 | images 99 | batch_size: Current batch_size of training iterator 100 | ''' 101 | if self.args.process == 'gan' or self.args.process == 'cgan': 102 | if not len(gan_images) == 0: 103 | if is_cond: 104 | per_k_batch = (self.args.batch_size - batch_size) // len(gan_labels) 105 | for k in gan_labels: 106 | random_indices = torch.randperm(gan_images[k].shape[0])[0:per_k_batch] 107 | new_targets = (torch.ones(per_k_batch) * k).long() 108 | data = torch.cat((data, gan_images[k][random_indices]), dim=0) 109 | target = torch.cat((target, new_targets), dim=0) 110 | else: 111 | batch = self.args.batch_size - batch_size 112 | gan_images = gan_images.cpu() 113 | random_indices = torch.randperm(gan_images.shape[0])[0:batch] 114 | data = torch.cat((data, gan_images[random_indices].data), dim=0) 115 | target = None 116 | return data, target 117 | 118 | def train(self, gan_images=None, gan_labels=None, batch_size=None, D=None, epoch=0): 119 | torch.manual_seed(self.args.seed + epoch) 120 | self.model.train() 121 | #TODO CHECK MEMORY 122 | if D is not None: 123 | for param in D.parameters(): 124 | param.requires_grad = False 125 | # D.eval() 126 | 127 | for batch_idx, (data, target) in enumerate(self.train_data_iterator): 128 | if self.args.cuda: 129 | data, target = data.cuda(), target.cuda() 130 | 131 | weight_vector = (target * 0).int() 132 | for elem in self.older_classes: 133 | weight_vector = weight_vector + (target == elem).int() 134 | 135 | old_classes_indices = torch.squeeze(torch.nonzero((weight_vector > 0)).long()) 136 | new_classes_indices = torch.squeeze(torch.nonzero((weight_vector == 0)).long()) 137 | self.optimizer.zero_grad() 138 | 139 | y_onehot = torch.FloatTensor(len(data), self.dataset.classes) 140 | if self.args.cuda: 141 | y_onehot = y_onehot.cuda() 142 | 143 | y_onehot.zero_() 144 | target.unsqueeze_(1) 145 | y_onehot.scatter_(1, target, 1) 146 | 147 | output, output2 = self.model(Variable(data), T=self.args.T, both=True) 148 | if self.args.ac_distill and D is not None and len(self.older_classes) > 0: 149 | pred2 = D(Variable(data, True), T=self.args.T)[1] 150 | loss2 = F.kl_div(output2, Variable(pred2.data)) 151 | loss2.backward(retain_graph=True) 152 | alpha = 1 153 | for param in self.model.parameters(): 154 | if param.grad is not None: 155 | param.grad = param.grad * (self.args.T * self.args.T) * alpha 156 | # y_onehot = pred2.data 157 | elif not self.args.no_distill: 158 | if len(self.older_classes) > 0: 159 | pred2 = self.model_fixed(Variable(data), labels=True, T=self.args.T) 160 | loss2 = F.kl_div(output2, Variable(pred2.data)) 161 | loss2.backward(retain_graph=True) 162 | alpha=1 163 | for param in self.model.parameters(): 164 | if param.grad is not None: 165 | param.grad = param.grad * (self.args.T * self.args.T) * alpha 166 | # y_onehot[:, self.older_classes] = pred2.data[:, self.older_classes] 167 | loss = F.kl_div(output, Variable(y_onehot)) 168 | loss.backward() 169 | self.optimizer.step() 170 | 171 | #TODO Add generated images here 172 | def evaluate(self, loader, D=None): 173 | self.model.eval() 174 | test_loss = 0 175 | correct = 0 176 | 177 | for data, target in loader: 178 | if self.args.cuda: 179 | data, target = data.cuda(), target.cuda() 180 | data, target = Variable(data, volatile=True), Variable(target, volatile=True) 181 | if D is not None: 182 | output = D(data, T=1)[1] 183 | else: 184 | output = self.model(data) 185 | test_loss += F.nll_loss(output, target, size_average=False).data[0] # sum up batch loss 186 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 187 | correct += pred.eq(target.data.view_as(pred)).cpu().sum() 188 | test_loss /= len(loader.dataset) 189 | return 100. * correct / len(loader.dataset) 190 | -------------------------------------------------------------------------------- /trainer/gans/gutils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import itertools 3 | 4 | import torch 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | from torch.autograd import Variable 8 | 9 | 10 | def save_results(args, images, name, is_tensor=False, axis_size=10, experiment=None): 11 | ''' 12 | Saves the images in a grid of axis_size * axis_size 13 | args: args 14 | images: Dict of images to save 15 | is_tensor: Whether the images are a torch tensor 16 | axis_size: Number of images on each axis 17 | experiment: Experiment object (needed to get destination path) 18 | ''' 19 | axis_size = int(axis_size) 20 | _, sub = plt.subplots(axis_size, axis_size, figsize=(5, 5)) 21 | for i, j in itertools.product(range(axis_size), range(axis_size)): 22 | sub[i, j].get_xaxis().set_visible(False) 23 | sub[i, j].get_yaxis().set_visible(False) 24 | 25 | for k in range(axis_size * axis_size): 26 | i = k // axis_size 27 | j = k % axis_size 28 | sub[i, j].cla() 29 | if args.dataset == "CIFAR100" or args.dataset == "CIFAR10": 30 | if is_tensor: 31 | sub[i, j].imshow((images[k].cpu().numpy().transpose(1, 2, 0) + 1)/2) 32 | else: 33 | sub[i, j].imshow((images[k].cpu().data.numpy().transpose(1, 2, 0) + 1)/2) 34 | elif args.dataset == "MNIST": 35 | if is_tensor: 36 | sub[i, j].imshow(images[k, 0].cpu().numpy(), cmap='gray') 37 | else: 38 | sub[i, j].imshow(images[k, 0].cpu().data.numpy(), cmap='gray') 39 | 40 | plt.savefig(experiment.path + "results/" + name + ".png") 41 | plt.cla() 42 | plt.clf() 43 | plt.close() 44 | 45 | 46 | def generate_examples( 47 | args, G, num_examples, active_classes, total_classes, 48 | noise, experiment, name="", save=False, is_cond=False, D=None): 49 | ''' 50 | Returns a dict[class] of generated samples. 51 | In case of Non-Conditional GAN, the samples in the dict are random, they do 52 | not correspond to the keys in the dict 53 | Just passing in random noise to the generator and storing the results in dict 54 | Generates a batch of 100 examples at a time 55 | args: args 56 | num_examples: Total number of examples to generate 57 | active_classes: List of all classes trained on till now 58 | total_classes: Total number of classes in the dataset 59 | noise: A noise vector of size [100,100,1,1] to generate examples 60 | experiment: Experiment object 61 | save: If True, also save samples of generated images to disk 62 | is_cond: If True, use the label information too (Only use with supported GANs) 63 | ''' 64 | print("Note: Ignoring the fixed noise") 65 | #G.eval() 66 | #for param in G.parameters(): 67 | # param.requires_grad = False 68 | #if D is not None: 69 | # D.eval() 70 | #if D is not None: 71 | # for param in D.parameters(): 72 | # param.requires_grad = False 73 | examples = {} 74 | num_iter = 0 75 | for idx, klass in enumerate(active_classes): 76 | while ((not klass in examples.keys()) or (len(examples[klass]) < num_examples)): 77 | num_iter += 1 78 | if args.process == "cdcgan": 79 | targets = torch.zeros(100, total_classes, 1, 1) 80 | targets[:, klass] = 1 81 | noise = torch.randn(100, 100, 1, 1) 82 | elif args.process == "acgan": 83 | targets = np.zeros((100, total_classes)) 84 | targets[:, klass] = 1 85 | nz_noise = np.random.normal(0, 1, (100, 100)) 86 | combined_noise = np.append(targets, nz_noise, axis=1) 87 | noise = torch.from_numpy(combined_noise) 88 | noise = noise.view(100, 100+total_classes, 1, 1).float() 89 | else: 90 | noise = torch.randn(100, 100, 1, 1) 91 | if args.cuda: 92 | noise = Variable(noise.cuda(), volatile=True) 93 | if args.process == "cdcgan": 94 | targets = Variable(targets.cuda(), volatile=True) 95 | if args.process == "cdcgan": 96 | images = G(noise, targets) 97 | else: 98 | images = G(noise) 99 | if args.filter_using_disc and D is not None: 100 | d_output = D(images) 101 | #Select imges that whose real-fake value > filter_val 102 | #indices = (d_output[0] > args.filter_val).nonzero().squeeze() 103 | #Select imges with P(img) belonging to class klass > filter_val 104 | indices = (d_output[1][:, klass] > args.filter_val).nonzero().squeeze() 105 | if indices.dim() == 0: 106 | continue 107 | images = torch.index_select(images, 0, indices) 108 | if not klass in examples.keys(): 109 | examples[klass] = images 110 | else: 111 | examples[klass] = torch.cat((examples[klass],images), dim=0) 112 | 113 | # Dont save more than the required number of classes 114 | if save and idx <= args.gan_save_classes and args.gan_num_examples > 0: 115 | save_results(args, examples[klass][0:100], 116 | name + "_C" + str(klass), 117 | False, 10, experiment) 118 | # Trim extra examples 119 | if D is not None: 120 | for klass in active_classes: 121 | examples[klass] = examples[klass][0:num_examples] 122 | print("[INFO] Examples matching the filter: ", len(active_classes) * (num_examples / num_iter), "%") 123 | return examples 124 | 125 | 126 | def save_gan_losses(g_loss, d_loss, epochs, increment, experiment, name='GAN_LOSS'): 127 | ''' 128 | Plots the GAN loss curves for both G and D 129 | ''' 130 | x = range(len(g_loss)) 131 | plt.plot(x, g_loss, label='G_loss') 132 | plt.plot(x, d_loss, label='D_loss') 133 | 134 | plt.xlabel('Epoch') 135 | plt.ylabel('Loss') 136 | plt.legend(loc=4) 137 | plt.grid(True) 138 | plt.xlim((0, epochs)) 139 | 140 | plt.savefig(experiment.path + name + "_" + str(increment) + ".png") 141 | plt.cla() 142 | plt.clf() 143 | plt.close() 144 | 145 | 146 | def save_checkpoint(epoch, increment, experiment, G, D=None): 147 | ''' 148 | Saves the ckpt to disk 149 | ''' 150 | if epoch == 0: 151 | return 152 | print("[*] Saving Generator checkpoint") 153 | path = experiment.path + "checkpoints/" 154 | torch.save(G.state_dict(), 155 | '{0}G_inc_{1}_e_{2}.pth'.format(path, increment, epoch)) 156 | if D is None: 157 | return 158 | print("[*] Saving Discriminator checkpoint") 159 | path = experiment.path + "checkpoints/" 160 | torch.save(D.state_dict(), 161 | '{0}D_inc_{1}_e_{2}.pth'.format(path, increment, epoch)) 162 | 163 | 164 | def load_checkpoint(ckpt_path, increment, G, D=None): 165 | ''' 166 | Loads the latest generator (and discriminator) for given increment 167 | ckpt_path: path to checkpoints folder 168 | increment: current increment number 169 | G: Generator 170 | D: Discriminator (optional) 171 | ''' 172 | max_e = -1 173 | filename = None 174 | for f in os.listdir(ckpt_path): 175 | vals = f.split('_') 176 | #TODO Load the discriminator too 177 | if vals[0] != "G": 178 | continue 179 | incr = int(vals[2]) 180 | epoch = int(vals[4].split('.')[0]) 181 | if incr == increment and epoch > max_e: 182 | max_e = epoch 183 | filename = f 184 | if max_e == -1: 185 | print('[*] Failed to load checkpoint') 186 | return False 187 | g_path = os.path.join(ckpt_path, filename) 188 | G.load_state_dict(torch.load(g_path)) 189 | print('[*] Loaded Generator from %s' % g_path) 190 | if D is not None: 191 | d_path = os.path.join(ckpt_path, "D_" + "_".join(vals[1:])) 192 | D.load_state_dict(torch.load(d_path)) 193 | print('[*] Loaded Discriminator from %s' % d_path) 194 | return True 195 | 196 | 197 | def update_lr(epoch, g_opt, d_opt, gan_schedule, gan_gammas): 198 | ''' 199 | Update the lr for both optimizers 200 | ''' 201 | for temp in range(0, len(gan_schedule)): 202 | if gan_schedule[temp] == epoch: 203 | #Update Generator LR 204 | for param_group in g_opt.param_groups: 205 | current_lr_g = param_group['lr'] 206 | param_group['lr'] = current_lr_g * gan_gammas[temp] 207 | print("Changing GAN Generator learning rate from", 208 | current_lr_g, "to", current_lr_g * gan_gammas[temp]) 209 | #Update Discriminator LR 210 | for param_group in d_opt.param_groups: 211 | current_lr_d = param_group['lr'] 212 | param_group['lr'] = current_lr_d * gan_gammas[temp] 213 | print("Changing GAN Discriminator learning rate from", 214 | current_lr_d, "to", current_lr_d * gan_gammas[temp]) 215 | -------------------------------------------------------------------------------- /trainer/gans/cdcgan.py: -------------------------------------------------------------------------------- 1 | import time 2 | import itertools 3 | 4 | import torch 5 | import numpy as np 6 | import torch.nn as nn 7 | import torch.optim as optim 8 | 9 | from trainer.gans.gan import GAN 10 | from torch.autograd import Variable 11 | import trainer.gans.gutils as gutils 12 | 13 | class CDCGAN(GAN): 14 | ''' 15 | CDCGAN Trainer 16 | __init__ in the base class 17 | ''' 18 | 19 | def train(self, G, D, active_classes, increment): 20 | ''' 21 | G: Generator 22 | D: Discriminator 23 | active_classes: All classes trained on so far 24 | increment: Which increment number are we on (0 indexed) 25 | K: Total number of classes in the dataset (just for convenience) 26 | ''' 27 | g_losses = [] 28 | d_losses = [] 29 | print("ACTIVE CLASSES: ", active_classes) 30 | 31 | criterion = nn.BCELoss() 32 | g_opt = optim.Adam(G.parameters(), lr=self.args.gan_lr, betas=(0.5, 0.999)) 33 | d_opt = optim.Adam(D.parameters(), lr=self.args.gan_lr, betas=(0.5, 0.999)) 34 | 35 | #Matrix of shape [K,K,1,1] with 1s at positions 36 | #where shape[0]==shape[1] 37 | K = self.total_classes #We can use len(active_classes) in case of no-persist 38 | tensor = [] 39 | g_vec = torch.zeros(K, K) 40 | for i in range(K): 41 | tensor.append(i) 42 | g_vec = g_vec.scatter_(1, torch.LongTensor(tensor).view(K,1), 43 | 1).view(K, K, 1, 1) 44 | #Matrix of shape [K,K,32,32] with 32x32 matrix of 1s 45 | #where shape[0]==shape[1] 46 | d_vec = torch.zeros([K, K, 32, 32]) 47 | for i in range(K): 48 | d_vec[i, i, :, :] = 1 49 | 50 | one_sample_saved = False 51 | # Count G and D iters 52 | a = 0 53 | b = 0 54 | print("Starting GAN Training") 55 | #-------------Start Epoch-----------# 56 | for epoch in range(int(self.args.gan_epochs[increment])): 57 | G.train() 58 | d_losses_e = [] 59 | g_losses_e = [] 60 | dist_losses_e = [] 61 | start_time = time.time() 62 | gutils.update_lr(epoch, g_opt, d_opt, 63 | self.args.gan_schedule, 64 | self.args.gan_gammas) 65 | 66 | #Iterate over examples that the classifier Trainer just iterated on 67 | for batch_idx, (image, label) in enumerate(self.train_iterator): 68 | #Handle smaller last batch size 69 | batch_size = image.shape[0] 70 | #Save one real sample each epoch 71 | if not one_sample_saved: 72 | gutils.save_results(self.args, image, "sample_E" + str(epoch), True, 73 | np.sqrt(self.args.batch_size), 74 | self.experiment) 75 | one_sample_saved = True 76 | 77 | #Make vectors of ones and zeros of same shape as output by 78 | #Discriminator so that it can be used in BCELoss 79 | smoothing_val = 0 80 | if self.args.label_smoothing: 81 | smoothing_val = 0.1 82 | d_like_real = torch.ones(batch_size) - smoothing_val 83 | d_like_fake = torch.zeros(batch_size) 84 | if self.args.cuda: 85 | d_like_real = Variable(d_like_real.cuda()) 86 | d_like_fake = Variable(d_like_fake.cuda()) 87 | 88 | #-------------Train Discriminator-----------# 89 | ##Train using real images 90 | D.zero_grad() 91 | a = a + 1 92 | #Shape [batch_size, 10, 32, 32]. Each entry at d_labels[0] 93 | #contains 32x32 matrix of 1s inside d_labels[label] index 94 | #and 32x32 matrix of 0s otherwise 95 | d_labels = d_vec[label] 96 | if self.args.cuda: 97 | image = Variable(image.cuda()) 98 | d_labels = Variable(d_labels.cuda()) 99 | #Discriminator output for real image and labels 100 | d_output_real = D(image, d_labels).squeeze() 101 | 102 | ##Train using fake images 103 | g_random_noise = torch.randn((batch_size, 100)) 104 | g_random_noise = g_random_noise.view(-1, 100, 1, 1) 105 | #Generating random batch_size of labels from those present in activeClass 106 | random_labels = torch.from_numpy(np.random.choice(active_classes, batch_size)) 107 | #Convert labels to appropriate shapes 108 | g_random_labels = g_vec[random_labels] 109 | d_random_labels = d_vec[random_labels] 110 | if self.args.cuda: 111 | g_random_noise = Variable(g_random_noise.cuda()) 112 | g_random_labels = Variable(g_random_labels.cuda()) 113 | d_random_labels = Variable(d_random_labels.cuda()) 114 | #Generating fake images and passing them to discriminator 115 | g_output = G(g_random_noise, g_random_labels) 116 | #Detach gradient from Generator 117 | g_output = g_output.detach() 118 | d_output_fake = D(g_output, d_random_labels).squeeze() 119 | 120 | #Calculate BCE loss 121 | d_real_loss = criterion(d_output_real, d_like_real) 122 | d_fake_loss = criterion(d_output_fake, d_like_fake) 123 | d_loss = d_real_loss + d_fake_loss 124 | 125 | #Perform a backward step 126 | d_losses_e.append(d_loss.cpu().data.numpy()) 127 | d_loss.backward() 128 | d_opt.step() 129 | 130 | #-------------Train Generator-----------# 131 | #TODO Disabled regenerating of noise, check if it still works 132 | b = b + 1 133 | G.zero_grad() 134 | g_output = G(g_random_noise, g_random_labels) 135 | d_output = D(g_output, d_random_labels).squeeze() 136 | 137 | #Calculate BCE loss 138 | g_loss = criterion(d_output, d_like_real) 139 | total_loss = g_loss 140 | 141 | #Jointly optimizes the GAN Generator loss and lowers 142 | #the euclidean distance between features of generated 143 | #and real images 144 | if self.args.joint_gan_obj: 145 | model = self.fixed_classifier 146 | euclidean_dist = nn.PairwiseDistance(2) 147 | # Generate features for real and fake images 148 | output_fake = model.forward(g_output, True) 149 | output_real = model.forward(image, True) 150 | # Calculate euclidean distance 151 | distance_loss = torch.mean(euclidean_dist(output_fake, output_real)) 152 | total_loss = g_loss + (self.args.joint_gan_alpha * distance_loss) 153 | dist_losses_e.append(distance_loss.cpu().data.numpy()) 154 | 155 | #Backward step 156 | total_loss.backward() 157 | g_losses_e.append(g_loss.cpu().data.numpy()) 158 | g_opt.step() 159 | 160 | #-------------End Epoch-----------# 161 | #Print Stats and save results 162 | mean_g = (sum(g_losses_e)/len(g_losses_e)) 163 | mean_d = (sum(d_losses_e)/len(d_losses_e)) 164 | mean_dist = None 165 | if self.args.joint_gan_obj: 166 | mean_dist = (sum(dist_losses_e)/len(dist_losses_e)) 167 | g_losses.append(mean_g) 168 | d_losses.append(mean_d) 169 | 170 | if epoch % self.args.gan_img_save_interval == 0: 171 | #Generate some examples for visualizing 172 | gutils.generate_examples(self.args, G, 100, 173 | active_classes, 174 | self.total_classes, 175 | self.fixed_noise, 176 | self.experiment, 177 | "Inc" + str(increment) + "_E" + str(epoch), 178 | True, True) 179 | #Plot GAN losses 180 | gutils.save_gan_losses(g_losses, d_losses, 181 | self.args.gan_epochs[increment], 182 | increment, 183 | self.experiment) 184 | 185 | #Save ckpt if interval is satisfied 186 | if self.args.save_g_ckpt and epoch % self.args.ckpt_interval == 0: 187 | gutils.save_checkpoint(epoch, increment, self.experiment, G) 188 | 189 | print("[GAN] Epoch:", epoch, 190 | "G_iters:", b, 191 | "D_iters:", a, 192 | "g_loss:", mean_g, 193 | "d_loss:", mean_d, 194 | "dist_Loss:", mean_dist, 195 | "Time taken:", time.time() - start_time) 196 | -------------------------------------------------------------------------------- /runExperiment.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import argparse 4 | 5 | import torch 6 | import torch.optim as optim 7 | import torch.utils.data as td 8 | 9 | import dataHandler.datasetFactory as dF 10 | import dataHandler.incrementalLoader as dL 11 | import experiment.experiment as ex 12 | import model.modelFactory as mF 13 | import trainer.classifierTrainer as t 14 | import trainer.nmcTrainer as nt 15 | import trainer.ganTrainer as gt 16 | 17 | parser = argparse.ArgumentParser(description='iCarl2.0') 18 | parser.add_argument('--batch-size', type=int, default=64, metavar='N', 19 | help='input batch size for training (default: 64)') 20 | parser.add_argument('--test-batch-size', type=int, default=128, metavar='N', 21 | help='input batch size for testing (default: 128)') 22 | parser.add_argument('--epochs', type=int, default=200, metavar='N', 23 | help='number of epochs to train (default: 200)') 24 | parser.add_argument('--lr', type=float, default=2.0, metavar='LR', 25 | help='learning rate (default: 0.1)') 26 | parser.add_argument('--schedule', type=int, nargs='+', default=[49, 63], 27 | help='Decrease learning rate at these epochs.') 28 | parser.add_argument('--gammas', type=float, nargs='+', default=[0.2, 0.2], 29 | help='LR is multiplied by gamma on schedule, number of gammas should be equal to schedule') 30 | 31 | parser.add_argument('--momentum', type=float, default=0.9, metavar='M', 32 | help='SGD momentum (default: 0.9)') 33 | parser.add_argument('--no-cuda', action='store_true', default=False, 34 | help='disables CUDA training') 35 | parser.add_argument('--no-distill', action='store_true', default=False, 36 | help='disable distillation loss') 37 | parser.add_argument('--no-random', action='store_true', default=False, 38 | help='Disable random shuffling of classes') 39 | parser.add_argument('--no-herding', action='store_true', default=False, 40 | help='Disable herding for NMC') 41 | parser.add_argument('--seed', type=int, default=1, metavar='S', 42 | help='random seed (default: 1)') 43 | parser.add_argument('--log-interval', type=int, default=2, metavar='N', 44 | help='how many batches to wait before logging training status') 45 | parser.add_argument('--model-type', default="resnet32", 46 | help='model type to be used. Example : resnet32, resnet20, densenet, test') 47 | parser.add_argument('--name', default="noname", 48 | help='Name of the experiment') 49 | parser.add_argument('--sortby', default="none", 50 | help='Examplars sorting strategy') 51 | parser.add_argument('--decay', type=float, default=0.00004, help='Weight decay (L2 penalty).') 52 | parser.add_argument('--step-size', type=int, default=2, help='How many classes to add in each increment') 53 | parser.add_argument('--memory-budget', type=int, default=2000, help='How many images can we store at max') 54 | parser.add_argument('--epochs-class', type=int, default=70, help='Number of epochs for each increment') 55 | parser.add_argument('--dataset', default="CIFAR100", help='dataset to be used; example CIFAR, MNIST') 56 | parser.add_argument('--no-upsampling', action='store_true', default=False, 57 | help='Do not do upsampling.') 58 | parser.add_argument('--process', default="nmc", help='Process to be used to prevent forgetting; Example: nmc, cdcgan, dcgan, wgan') 59 | 60 | parser.add_argument('--gan-epochs', type=int, nargs='+', default=[50, 30, 20, 20, 20], help='Epochs for each increment for training the GANs') 61 | parser.add_argument('--gan-d', type=int, default=64, help='GAN feature size multiplier') 62 | parser.add_argument('--gan-lr', type=float, default=0.0002, help='Learning Rate for training the GANs') 63 | parser.add_argument('--gan-batch-size', type=int, default=128, help='Batch Size for training the GANs') 64 | parser.add_argument('--gan-num-examples', type=int, default=1000, help='Number examples GAN will generate for each class') 65 | parser.add_argument('--gan-schedule', type=int, nargs='+', default=[11, 16], 66 | help='Decrease GAN learning rate at these epochs.') 67 | parser.add_argument('--gan-gammas', type=float, nargs='+', default=[0.1, 0.1], 68 | help='LR is multiplied by gamma on schedule, number of gammas should be equal to schedule') 69 | parser.add_argument('--persist-gan', action='store_true', default=False, 70 | help='GAN is not thrown away and trained from scratch each increment') 71 | parser.add_argument('--gan-img-save-interval', type=int, default=100, help='Save generator samples every x epochs') 72 | parser.add_argument('--d-iter', type=int, default=1, help='Number of iterations of discriminatori/critic for each iteration of generator.') 73 | parser.add_argument('--ckpt-interval', type=int, default=1000, help='After how many epochs should the Generator be saved') 74 | parser.add_argument('--load-g-ckpt', default="", help='Path to folder which contains Generator ckpts') 75 | parser.add_argument('--T', type=int, default=1, help='Temperature for Distillation') 76 | parser.add_argument('--save-g-ckpt', default=False, action='store_true', help='Whether the Generator ckpt will be saved or not') 77 | parser.add_argument('--gan-save-classes', default=10, type=int, help='Number of classes whose images will be saved every gan-img-save-interval') 78 | parser.add_argument('--label-smoothing', default=False, action='store_true', help='Whether to use one sided label smoothing in GAN') 79 | parser.add_argument('--minibatch-discrimination', default=False, action='store_true', help='Whether to use minibatch discrimination layer') 80 | parser.add_argument('--ideal-nmc', default=False, action='store_true', help='Whether to calculate ideal nmc') 81 | parser.add_argument('--optimize-features', default=False, action='store_true', help='Whether to minimize the distance between generated and real embeddings') 82 | parser.add_argument('--optimize-feat-epochs', type=int, default=20, help='How many epochs to run optimize-features for') 83 | parser.add_argument('--optimize-feat-lr', type=float, default=0.00001, help='lr for optimize-features') 84 | parser.add_argument('--joint-gan-obj', default=False, action='store_true', help='Whether to jointly train GAN and minimize the logit distance') 85 | parser.add_argument('--joint-gan-alpha', type=float, default=1, help='Contribution of logit distance minimizer in GAN loss') 86 | parser.add_argument('--ac-distill', default=False, action='store_true', help='Whether to use ACGAN\'s discriminator outputs in distillation') 87 | parser.add_argument('--filter-using-disc', default=False, action='store_true', help='Whether to use discriminator to filter generated samples') 88 | parser.add_argument('--filter-val', type=float, default=0.5, help='Value to be used when filtering using discriminator') 89 | parser.add_argument('--disc-eval', default=False, action='store_true', help='Whether to evaluate discriminator as a classifier (ACGAN only)') 90 | 91 | args = parser.parse_args() 92 | 93 | if args.process == "gan" and args.dataset == "MNIST" and len(args.gan_epochs) < 10//args.step_size: 94 | print("ERROR: Number of values in gan-epochs must be greater than number of increments") 95 | assert False 96 | 97 | if args.process == "wgan" and args.d_iter == 1: 98 | print("NOTICE: Recommended to set --d_iter to 5 for WGAN") 99 | 100 | if not args.minibatch_discrimination: 101 | print("NOTICE: Not using minibatch discrimination, is this intended?") 102 | 103 | print("Remember: Set decay 10x smaller on MNIST, was performing better") 104 | 105 | args.cuda = not args.no_cuda and torch.cuda.is_available() 106 | 107 | dataset = dF.DatasetFactory.get_dataset(args.dataset) 108 | torch.manual_seed(args.seed) 109 | if args.cuda: 110 | torch.cuda.manual_seed(args.seed) 111 | torch.backends.cudnn.deterministic = True 112 | 113 | 114 | train_dataset_loader = dL.IncrementalLoader(args.dataset, dataset.train_data.train_data, 115 | dataset.train_data.train_labels, 116 | dataset.labels_per_class_train, 117 | dataset.classes, [], transform=dataset.train_transform, 118 | cuda=args.cuda, oversampling=args.no_upsampling, 119 | alt_transform=dataset.alt_transform 120 | ) 121 | 122 | test_dataset_loader = dL.IncrementalLoader(args.dataset, dataset.test_data.test_data, 123 | dataset.test_data.test_labels, 124 | dataset.labels_per_class_test, dataset.classes, 125 | [], transform=dataset.test_transform, cuda=args.cuda, 126 | oversampling=args.no_upsampling 127 | ) 128 | 129 | train_dataset_loader_ideal = None 130 | if args.ideal_nmc: 131 | train_dataset_loader_ideal = dL.IncrementalLoader(args.dataset, dataset.train_data.train_data, 132 | dataset.train_data.train_labels, 133 | dataset.labels_per_class_train, 134 | dataset.classes, [], transform=dataset.train_transform, 135 | cuda=args.cuda, oversampling=args.no_upsampling 136 | ) 137 | 138 | 139 | kwargs = {'num_workers': 0, 'pin_memory': True} if args.cuda else {} 140 | 141 | train_iterator = torch.utils.data.DataLoader(train_dataset_loader, 142 | batch_size=args.batch_size, shuffle=True, **kwargs) 143 | test_iterator = torch.utils.data.DataLoader(test_dataset_loader, 144 | batch_size=args.test_batch_size, shuffle=True, **kwargs) 145 | 146 | train_iterator_ideal = None 147 | if args.ideal_nmc: 148 | train_iterator_ideal = torch.utils.data.DataLoader(train_dataset_loader_ideal, 149 | batch_size=args.batch_size, shuffle=True, **kwargs) 150 | 151 | my_factory = mF.ModelFactory() 152 | model = my_factory.get_model(args.model_type, args.dataset) 153 | if args.cuda: 154 | model.cuda() 155 | 156 | my_experiment = ex.Experiment(args.name, args) 157 | 158 | optimizer = optim.SGD(model.parameters(), args.lr, momentum=args.momentum, 159 | weight_decay=args.decay, nesterov=True) 160 | 161 | classifier_trainer = t.Trainer(train_iterator, test_iterator, dataset, model, 162 | args, optimizer, train_iterator_ideal) 163 | 164 | if args.process == "nmc": 165 | trainer = nt.Trainer(args, dataset, classifier_trainer, model, 166 | train_iterator, test_iterator, train_dataset_loader, 167 | my_experiment) 168 | 169 | else: 170 | trainer = gt.Trainer(args, dataset, classifier_trainer, model, 171 | train_iterator, test_iterator, train_dataset_loader, 172 | my_factory, my_experiment, train_iterator_ideal, train_dataset_loader_ideal) 173 | 174 | trainer.train() 175 | -------------------------------------------------------------------------------- /trainer/ganTrainer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import copy 3 | import time 4 | import itertools 5 | 6 | import torch 7 | import numpy as np 8 | import torch.nn as nn 9 | import utils.utils as ut 10 | import torch.optim as optim 11 | import torch.utils.data as td 12 | import matplotlib.pyplot as plt 13 | from torch.autograd import Variable 14 | 15 | import trainer.classifierTrainer as t 16 | import trainer.classifierFactory as tF 17 | import trainer.gans.gutils as gutils 18 | from trainer.gans.ganFactory import GANFactory 19 | 20 | class Trainer(): 21 | def __init__( 22 | self, args, dataset, classifier_trainer, model, train_iterator, 23 | test_iterator, train_loader, model_factory, experiment, 24 | train_iterator_ideal, train_loader_ideal): 25 | ''' 26 | ideal params are for ideal nmc calculation 27 | ''' 28 | self.args = args 29 | self.batch_size = args.batch_size 30 | self.dataset = dataset 31 | self.classifier_trainer = classifier_trainer 32 | self.model = model 33 | self.train_iterator = train_iterator 34 | self.test_iterator = test_iterator 35 | self.train_loader = train_loader 36 | self.model_factory = model_factory 37 | self.experiment = experiment 38 | self.old_classes = None 39 | self.G = None 40 | self.D = None 41 | self.fixed_g = None 42 | self.examples = {} 43 | self.increment = 0 44 | self.is_cond = (args.process == "cdcgan" or args.process == "acgan") 45 | if args.ideal_nmc: 46 | self.train_iterator_ideal = train_iterator_ideal 47 | self.train_loader_ideal = train_loader_ideal 48 | if args.dataset == "MNIST" or args.dataset == "CIFAR10": 49 | self.num_classes = 10 50 | else: 51 | self.num_classes = 100 52 | 53 | def train(self): 54 | x = [] 55 | y = [] 56 | z = [] 57 | y_nmc = [] 58 | y_nmc_ideal = [] 59 | 60 | #Get NMC and NMC ideal (if required) 61 | test_factory = tF.ClassifierFactory() 62 | nmc = test_factory.get_tester("nmc", self.args.cuda) 63 | if self.args.ideal_nmc: 64 | ideal_nmc = test_factory.get_tester("nmc", self.args.cuda) 65 | 66 | #Get the appropriate GAN trainer 67 | gan_trainer = GANFactory.get_trainer(self.args.process, 68 | self.args, 69 | self.num_classes, 70 | self.train_iterator, 71 | self.classifier_trainer.model_fixed, 72 | self.experiment) 73 | 74 | for class_group in range(0, self.dataset.classes, self.args.step_size): 75 | #Setup training and increment classes 76 | self.classifier_trainer.setup_training() 77 | self.classifier_trainer.increment_classes(class_group) 78 | #If not first increment, then generate examples and replace data 79 | if class_group > 0: 80 | self.increment = self.increment + 1 81 | self.old_classes = self.classifier_trainer.older_classes 82 | self.examples = gutils.generate_examples(self.args, 83 | self.fixed_g, 84 | self.args.gan_num_examples, 85 | self.old_classes, 86 | self.num_classes, 87 | None, 88 | self.experiment, 89 | "Final-Inc"+str(self.increment-1), 90 | True, self.is_cond, self.D) 91 | #TODO put trainLoader 92 | self.train_iterator.dataset.replace_data(self.examples, 93 | self.args.gan_num_examples) 94 | # Send examples to CPU 95 | if self.is_cond: 96 | for k in self.examples: 97 | self.examples[k] = self.examples[k].data.cpu() 98 | 99 | #-------------Train GAN-----------# 100 | #Get new G and D only if it doesn't exist and persist_gan is off 101 | if self.G == None or not self.args.persist_gan: 102 | self.G, self.D = self.model_factory.get_model(self.args.process, 103 | self.args.dataset, 104 | self.args.minibatch_discrimination, 105 | self.args.gan_d) 106 | if self.args.cuda: 107 | self.G = self.G.cuda() 108 | self.D = self.D.cuda() 109 | 110 | #Load if we should be loading from ckpt, otherwise train 111 | is_loaded = False 112 | if self.args.load_g_ckpt != '': 113 | D = self.D if self.args.process == "acgan" else None 114 | is_loaded = gutils.load_checkpoint(self.args.load_g_ckpt, 115 | self.increment, 116 | self.G, D) 117 | #Train the GAN, use alternative transform 118 | if not is_loaded: 119 | self.train_loader.do_alt_transform = True 120 | gan_trainer.train(self.G, 121 | self.D, 122 | self.train_iterator.dataset.active_classes, 123 | self.increment) 124 | self.train_loader.do_alt_transform = False 125 | 126 | #Optimize features, default off 127 | if self.args.optimize_features: 128 | self.optimize_features() 129 | self.update_frozen_generator() 130 | 131 | #Save ckpt if required 132 | if self.args.save_g_ckpt: 133 | gutils.save_checkpoint(self.args.gan_epochs[self.increment], 134 | self.increment, self.experiment, 135 | self.G, self.D) 136 | 137 | #-------------Train Classifier-----------# 138 | epoch = 0 139 | for epoch in range(0, self.args.epochs_class): 140 | self.classifier_trainer.update_lr(epoch) 141 | self.classifier_trainer.train(self.examples, self.old_classes, 142 | self.batch_size, self.D, epoch) 143 | if epoch % self.args.log_interval == 0: 144 | print("[Classifier] Train:", 145 | self.classifier_trainer.evaluate(self.train_iterator), 146 | "Test:", 147 | self.classifier_trainer.evaluate(self.test_iterator)) 148 | 149 | self.classifier_trainer.update_frozen_model() 150 | 151 | #-------------Using NMC Classifier-----------# 152 | nmc.update_means(self.model, self.train_iterator, self.args.cuda, 153 | self.dataset.classes, self.old_classes, self.is_cond) 154 | nmc_train = nmc.classify(self.model, self.train_iterator, 155 | self.args.cuda, True) 156 | nmc_test = nmc.classify(self.model, self.test_iterator, 157 | self.args.cuda, True) 158 | y_nmc.append(nmc_test) 159 | 160 | if self.args.ideal_nmc: 161 | ideal_nmc.update_means(self.model, self.train_iterator_ideal, self.args.cuda, 162 | self.dataset.classes, [], True) 163 | nmc_test_ideal = ideal_nmc.classify(self.model, self.test_iterator, 164 | self.args.cuda, True) 165 | y_nmc_ideal.append(nmc_test_ideal) 166 | 167 | print("Train NMC: ", nmc_train) 168 | print("Test NMC: ", nmc_test) 169 | if self.args.ideal_nmc: 170 | print("Test NMC (Ideal)", nmc_test_ideal) 171 | 172 | 173 | #-------------Save and Plot data-----------# 174 | #Saving confusion matrix 175 | ut.saveConfusionMatrix(int(class_group / self.args.step_size) * 176 | self.args.epochs_class + epoch, 177 | self.experiment.path + "CONFUSION", 178 | self.model, self.args, self.dataset, 179 | self.test_iterator) 180 | ut.get_confusion_matrix_nmc(self.experiment.path + "CONFUSION_NMC", 181 | self.model, self.test_iterator, 182 | self.num_classes, self.args, nmc.means, 183 | int(class_group / self.args.step_size) 184 | * self.args.epochs_class + epoch) 185 | 186 | #Plot 187 | y.append(self.classifier_trainer.evaluate(self.test_iterator)) 188 | x.append(class_group + self.args.step_size) 189 | results = [("Trained Classifier",y), ("NMC Classifier", y_nmc)] 190 | if self.args.disc_eval: 191 | z.append(self.classifier_trainer.evaluate(self.test_iterator, self.D)) 192 | results.append(("Auxiliary Classifier", z)) 193 | if self.args.ideal_nmc: 194 | results.append(("Ideal NMC Classifier", y_nmc_ideal)) 195 | ut.plotEmbeddings(self.experiment, 196 | [("NMC_means", nmc.means ), ("Ideal_NMC_means", ideal_nmc.means)], 197 | "Inc"+str(self.increment)) 198 | ut.plotAccuracy(self.experiment, x, 199 | results, 200 | self.dataset.classes + 1, self.args.name) 201 | 202 | def update_frozen_generator(self): 203 | #self.G.eval() 204 | self.fixed_g = copy.deepcopy(self.G) 205 | for param in self.fixed_g.parameters(): 206 | param.requires_grad = False 207 | 208 | def unfreeze_frozen_generator(self): 209 | self.G.train() 210 | for param in self.G.parameters(): 211 | param.requires_grad = True 212 | 213 | def optimize_features(self): 214 | ''' 215 | Attempts to reduce the Euclidean distance between 216 | the batches of features of generated and real images 217 | ''' 218 | self.unfreeze_frozen_generator() 219 | model = self.classifier_trainer.model_fixed 220 | optimizer = optim.Adam(self.G.parameters(), lr=self.args.optimize_feat_lr, betas=(0.5, 0.999)) 221 | euclidean_dist = nn.PairwiseDistance(2) 222 | print("Optimizing features") 223 | for epoch in range(self.args.optimize_feat_epochs): 224 | losses = [] 225 | start_time = time.time() 226 | for batch_idx, (image, label) in enumerate(self.train_iterator): 227 | batch_size = image.shape[0] 228 | # Generate noise 229 | g_random_noise = torch.randn((batch_size, 100)) 230 | g_random_noise = g_random_noise.view(-1, 100, 1, 1) 231 | #TODO wrap in variable for noncuda 232 | if self.args.cuda: 233 | image = Variable(image.cuda()) 234 | g_random_noise = Variable(g_random_noise.cuda()) 235 | # Generate examples 236 | g_output = self.G(g_random_noise) 237 | # Generate features for real and fake images 238 | output_fake = model.forward(g_output, True) 239 | output_real = model.forward(image, True) 240 | # Calculate euclidean distance 241 | loss = torch.mean(euclidean_dist(output_fake, output_real)) 242 | loss.backward() 243 | optimizer.step() 244 | losses.append(loss) 245 | 246 | # Calculate mean loss, save examples and print stats 247 | mean_loss = (sum(losses)/len(losses)).cpu().data.numpy()[0] 248 | if epoch % self.args.gan_img_save_interval == 0: 249 | self.generate_examples(self.G, 100, self.train_iterator.dataset.active_classes, 250 | "OPT-Inc"+str(self.increment) + 251 | "_E" + str(epoch), True) 252 | print("[GAN-OPTIMIZE] Epoch:", epoch, 253 | "Loss:", mean_loss, 254 | "Time taken:", time.time() - start_time) 255 | self.update_frozen_generator() 256 | 257 | -------------------------------------------------------------------------------- /dataHandler/incrementalLoader.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | import numpy as np 4 | import torch 5 | import torch.utils.data as td 6 | import torchvision 7 | from PIL import Image 8 | from torch.autograd import Variable 9 | from torchvision import datasets, transforms 10 | from skimage.transform import resize 11 | import model.modelFactory as mF 12 | 13 | 14 | class IncrementalLoader(td.Dataset): 15 | def __init__(self, dataset_name, data, labels, class_size, classes, active_classes, transform=None, cuda=False, oversampling=True, alt_transform=None): 16 | self.len = class_size * len(active_classes) 17 | self.dataset_name = dataset_name 18 | sort_index = np.argsort(labels) 19 | self.class_size = class_size 20 | if "torch" in str(type(data)): 21 | data = data.numpy() 22 | self.data = data[sort_index] 23 | labels = np.array(labels) 24 | self.labels = labels[sort_index] 25 | self.transform = transform 26 | self.active_classes = active_classes 27 | self.limited_classes = {} 28 | self.total_classes = classes 29 | self.means = {} 30 | self.cuda = cuda 31 | self.weights = np.zeros(self.total_classes * self.class_size) 32 | self.class_indices() 33 | self.transform_data() 34 | self.over_sampling = oversampling 35 | self.alt_transform = alt_transform 36 | self.do_alt_transform = False 37 | 38 | 39 | def transform_data(self): 40 | ''' 41 | Rescale the dataset to 32x32 42 | TODO: Complete all the transformations here instead of in __getItem__ 43 | ''' 44 | if not self.dataset_name == "MNIST": 45 | return 46 | temp_data = np.ndarray([self.data.shape[0], 32, 32]) 47 | self.data = np.expand_dims(self.data, axis=3) 48 | for i in range(len(self.data)): 49 | temp_data[i] = transforms.Scale(32)(transforms.ToPILImage()(self.data[i])) 50 | self.data = temp_data 51 | 52 | 53 | def class_indices(self): 54 | self.indices = {} 55 | cur = 0 56 | for temp in range(0, self.total_classes): 57 | cur_len = len(np.nonzero(np.uint8(self.labels == temp))[0]) 58 | self.indices[temp] = (cur, cur + cur_len) 59 | cur += cur_len 60 | 61 | def add_classes(self, n): 62 | if n in self.active_classes: 63 | return 64 | self.active_classes.append(n) 65 | self.len = self.class_size * len(self.active_classes) 66 | self.update_len() 67 | 68 | def replace_data(self, data, k): 69 | ''' 70 | Code to replace images with GAN generated images 71 | data: Generated images with values in range [-1,1] and of 72 | shape [C x W x H] 73 | k : Number of images to replace per class 74 | ''' 75 | print ("Replacing data") 76 | for a in data: 77 | nump = data[a].data.squeeze().cpu().numpy() 78 | #nump = resize(new_data, (k, 28, 28), anti_aliasing=True, preserve_range=True) 79 | 80 | #Converting from [-1,1] range to [0,255] because that is what 81 | #toTensor transform expects 82 | nump = (((nump/2) + 0.5) * 255).astype(np.uint8) 83 | if self.dataset_name == "CIFAR100" or self.dataset_name == "CIFAR10": 84 | #TODO I think .transpose or .permute does this in one line? 85 | nump = np.swapaxes(nump, 1, 3) 86 | nump = np.swapaxes(nump, 1, 2) 87 | self.data[self.indices[a][0]:self.indices[a][0]+k] = nump 88 | 89 | if a not in self.active_classes: 90 | self.active_classes.append(a) 91 | self.limit_class(a, k) 92 | 93 | 94 | def update_len(self): 95 | ''' 96 | Function to compute length of the active elements of the data. 97 | :return: 98 | ''' 99 | # Computing len if no oversampling 100 | # for a in self.active_classes: 101 | # if a in self.limited_classes: 102 | # self.weights[len_var:len_var + min(self.class_size, self.limited_classes[a])] = 1.0 / float( 103 | # self.limited_classes[a]) 104 | # if self.class_size > self.limited_classes[a]: 105 | # self.weights[len_var + self.limited_classes[a]:len_var + self.class_size] = 0 106 | # len_var += min(self.class_size, self.limited_classes[a]) 107 | # 108 | # else: 109 | # self.weights[len_var:len_var + self.class_size] = 1.0 / float(self.class_size) 110 | # len_var += self.class_size 111 | # 112 | # self.len = len_var 113 | # Computing len if oversampling is turned on. 114 | 115 | len_var = 0 116 | for a in self.active_classes: 117 | len_var += self.indices[a][1] - self.indices[a][0] 118 | self.len = len_var 119 | 120 | return 121 | 122 | def limit_class(self, n, k): 123 | if k == 0: 124 | self.remove_class(n) 125 | #print("Removed class", n) 126 | #print("Current classes", self.active_classes) 127 | return False 128 | if k > self.class_size: 129 | k = self.class_size 130 | if n in self.limited_classes: 131 | self.limited_classes[n] = k 132 | # Remove this line; this turns off oversampling 133 | if not self.over_sampling: 134 | self.indices[n] = (self.indices[n][0], self.indices[n][0] + k) 135 | self.update_len() 136 | return False 137 | else: 138 | if not self.over_sampling: 139 | self.indices[n] = (self.indices[n][0], self.indices[n][0] + k) 140 | self.limited_classes[n] = k 141 | self.update_len() 142 | return True 143 | 144 | def remove_class(self, n): 145 | while n in self.active_classes: 146 | self.active_classes.remove(n) 147 | self.update_len() 148 | 149 | 150 | def limit_class_and_sort(self, n, k, model): 151 | ''' This function should only be called the first time a class is limited. To change the limitation, 152 | call the limiClass(self, n, k) function 153 | :param n: Class to limit 154 | :param k: No of exemplars to keep 155 | :param model: Features extracted from this model for sorting. 156 | :return: 157 | ''' 158 | 159 | if self.limit_class(n, k): 160 | start = self.indices[n][0] 161 | end = self.indices[n][1] 162 | buff = np.zeros(self.data[start:end].shape) 163 | images = [] 164 | # Get input features of all the images of the class 165 | for ind in range(start, end): 166 | img = self.data[ind] 167 | if "torch" in str(type(img)): 168 | img = img.numpy() 169 | img = Image.fromarray(img) 170 | 171 | if self.transform is not None: 172 | img = self.transform(img) 173 | images.append(img) 174 | data_tensor = torch.stack(images) 175 | if self.cuda: 176 | data_tensor = data_tensor.cuda() 177 | 178 | # Get features 179 | features = model.forward(Variable(data_tensor), True) 180 | features_copy = copy.deepcopy(features.data) 181 | mean = torch.mean(features, 0, True) 182 | list_of_selected = [] 183 | 184 | # Select exemplars 185 | for exmp_no in range(0, min(k, self.class_size)): 186 | if exmp_no > 0: 187 | to_add = torch.sum(features_copy[0:exmp_no], dim=0).unsqueeze(0) 188 | if self.cuda: 189 | to_add = to_add.cuda() 190 | features_temp = (features + Variable(to_add)) / (exmp_no + 1) - mean 191 | else: 192 | features_temp = features - mean 193 | features_norm = torch.norm(features_temp.data, 2, dim=1) 194 | # features_norm = features_temp.norm(dim=1) 195 | if self.cuda: 196 | features_norm = features_norm.cpu() 197 | arg_min = np.argmin(features_norm.numpy()) 198 | if arg_min in list_of_selected: 199 | assert (False) 200 | list_of_selected.append(arg_min) 201 | buff[exmp_no] = self.data[start + arg_min] 202 | features_copy[exmp_no] = features.data[arg_min] 203 | # print (features_copy[exmp_no]) 204 | features[arg_min] = features[arg_min] + 1000 205 | print("Exmp shape", buff[0:min(k, self.class_size)].shape) 206 | self.data[start:start + min(k, self.class_size)] = buff[0:min(k, self.class_size)] 207 | 208 | self.update_len() 209 | 210 | def __len__(self): 211 | return self.len 212 | 213 | def get_start_index(self, n): 214 | ''' 215 | :param n: 216 | :return: Returns starting index of classs n 217 | ''' 218 | return self.indices[n][0] 219 | 220 | def __getitem__(self, index): 221 | ''' 222 | Replacing this with a more efficient implemnetation selection; removing c 223 | :param index: 224 | :return: 225 | ''' 226 | assert (index < self.class_size * self.total_classes) 227 | 228 | len = 0 229 | temp_a = 0 230 | old_len = 0 231 | for a in self.active_classes: 232 | temp_a = a 233 | old_len = len 234 | len += self.indices[a][1] - self.indices[a][0] 235 | if len > index: 236 | break 237 | base = self.indices[temp_a][0] 238 | incre = index - old_len 239 | if temp_a in self.limited_classes: 240 | incre = incre % self.limited_classes[temp_a] 241 | index = base + incre 242 | img = self.data[index] 243 | if "torch" in str(type(img)): 244 | img = img.numpy() 245 | img = Image.fromarray(img) 246 | 247 | #if self.data.shape[0] == 60000: 248 | if self.dataset_name == "MNIST": 249 | img = np.expand_dims(img, axis=2) 250 | 251 | if (not (self.do_alt_transform and self.alt_transform is not None)) and self.transform is not None: 252 | img = self.transform(img) 253 | else: 254 | img = self.alt_transform(img) 255 | 256 | if not self.labels[index] in self.active_classes: 257 | print("Active classes", self.active_classes) 258 | print("Label ", self.labels[index]) 259 | assert (False) 260 | 261 | return img, self.labels[index] 262 | 263 | def sort_by_importance(self, algorithm="Kennard-Stone"): 264 | if algorithm == "LDIS": 265 | data_file = "dataHandler/selectedCIFARIndicesForTrainingDataK1.txt" 266 | elif algorithm == "Kennard-Stone": 267 | data_file = "dataHandler/selectedCIFARIndicesForTrainingDataKenStone.txt" 268 | 269 | # load sorted (training) data indices 270 | lines = [line.rstrip('\n') for line in open(data_file)] 271 | sorted_data = [] 272 | 273 | # iterate for each class 274 | h = 0 275 | class_num = 0 276 | for line in lines: 277 | line = line[(line.find(":") + 1):] 278 | # select instances based on priority 279 | prioritized_indices = line.split(",") 280 | for index in prioritized_indices: 281 | sorted_data.append(self.data[int(index)]) 282 | # select remaining instances 283 | for i in range(class_num * self.class_size, (class_num + 1) * self.class_size): 284 | if str(i) not in prioritized_indices: 285 | sorted_data.append(self.data[i]) 286 | h += 1 287 | class_num += 1 288 | self.data = np.concatenate(sorted_data).reshape(self.data.shape) 289 | 290 | def get_bottlenecks(self): 291 | pass 292 | 293 | 294 | if __name__ == "__main__": 295 | # To do : Remove the hard-coded mean and just compute it once using the data 296 | mean = [x / 255 for x in [125.3, 123.0, 113.9]] 297 | std = [x / 255 for x in [63.0, 62.1, 66.7]] 298 | 299 | train_transform = transforms.Compose( 300 | [transforms.RandomHorizontalFlip(), torchvision.transforms.ColorJitter(0.5, 0.5, 0.5, 0.5), 301 | transforms.RandomCrop(32, padding=6), torchvision.transforms.RandomRotation((-30, 30)), transforms.ToTensor(), 302 | transforms.Normalize(mean, std)]) 303 | 304 | train_data = datasets.CIFAR100("data", train=True, transform=train_transform, download=True) 305 | train_dataset_full = IncrementalLoader(train_data.train_data, train_data.train_labels, 500, 100, [], 306 | transform=train_transform) 307 | 308 | train_loader_full = torch.utils.data.DataLoader(train_dataset_full, 309 | batch_size=10, shuffle=True) 310 | my_factory = mF.ModelFactory() 311 | model = my_factory.get_model("test", 100) 312 | 313 | train_dataset_full.add_classes(2) 314 | train_dataset_full.limit_class_and_sort(2, 60, model) 315 | -------------------------------------------------------------------------------- /dataHandler/selectedCIFARIndicesForTrainingDataK1.txt: -------------------------------------------------------------------------------- 1 | Class#1:2,8,15,17,26,28,29,30,32,34,36,37,40,41,43,44,45,50,53,55,58,59,63,66,69,70,76,81,84,88,90,92,95,98,101,102,104,105,106,111,112,113,125,126,127,134,137,141,142,143,144,149,153,155,160,161,163,166,168,169,170,173,178,183,184,188,190,195,198,200,201,203,204,215,217,218,223,224,228,229,233,237,240,241,243,259,270,271,272,275,278,280,282,290,291,292,294,297,298,300,305,306,309,310,311,317,318,321,325,326,329,333,335,338,339,340,341,344,346,348,352,353,354,356,357,359,361,371,373,374,376,377,378,379,380,381,382,383,384,386,389,390,394,395,407,408,410,411,413,416,417,419,421,424,426,429,431,433,436,438,439,440,441,445,448,450,451,454,455,460,463,464,467,469,473,478,480,486,487,489,492,494,495,498 2 | Class#2:510,515,527,528,533,534,538,547,548,551,553,557,558,560,563,564,569,573,579,587,588,600,604,606,608,611,612,615,621,626,634,635,636,648,652,654,655,657,658,667,669,676,677,684,687,690,701,705,706,709,711,716,718,723,724,725,726,727,732,746,752,753,762,767,768,769,774,780,781,784,785,786,789,793,795,797,802,813,822,825,833,838,840,841,844,848,850,854,858,864,869,874,879,881,882,883,885,896,902,903,904,905,911,922,925,926,927,933,939,944,948,955,958,968,971,983,985,987,991,992,998 3 | Class#3:1000,1001,1003,1006,1011,1019,1021,1022,1031,1037,1042,1044,1049,1052,1058,1060,1063,1071,1072,1093,1095,1105,1108,1110,1116,1117,1118,1121,1122,1128,1129,1143,1145,1157,1161,1164,1166,1172,1176,1181,1192,1194,1195,1197,1198,1203,1208,1209,1212,1216,1217,1223,1231,1237,1241,1242,1243,1251,1253,1255,1261,1265,1270,1271,1274,1287,1288,1290,1292,1294,1298,1306,1309,1311,1312,1319,1326,1327,1329,1330,1336,1340,1363,1366,1371,1372,1378,1387,1390,1391,1392,1397,1412,1413,1414,1419,1420,1429,1430,1431,1432,1435,1439,1442,1443,1445,1446,1448,1454,1465,1469,1470,1475,1476,1477,1482,1483,1484,1485,1491,1494 4 | Class#4:1501,1505,1519,1521,1527,1532,1541,1543,1546,1557,1558,1560,1562,1563,1572,1574,1575,1580,1593,1596,1600,1606,1610,1619,1621,1627,1634,1637,1638,1644,1651,1656,1666,1672,1676,1679,1695,1702,1703,1716,1718,1719,1724,1726,1741,1743,1752,1763,1764,1769,1772,1788,1790,1798,1805,1806,1810,1812,1815,1822,1827,1832,1839,1847,1851,1853,1854,1856,1860,1862,1863,1867,1876,1883,1886,1894,1904,1907,1912,1915,1916,1918,1920,1925,1937,1939,1944,1948,1966,1979,1980,1983,1990,1995 5 | Class#5:2000,2001,2006,2011,2012,2020,2027,2028,2029,2037,2039,2044,2045,2050,2051,2052,2062,2063,2068,2069,2070,2081,2083,2087,2088,2092,2093,2096,2098,2100,2101,2112,2116,2118,2134,2136,2148,2149,2151,2152,2156,2159,2166,2168,2169,2173,2174,2177,2180,2191,2193,2196,2197,2199,2201,2203,2213,2215,2217,2219,2225,2227,2229,2231,2238,2239,2244,2245,2250,2258,2262,2275,2284,2290,2293,2295,2302,2305,2308,2310,2317,2318,2321,2322,2327,2333,2338,2345,2346,2350,2354,2360,2362,2366,2368,2370,2374,2375,2376,2381,2394,2398,2404,2405,2410,2413,2416,2422,2427,2429,2442,2444,2453,2454,2463,2464,2469,2470,2472,2475,2479,2483,2490,2493 6 | Class#6:2500,2504,2508,2510,2525,2526,2537,2540,2541,2543,2545,2547,2550,2551,2553,2555,2556,2559,2561,2567,2568,2582,2593,2600,2604,2605,2628,2629,2633,2637,2642,2643,2644,2650,2651,2669,2675,2676,2679,2680,2696,2700,2705,2707,2708,2711,2713,2717,2721,2732,2736,2752,2756,2758,2762,2764,2766,2776,2782,2800,2809,2813,2819,2823,2825,2826,2827,2835,2839,2840,2846,2851,2855,2856,2859,2862,2864,2866,2867,2874,2879,2881,2884,2892,2901,2907,2909,2911,2912,2918,2922,2927,2929,2934,2935,2939,2940,2943,2946,2953,2954,2965,2967,2969,2970,2971,2973,2974,2998 7 | Class#7:3000,3001,3005,3009,3012,3013,3017,3018,3019,3023,3024,3029,3035,3042,3046,3049,3064,3071,3074,3084,3085,3091,3093,3094,3095,3106,3109,3110,3112,3116,3117,3125,3126,3127,3128,3137,3139,3144,3146,3149,3153,3157,3158,3162,3169,3174,3191,3193,3194,3198,3212,3214,3220,3221,3225,3231,3235,3242,3247,3251,3256,3260,3262,3269,3272,3277,3284,3286,3288,3289,3290,3291,3295,3296,3299,3302,3307,3308,3311,3312,3317,3323,3324,3325,3327,3330,3331,3332,3338,3343,3344,3349,3355,3356,3361,3364,3368,3371,3372,3375,3377,3382,3391,3392,3394,3396,3401,3406,3408,3410,3413,3420,3421,3424,3427,3438,3449,3450,3454,3456,3458,3459,3461,3465,3470,3476,3480,3481,3485,3486,3490,3496,3497,3498 8 | Class#8:3502,3508,3511,3512,3514,3523,3530,3532,3533,3538,3541,3542,3548,3551,3555,3558,3562,3569,3572,3576,3578,3594,3597,3610,3615,3618,3623,3626,3628,3634,3640,3641,3642,3645,3647,3649,3658,3660,3661,3674,3678,3680,3684,3693,3698,3699,3700,3702,3703,3704,3709,3716,3720,3721,3730,3733,3737,3740,3742,3744,3753,3756,3760,3778,3791,3795,3796,3800,3803,3807,3808,3816,3821,3830,3832,3834,3836,3842,3845,3846,3847,3857,3858,3871,3873,3877,3879,3885,3886,3887,3891,3895,3908,3909,3915,3921,3930,3942,3947,3948,3950,3954,3956,3962,3963,3967,3973,3975,3978,3983,3988,3991,3992,3996 9 | Class#9:4001,4004,4006,4007,4014,4015,4023,4025,4028,4034,4036,4038,4039,4043,4044,4063,4069,4072,4075,4076,4084,4090,4107,4110,4113,4118,4123,4124,4125,4129,4133,4135,4140,4149,4151,4161,4163,4167,4179,4184,4187,4191,4192,4194,4204,4207,4213,4216,4223,4225,4228,4230,4232,4243,4245,4251,4252,4255,4257,4262,4266,4268,4270,4273,4277,4280,4284,4288,4296,4300,4301,4305,4309,4310,4312,4318,4331,4333,4334,4335,4339,4344,4352,4359,4361,4362,4365,4366,4374,4384,4385,4390,4395,4402,4408,4410,4418,4419,4426,4431,4443,4449,4453,4466,4472,4473,4477,4478,4484,4485,4499 10 | Class#10:4500,4501,4502,4504,4505,4507,4508,4509,4511,4513,4515,4518,4524,4525,4530,4531,4533,4536,4541,4544,4557,4560,4563,4568,4571,4572,4573,4578,4582,4585,4586,4588,4590,4592,4593,4594,4597,4599,4605,4607,4617,4619,4621,4626,4627,4634,4647,4648,4650,4651,4652,4656,4662,4663,4665,4667,4672,4676,4684,4688,4693,4695,4696,4697,4701,4702,4704,4707,4708,4713,4714,4726,4728,4729,4734,4737,4740,4741,4743,4744,4749,4752,4754,4756,4757,4761,4762,4767,4768,4769,4770,4774,4775,4776,4778,4782,4785,4786,4789,4795,4805,4807,4808,4809,4814,4815,4817,4820,4823,4825,4833,4835,4836,4837,4838,4840,4841,4843,4850,4851,4852,4856,4857,4860,4864,4869,4871,4873,4877,4878,4882,4884,4887,4891,4899,4908,4909,4910,4911,4913,4914,4916,4919,4920,4922,4923,4924,4927,4937,4941,4943,4947,4956,4963,4965,4966,4967,4974,4980,4981,4987,4991,4992,4993,4999 11 | Class#11:5000,5012,5014,5025,5031,5033,5035,5039,5041,5043,5047,5054,5065,5066,5070,5071,5072,5075,5078,5095,5097,5099,5102,5107,5110,5111,5113,5115,5124,5126,5130,5131,5132,5134,5135,5137,5142,5143,5148,5150,5152,5156,5157,5163,5169,5170,5175,5179,5184,5185,5186,5187,5188,5193,5195,5198,5202,5206,5217,5222,5224,5228,5229,5230,5233,5240,5241,5242,5249,5253,5254,5259,5260,5262,5263,5269,5273,5277,5278,5280,5281,5282,5284,5286,5290,5291,5293,5297,5299,5302,5303,5324,5326,5332,5334,5335,5339,5340,5341,5351,5353,5368,5371,5373,5375,5376,5388,5395,5396,5397,5400,5401,5405,5410,5416,5417,5421,5423,5425,5426,5431,5432,5440,5446,5447,5448,5452,5458,5459,5460,5463,5465,5466,5470,5471,5475,5476,5484,5486,5487,5492 12 | Class#12:5500,5501,5508,5510,5512,5516,5520,5522,5524,5526,5527,5528,5532,5536,5537,5541,5542,5548,5552,5555,5556,5558,5567,5574,5576,5586,5588,5593,5594,5601,5613,5615,5616,5618,5619,5621,5622,5623,5624,5625,5630,5635,5639,5643,5647,5650,5653,5654,5658,5661,5663,5668,5673,5678,5684,5688,5697,5700,5702,5712,5713,5714,5715,5716,5718,5721,5724,5728,5729,5730,5732,5733,5734,5737,5740,5743,5745,5746,5747,5748,5749,5754,5761,5762,5763,5766,5767,5771,5776,5777,5778,5780,5782,5784,5792,5793,5801,5808,5820,5823,5830,5836,5837,5838,5839,5847,5850,5854,5856,5857,5861,5862,5866,5867,5870,5874,5875,5876,5881,5884,5886,5894,5899,5901,5902,5905,5907,5908,5909,5911,5912,5915,5916,5920,5923,5932,5936,5937,5938,5948,5951,5958,5959,5964,5967,5971,5976,5977,5983,5985,5998,5999 13 | Class#13:6002,6007,6008,6009,6010,6014,6015,6016,6017,6019,6023,6032,6036,6037,6041,6047,6048,6053,6056,6061,6064,6065,6068,6072,6075,6076,6081,6082,6083,6088,6096,6105,6106,6108,6111,6113,6121,6125,6143,6152,6155,6158,6163,6169,6172,6185,6200,6203,6215,6223,6226,6233,6236,6239,6244,6248,6250,6259,6264,6270,6271,6272,6275,6289,6309,6311,6313,6316,6317,6318,6319,6323,6324,6328,6332,6338,6341,6350,6356,6360,6366,6368,6383,6393,6407,6423,6424,6428,6433,6437,6441,6449,6458,6463,6464,6471,6475,6482,6483 14 | Class#14:6514,6515,6523,6524,6526,6530,6532,6537,6545,6546,6548,6551,6554,6555,6556,6557,6558,6559,6566,6571,6576,6599,6601,6618,6627,6636,6639,6645,6655,6656,6657,6659,6660,6661,6668,6670,6672,6673,6675,6680,6683,6687,6699,6702,6707,6716,6722,6726,6728,6730,6736,6739,6746,6755,6762,6765,6766,6772,6777,6790,6795,6797,6808,6809,6834,6845,6847,6848,6851,6855,6858,6859,6860,6862,6865,6869,6873,6885,6887,6888,6892,6895,6901,6908,6909,6913,6922,6926,6929,6942,6943,6945,6946,6953,6955,6958,6960,6962,6968,6971,6977,6978,6979,6982,6983,6996 15 | Class#15:7008,7012,7019,7027,7034,7036,7050,7054,7057,7062,7085,7110,7112,7116,7121,7124,7132,7133,7134,7140,7165,7169,7177,7186,7194,7196,7198,7203,7207,7209,7221,7224,7225,7230,7231,7232,7233,7236,7244,7250,7263,7265,7277,7280,7281,7294,7296,7302,7305,7312,7318,7323,7326,7348,7349,7353,7357,7364,7367,7368,7370,7374,7384,7387,7389,7392,7399,7406,7412,7418,7434,7448,7451,7452,7457,7463,7468,7472,7476,7478,7480,7492,7497 16 | Class#16:7504,7508,7511,7513,7516,7523,7534,7535,7541,7542,7554,7561,7562,7567,7568,7569,7575,7586,7592,7597,7599,7601,7606,7609,7617,7618,7621,7625,7629,7635,7636,7645,7656,7657,7661,7662,7667,7672,7674,7677,7681,7682,7684,7686,7690,7691,7694,7704,7706,7708,7709,7719,7726,7727,7729,7730,7732,7738,7739,7746,7754,7768,7775,7787,7788,7793,7796,7797,7801,7806,7821,7829,7830,7836,7843,7846,7851,7852,7857,7858,7859,7863,7866,7890,7891,7896,7901,7919,7922,7924,7926,7935,7938,7940,7942,7943,7945,7954,7958,7966,7977,7979,7986,7992,7995,7996 17 | Class#17:8003,8015,8017,8019,8020,8023,8024,8029,8037,8047,8048,8052,8055,8058,8066,8073,8083,8086,8093,8096,8097,8102,8104,8109,8111,8113,8114,8117,8119,8121,8122,8123,8124,8126,8133,8135,8145,8146,8151,8152,8156,8158,8159,8164,8165,8172,8177,8180,8189,8190,8194,8197,8198,8202,8205,8206,8214,8217,8218,8221,8223,8224,8226,8228,8231,8236,8238,8240,8241,8243,8246,8247,8249,8250,8251,8254,8255,8257,8261,8265,8267,8269,8272,8280,8286,8294,8298,8300,8305,8308,8310,8311,8314,8317,8324,8326,8327,8328,8329,8330,8339,8340,8347,8353,8355,8360,8362,8364,8365,8370,8373,8375,8378,8379,8383,8386,8393,8398,8409,8410,8412,8414,8415,8416,8417,8419,8420,8431,8432,8435,8444,8451,8457,8459,8465,8466,8470,8477,8478,8479,8481,8483,8487,8494,8499 18 | Class#18:8501,8512,8518,8524,8525,8535,8536,8545,8552,8561,8567,8572,8581,8582,8583,8584,8591,8594,8597,8599,8602,8608,8610,8612,8616,8618,8628,8629,8650,8663,8689,8700,8712,8716,8719,8721,8726,8733,8744,8745,8750,8757,8760,8761,8762,8763,8766,8769,8777,8785,8791,8804,8807,8820,8828,8838,8841,8848,8851,8857,8864,8865,8871,8882,8883,8889,8890,8893,8902,8905,8908,8912,8913,8915,8921,8926,8927,8940,8947,8948,8963,8967,8968,8969,8980,8982,8991 19 | Class#19:9003,9005,9007,9016,9019,9027,9041,9045,9047,9049,9050,9065,9071,9084,9094,9100,9101,9107,9111,9122,9124,9132,9136,9138,9139,9142,9146,9147,9151,9152,9163,9184,9188,9192,9194,9201,9202,9214,9219,9220,9222,9236,9255,9267,9273,9274,9275,9285,9286,9287,9288,9300,9325,9331,9339,9342,9343,9344,9349,9359,9369,9382,9384,9386,9397,9402,9411,9417,9420,9423,9432,9450,9451,9457,9467,9469,9474,9478,9486,9489,9499 20 | Class#20:9511,9518,9532,9534,9538,9542,9544,9546,9550,9559,9563,9567,9573,9578,9580,9582,9585,9586,9589,9600,9609,9616,9620,9622,9627,9628,9634,9654,9655,9665,9666,9671,9672,9673,9681,9682,9686,9687,9699,9700,9706,9722,9728,9729,9746,9752,9795,9799,9804,9808,9817,9818,9822,9827,9832,9840,9842,9862,9865,9866,9882,9894,9908,9920,9921,9923,9928,9939,9942,9943,9945,9952,9957,9970,9973,9977,9985,9986,9987,9989,9993,9995 21 | Class#21:10000,10002,10004,10006,10015,10016,10017,10019,10020,10021,10022,10024,10025,10028,10033,10037,10040,10045,10053,10054,10057,10065,10066,10067,10072,10083,10086,10089,10090,10096,10097,10103,10108,10111,10112,10118,10123,10129,10132,10133,10134,10141,10142,10143,10144,10145,10157,10159,10161,10164,10166,10167,10173,10174,10176,10178,10181,10183,10185,10189,10196,10198,10200,10201,10213,10214,10215,10217,10218,10220,10222,10223,10225,10226,10228,10233,10234,10244,10248,10252,10253,10254,10257,10261,10264,10266,10267,10268,10269,10270,10273,10275,10276,10279,10280,10283,10284,10287,10296,10299,10302,10311,10316,10320,10322,10325,10328,10331,10335,10338,10345,10346,10347,10351,10353,10355,10358,10360,10363,10372,10373,10376,10378,10380,10390,10402,10404,10406,10409,10413,10414,10418,10422,10427,10429,10430,10431,10432,10433,10438,10441,10444,10447,10450,10452,10453,10461,10462,10468,10470,10472,10475,10480,10483,10487,10490,10491,10492,10493,10494 22 | Class#22:10509,10519,10525,10526,10528,10535,10536,10540,10542,10544,10547,10559,10560,10561,10565,10566,10570,10574,10575,10576,10579,10583,10585,10592,10594,10595,10601,10604,10609,10613,10615,10616,10618,10619,10623,10624,10625,10626,10628,10629,10630,10634,10643,10646,10647,10649,10650,10651,10656,10662,10664,10672,10686,10687,10697,10698,10702,10709,10712,10723,10724,10727,10728,10732,10736,10737,10739,10740,10741,10744,10750,10751,10752,10762,10769,10775,10777,10785,10794,10803,10805,10808,10816,10821,10827,10834,10835,10836,10838,10839,10843,10846,10851,10856,10857,10859,10866,10868,10874,10875,10879,10881,10884,10885,10886,10887,10888,10898,10901,10904,10915,10917,10918,10920,10927,10930,10939,10943,10947,10956,10959,10964,10965,10969,10978,10982,10983,10987,10988,10990,10991,10999 23 | Class#23:11011,11020,11022,11028,11037,11039,11041,11042,11044,11050,11054,11058,11059,11065,11067,11069,11070,11072,11074,11079,11080,11085,11086,11089,11091,11094,11095,11098,11101,11102,11104,11112,11114,11116,11117,11119,11120,11121,11122,11126,11129,11137,11141,11144,11146,11148,11152,11155,11161,11162,11165,11168,11169,11171,11172,11175,11176,11182,11183,11189,11191,11193,11196,11200,11201,11208,11210,11211,11217,11218,11222,11225,11228,11229,11232,11237,11240,11241,11242,11243,11250,11255,11261,11262,11264,11266,11272,11273,11274,11280,11281,11282,11284,11288,11291,11296,11303,11304,11308,11309,11312,11314,11317,11320,11324,11330,11331,11335,11341,11343,11344,11346,11349,11350,11358,11359,11364,11365,11369,11371,11374,11388,11392,11399,11400,11402,11405,11411,11415,11423,11425,11433,11441,11443,11447,11449,11450,11451,11452,11455,11458,11459,11465,11466,11467,11471,11472,11479,11484,11490,11495,11496,11499 24 | Class#24:11502,11512,11521,11522,11523,11526,11531,11534,11543,11546,11547,11554,11558,11571,11572,11583,11589,11591,11597,11599,11600,11601,11608,11613,11623,11626,11633,11636,11640,11641,11646,11649,11651,11654,11664,11670,11673,11675,11678,11679,11680,11682,11684,11686,11690,11705,11715,11721,11723,11733,11739,11740,11749,11754,11759,11761,11766,11778,11788,11792,11793,11798,11809,11812,11822,11831,11839,11844,11857,11868,11872,11873,11878,11886,11897,11904,11906,11907,11910,11919,11920,11921,11934,11939,11944,11950,11952,11953,11957,11962,11965,11968,11986,11987,11988,11992,11994,11997,11999 25 | Class#25:12000,12002,12003,12005,12006,12014,12015,12020,12021,12023,12025,12026,12030,12033,12037,12038,12039,12041,12044,12046,12047,12049,12053,12056,12064,12065,12066,12073,12079,12081,12083,12087,12088,12094,12095,12098,12099,12101,12104,12107,12108,12110,12113,12114,12115,12116,12118,12119,12120,12121,12122,12123,12124,12128,12132,12136,12139,12141,12145,12147,12150,12151,12152,12153,12156,12158,12162,12169,12174,12175,12176,12183,12184,12185,12187,12190,12198,12199,12200,12205,12208,12209,12215,12224,12232,12236,12239,12241,12246,12249,12251,12253,12254,12264,12265,12269,12273,12274,12283,12285,12287,12288,12289,12290,12295,12307,12311,12314,12315,12317,12323,12326,12328,12330,12337,12343,12344,12349,12350,12351,12352,12353,12354,12361,12364,12368,12371,12374,12376,12377,12379,12380,12384,12385,12386,12387,12391,12394,12395,12398,12403,12404,12410,12412,12416,12421,12422,12426,12427,12431,12432,12438,12440,12441,12444,12446,12448,12449,12450,12453,12454,12455,12456,12457,12458,12461,12463,12464,12467,12468,12469,12471,12475,12476,12477,12479,12483,12485,12488,12492,12496,12497,12498,12499 26 | Class#26:12501,12509,12519,12521,12522,12533,12538,12539,12541,12542,12547,12549,12550,12552,12553,12556,12558,12565,12566,12568,12574,12577,12580,12585,12589,12597,12598,12600,12601,12602,12607,12609,12612,12623,12627,12636,12637,12641,12644,12650,12652,12657,12658,12659,12660,12667,12668,12670,12671,12672,12674,12677,12678,12685,12692,12694,12701,12728,12736,12737,12739,12740,12741,12742,12743,12751,12758,12759,12765,12776,12777,12780,12781,12787,12788,12791,12792,12794,12800,12809,12810,12818,12822,12825,12828,12840,12841,12843,12853,12855,12858,12860,12862,12863,12866,12873,12878,12879,12881,12883,12888,12891,12895,12898,12901,12904,12913,12915,12918,12921,12929,12930,12938,12952,12954,12957,12961,12966,12967,12968,12984,12997,12998 27 | Class#27:13006,13009,13011,13013,13015,13016,13028,13029,13032,13042,13049,13057,13061,13068,13069,13073,13075,13076,13081,13082,13083,13088,13091,13093,13097,13100,13111,13112,13115,13132,13140,13141,13144,13145,13149,13157,13161,13162,13163,13175,13176,13186,13187,13191,13193,13194,13198,13200,13201,13203,13207,13210,13212,13219,13221,13224,13232,13234,13236,13241,13243,13248,13250,13251,13253,13254,13256,13262,13265,13269,13271,13274,13279,13281,13283,13284,13285,13286,13288,13301,13303,13305,13308,13314,13318,13321,13323,13326,13330,13336,13339,13341,13343,13345,13347,13348,13352,13353,13355,13362,13367,13371,13372,13373,13376,13378,13379,13381,13385,13390,13396,13399,13400,13402,13407,13409,13414,13415,13417,13418,13427,13429,13431,13433,13434,13437,13452,13454,13455,13457,13464,13465,13466,13471,13475,13479,13489,13493,13497,13498 28 | Class#28:13510,13515,13519,13520,13525,13535,13544,13551,13557,13559,13565,13579,13591,13596,13599,13607,13609,13633,13650,13654,13655,13657,13674,13676,13679,13680,13684,13688,13696,13703,13707,13709,13722,13732,13735,13741,13742,13743,13752,13780,13787,13788,13793,13795,13804,13811,13813,13820,13832,13835,13849,13860,13861,13869,13908,13926,13928,13936,13938,13947,13957,13960,13969,13984,13987,13988,13989,13990 29 | Class#29:14003,14005,14010,14012,14016,14017,14019,14020,14027,14028,14030,14036,14037,14041,14043,14046,14049,14051,14053,14054,14060,14063,14064,14073,14075,14076,14077,14078,14086,14090,14093,14097,14098,14099,14100,14102,14104,14108,14109,14113,14116,14122,14124,14126,14128,14131,14132,14133,14134,14136,14140,14141,14144,14147,14152,14155,14160,14161,14162,14163,14164,14165,14168,14187,14189,14192,14193,14194,14197,14199,14205,14207,14208,14210,14211,14215,14216,14217,14218,14220,14221,14224,14230,14231,14234,14235,14237,14243,14244,14248,14251,14252,14256,14259,14260,14261,14263,14264,14265,14267,14269,14272,14276,14278,14279,14280,14282,14284,14288,14289,14291,14297,14299,14300,14301,14303,14304,14308,14309,14310,14315,14316,14321,14322,14324,14326,14328,14329,14330,14331,14335,14337,14338,14341,14343,14344,14346,14348,14349,14350,14351,14355,14363,14368,14369,14370,14372,14373,14378,14380,14382,14385,14386,14387,14391,14392,14398,14400,14404,14405,14406,14407,14410,14411,14413,14417,14418,14421,14423,14424,14427,14429,14434,14436,14438,14443,14447,14448,14450,14454,14459,14460,14465,14466,14467,14468,14472,14473,14479,14482,14485,14488,14496 30 | Class#30:14500,14501,14502,14504,14511,14512,14513,14518,14519,14520,14521,14526,14527,14529,14531,14532,14536,14539,14545,14548,14549,14552,14553,14554,14555,14556,14559,14560,14563,14566,14567,14569,14571,14572,14579,14581,14583,14584,14585,14586,14588,14589,14594,14595,14596,14603,14604,14609,14613,14617,14619,14621,14623,14628,14629,14630,14631,14635,14636,14638,14640,14641,14647,14651,14653,14655,14656,14658,14659,14661,14662,14664,14666,14667,14670,14675,14676,14677,14680,14681,14682,14685,14687,14691,14692,14693,14695,14696,14698,14699,14704,14709,14710,14711,14712,14718,14720,14722,14724,14726,14729,14733,14736,14737,14738,14740,14743,14745,14752,14758,14759,14762,14763,14765,14768,14772,14773,14774,14776,14778,14779,14780,14783,14784,14785,14787,14792,14793,14796,14798,14801,14807,14808,14811,14814,14816,14817,14818,14819,14820,14821,14825,14830,14832,14833,14836,14837,14839,14840,14842,14846,14849,14852,14853,14854,14855,14859,14860,14861,14862,14865,14866,14871,14876,14879,14884,14885,14886,14888,14890,14891,14892,14894,14896,14898,14900,14902,14903,14905,14906,14908,14910,14911,14912,14913,14922,14924,14927,14929,14930,14931,14932,14934,14935,14936,14937,14938,14941,14945,14947,14948,14950,14952,14959,14964,14968,14971,14974,14976,14978,14979,14983,14989,14992,14995,14996,14999 31 | Class#31:15000,15005,15009,15021,15024,15025,15027,15032,15033,15037,15040,15046,15049,15053,15070,15078,15084,15085,15086,15089,15092,15107,15110,15124,15125,15129,15131,15134,15135,15136,15142,15152,15157,15161,15168,15174,15175,15183,15198,15200,15203,15209,15219,15220,15228,15231,15243,15245,15251,15259,15266,15267,15269,15270,15277,15280,15281,15282,15287,15290,15292,15299,15324,15329,15332,15338,15350,15356,15360,15361,15378,15379,15386,15387,15389,15393,15394,15401,15409,15414,15416,15423,15427,15431,15437,15439,15442,15448,15456,15460,15463,15468,15470,15471,15475,15487 32 | Class#32:15501,15506,15507,15513,15524,15543,15546,15548,15558,15571,15574,15575,15576,15578,15579,15587,15595,15597,15601,15602,15604,15606,15607,15622,15637,15640,15649,15651,15656,15673,15676,15681,15684,15685,15694,15698,15700,15703,15710,15713,15714,15720,15721,15728,15737,15748,15752,15755,15758,15762,15767,15771,15775,15784,15789,15807,15809,15811,15812,15814,15817,15820,15826,15834,15836,15841,15867,15870,15873,15877,15879,15883,15889,15890,15892,15905,15916,15923,15925,15928,15929,15935,15942,15947,15955,15959,15963,15967,15990,15994,15995 33 | Class#33:16008,16012,16018,16024,16025,16032,16034,16037,16041,16045,16048,16049,16051,16052,16056,16065,16073,16089,16091,16092,16100,16103,16106,16107,16120,16121,16122,16127,16132,16133,16136,16153,16158,16160,16163,16171,16184,16186,16196,16201,16202,16206,16216,16221,16225,16231,16233,16236,16237,16240,16242,16250,16255,16256,16258,16263,16266,16268,16273,16281,16283,16285,16291,16293,16299,16307,16309,16312,16317,16329,16332,16343,16350,16352,16354,16373,16375,16380,16383,16385,16391,16392,16393,16403,16409,16413,16422,16423,16425,16432,16452,16467,16471,16482,16491,16494,16497,16499 34 | Class#34:16500,16506,16513,16516,16521,16528,16540,16547,16555,16556,16565,16570,16574,16588,16602,16608,16610,16623,16632,16633,16647,16656,16664,16672,16682,16686,16696,16700,16704,16713,16718,16724,16728,16730,16733,16740,16745,16757,16768,16769,16772,16780,16782,16785,16789,16792,16797,16807,16809,16810,16836,16847,16853,16860,16876,16878,16879,16885,16886,16887,16890,16893,16899,16908,16914,16922,16931,16954,16955,16961,16965,16967,16970,16983,16985,16987 35 | Class#35:17001,17010,17016,17018,17020,17021,17025,17031,17034,17042,17051,17052,17053,17057,17063,17067,17069,17070,17085,17087,17094,17096,17100,17101,17104,17105,17108,17112,17122,17123,17128,17129,17133,17135,17137,17164,17171,17173,17177,17181,17184,17197,17200,17208,17214,17217,17225,17250,17255,17271,17276,17279,17296,17302,17312,17313,17315,17316,17317,17332,17338,17339,17341,17346,17364,17371,17372,17375,17376,17381,17387,17392,17394,17408,17410,17411,17414,17415,17419,17424,17426,17429,17437,17443,17446,17452,17456,17462,17463,17467,17468,17471,17473,17476,17479,17493,17498 36 | Class#36:17500,17501,17502,17504,17506,17507,17510,17513,17516,17517,17521,17522,17530,17531,17538,17541,17542,17543,17544,17545,17548,17550,17551,17555,17560,17565,17576,17588,17591,17594,17610,17614,17617,17619,17636,17652,17665,17667,17671,17681,17684,17685,17687,17691,17697,17698,17699,17705,17712,17714,17721,17726,17727,17729,17730,17737,17746,17747,17750,17768,17773,17776,17779,17781,17789,17806,17810,17811,17812,17814,17821,17826,17827,17828,17831,17833,17834,17835,17845,17847,17849,17853,17855,17859,17864,17874,17882,17889,17894,17896,17897,17902,17921,17936,17939,17948,17950,17951,17955,17957,17963,17976,17982,17985,17988,17992 37 | Class#37:18004,18009,18010,18011,18012,18015,18019,18022,18023,18028,18029,18031,18036,18040,18041,18048,18053,18056,18059,18060,18065,18068,18069,18072,18073,18075,18076,18079,18081,18083,18086,18087,18090,18092,18096,18097,18102,18105,18106,18107,18117,18120,18122,18124,18127,18137,18139,18143,18144,18151,18156,18157,18158,18159,18160,18163,18167,18168,18169,18170,18176,18178,18180,18181,18187,18189,18194,18196,18197,18199,18202,18203,18214,18215,18225,18226,18227,18234,18238,18240,18242,18243,18246,18249,18251,18254,18256,18258,18263,18269,18271,18272,18284,18289,18291,18293,18295,18300,18302,18303,18314,18318,18320,18331,18342,18347,18348,18349,18352,18363,18365,18369,18370,18375,18376,18377,18378,18380,18381,18382,18383,18388,18397,18402,18407,18413,18420,18423,18426,18427,18429,18430,18432,18442,18445,18450,18451,18452,18453,18458,18460,18464,18465,18471,18473,18475,18476,18481,18489,18493,18495,18498 38 | Class#38:18504,18505,18509,18510,18529,18540,18548,18550,18551,18566,18568,18571,18579,18580,18584,18588,18589,18593,18594,18595,18600,18604,18605,18607,18608,18611,18612,18613,18622,18626,18627,18629,18641,18643,18656,18669,18670,18672,18676,18679,18684,18686,18687,18699,18706,18707,18710,18712,18714,18721,18724,18729,18750,18754,18761,18762,18764,18773,18774,18775,18780,18787,18797,18800,18802,18811,18812,18818,18820,18822,18825,18835,18840,18846,18849,18860,18861,18864,18869,18878,18883,18886,18890,18926,18931,18940,18941,18942,18943,18948,18952,18955,18978,18986,18993,18994 39 | Class#39:19001,19007,19011,19018,19031,19063,19067,19073,19090,19092,19099,19104,19108,19116,19125,19127,19149,19151,19152,19154,19156,19163,19165,19167,19176,19182,19185,19186,19193,19195,19205,19214,19216,19220,19229,19230,19235,19236,19237,19241,19244,19246,19252,19279,19280,19282,19289,19306,19309,19313,19321,19331,19338,19340,19349,19373,19378,19383,19386,19389,19397,19401,19403,19410,19412,19417,19423,19425,19431,19434,19438,19448,19459,19462,19468,19480,19481,19486 40 | Class#40:19504,19515,19522,19524,19532,19536,19537,19539,19543,19544,19549,19553,19558,19561,19562,19565,19573,19577,19580,19582,19583,19584,19589,19601,19602,19607,19611,19616,19617,19619,19630,19631,19633,19635,19639,19641,19643,19647,19648,19649,19653,19655,19661,19662,19664,19665,19667,19669,19670,19673,19679,19682,19686,19687,19691,19695,19697,19698,19699,19705,19709,19713,19717,19720,19721,19722,19724,19726,19727,19729,19730,19732,19737,19740,19748,19751,19753,19762,19764,19772,19774,19780,19791,19792,19793,19795,19796,19799,19803,19805,19807,19810,19811,19812,19814,19815,19818,19821,19824,19826,19829,19833,19837,19839,19847,19848,19851,19852,19853,19854,19865,19866,19867,19868,19872,19874,19877,19887,19888,19891,19895,19897,19900,19901,19906,19908,19909,19911,19917,19919,19925,19932,19934,19936,19937,19944,19946,19948,19949,19958,19959,19961,19962,19964,19967,19972,19973,19974,19975,19976,19977,19980,19981,19984,19985,19988,19989,19994,19999 41 | Class#41:20002,20004,20008,20013,20016,20020,20022,20027,20029,20033,20038,20040,20041,20044,20048,20054,20055,20057,20059,20063,20065,20067,20074,20078,20083,20084,20085,20087,20088,20094,20096,20097,20105,20112,20113,20114,20116,20119,20121,20126,20128,20130,20133,20134,20135,20137,20140,20142,20144,20146,20149,20156,20158,20159,20160,20161,20162,20163,20164,20165,20166,20169,20172,20173,20175,20177,20178,20179,20183,20185,20190,20194,20201,20202,20204,20206,20209,20212,20215,20216,20218,20221,20222,20224,20226,20227,20229,20230,20232,20233,20235,20236,20239,20241,20252,20253,20254,20256,20263,20264,20267,20272,20273,20278,20280,20281,20282,20285,20287,20291,20292,20295,20296,20297,20299,20303,20306,20310,20312,20313,20314,20316,20321,20324,20326,20328,20330,20331,20337,20340,20341,20345,20347,20351,20356,20358,20360,20368,20370,20374,20379,20380,20381,20384,20392,20395,20400,20401,20403,20406,20407,20408,20409,20411,20412,20413,20414,20418,20421,20424,20425,20432,20437,20440,20441,20443,20444,20446,20451,20452,20456,20458,20459,20461,20462,20463,20464,20465,20466,20472,20476,20477,20483,20488,20490,20491,20492,20493,20498,20499 42 | Class#42:20500,20502,20504,20511,20515,20520,20522,20523,20524,20525,20526,20528,20529,20533,20535,20538,20541,20542,20545,20546,20547,20548,20551,20553,20555,20556,20558,20566,20569,20570,20575,20576,20577,20579,20583,20584,20586,20594,20596,20598,20599,20609,20616,20617,20625,20627,20630,20631,20632,20634,20638,20639,20641,20642,20645,20646,20648,20649,20650,20653,20660,20665,20667,20668,20669,20671,20672,20677,20679,20680,20681,20682,20688,20689,20694,20696,20699,20704,20708,20713,20716,20720,20727,20729,20730,20731,20733,20738,20744,20745,20747,20748,20751,20752,20753,20755,20756,20758,20759,20761,20764,20765,20767,20771,20773,20782,20783,20784,20786,20787,20789,20794,20803,20806,20807,20809,20812,20814,20818,20827,20829,20832,20837,20839,20841,20846,20853,20856,20858,20860,20861,20864,20865,20868,20871,20874,20875,20876,20879,20880,20887,20891,20893,20896,20897,20898,20900,20902,20903,20904,20906,20908,20909,20910,20916,20917,20919,20922,20924,20926,20928,20930,20932,20934,20937,20938,20939,20946,20948,20952,20956,20957,20958,20960,20961,20962,20966,20968,20970,20971,20972,20976,20977,20980,20984,20985,20988,20991,20997 43 | Class#43:21005,21006,21010,21011,21015,21019,21025,21027,21032,21034,21039,21049,21052,21053,21072,21073,21095,21106,21108,21110,21114,21118,21120,21131,21134,21150,21153,21178,21182,21185,21189,21190,21192,21198,21203,21211,21212,21215,21227,21235,21237,21240,21243,21247,21249,21250,21251,21259,21261,21271,21273,21278,21281,21289,21291,21293,21296,21303,21311,21313,21317,21318,21321,21330,21331,21335,21337,21346,21354,21359,21362,21374,21386,21387,21392,21401,21409,21417,21425,21433,21446,21456,21457,21459,21462,21463,21474,21476,21492 44 | Class#44:21506,21507,21535,21538,21546,21548,21550,21552,21558,21564,21574,21580,21581,21584,21591,21594,21596,21603,21607,21613,21615,21620,21633,21635,21638,21642,21650,21662,21668,21669,21672,21676,21678,21683,21685,21692,21693,21712,21722,21723,21728,21732,21735,21749,21767,21772,21792,21793,21805,21810,21815,21829,21830,21833,21841,21843,21845,21848,21850,21851,21865,21867,21870,21877,21887,21895,21904,21918,21919,21923,21926,21934,21964,21966,21970,21978,21981,21983,21984,21993 45 | Class#45:22003,22005,22024,22029,22045,22048,22050,22066,22072,22081,22085,22094,22105,22108,22113,22120,22125,22129,22131,22139,22142,22147,22148,22149,22157,22169,22182,22184,22188,22195,22210,22211,22221,22222,22225,22226,22233,22250,22252,22258,22261,22266,22270,22273,22277,22280,22286,22293,22295,22310,22313,22314,22318,22320,22340,22343,22350,22355,22360,22368,22370,22372,22373,22377,22381,22396,22406,22409,22410,22417,22421,22422,22431,22435,22436,22437,22440,22455,22457,22460,22471,22480,22491 46 | Class#46:22502,22505,22508,22513,22515,22516,22517,22519,22524,22532,22533,22534,22542,22550,22551,22555,22558,22561,22563,22564,22565,22566,22567,22568,22572,22574,22577,22578,22579,22581,22583,22588,22589,22590,22593,22595,22596,22602,22610,22612,22613,22621,22623,22630,22632,22635,22641,22648,22672,22679,22684,22685,22689,22691,22694,22702,22707,22710,22714,22715,22717,22719,22720,22726,22727,22735,22739,22742,22748,22749,22752,22753,22770,22772,22774,22778,22779,22781,22782,22783,22786,22792,22797,22799,22800,22802,22803,22807,22808,22809,22810,22811,22814,22817,22819,22824,22825,22828,22829,22832,22833,22836,22837,22845,22848,22857,22858,22866,22868,22869,22870,22873,22880,22884,22888,22890,22892,22893,22895,22899,22905,22906,22910,22911,22914,22918,22919,22922,22927,22929,22932,22938,22939,22947,22951,22952,22957,22959,22960,22967,22969,22970,22971,22972,22973,22975,22977,22983,22993,22995,22999 47 | Class#47:23008,23011,23017,23019,23024,23026,23031,23033,23042,23045,23046,23051,23060,23069,23073,23074,23079,23081,23082,23084,23087,23088,23094,23096,23099,23102,23103,23105,23111,23117,23118,23119,23120,23123,23126,23127,23132,23133,23139,23142,23143,23151,23155,23161,23163,23164,23171,23178,23180,23182,23183,23192,23196,23197,23198,23205,23209,23211,23216,23221,23223,23225,23227,23242,23243,23250,23254,23259,23261,23266,23270,23274,23279,23283,23289,23291,23294,23295,23297,23309,23310,23311,23312,23321,23325,23329,23330,23338,23343,23349,23350,23355,23361,23363,23367,23368,23375,23377,23379,23388,23395,23398,23399,23405,23412,23414,23431,23432,23440,23443,23448,23449,23454,23455,23460,23472,23474,23477,23480,23485,23493,23494,23496 48 | Class#48:23502,23505,23506,23509,23518,23524,23532,23533,23535,23537,23542,23555,23557,23560,23561,23570,23572,23577,23582,23587,23588,23594,23598,23602,23604,23605,23607,23616,23617,23623,23626,23628,23629,23633,23634,23637,23642,23643,23646,23647,23648,23649,23650,23661,23666,23671,23674,23675,23677,23680,23681,23682,23689,23691,23694,23696,23699,23700,23705,23709,23715,23717,23724,23728,23730,23734,23735,23741,23750,23753,23757,23759,23761,23765,23771,23772,23774,23775,23786,23796,23797,23800,23801,23807,23809,23810,23814,23817,23819,23822,23823,23824,23825,23829,23833,23836,23838,23846,23847,23849,23851,23853,23856,23860,23862,23866,23872,23877,23879,23882,23884,23885,23886,23903,23912,23918,23920,23925,23926,23932,23934,23935,23937,23938,23940,23947,23949,23952,23957,23964,23965,23968,23971,23975,23984,23988,23990,23992,23993,23999 49 | Class#49:24002,24008,24010,24012,24013,24020,24023,24025,24029,24035,24039,24041,24046,24047,24048,24053,24054,24058,24059,24063,24064,24070,24072,24073,24074,24077,24079,24081,24084,24087,24095,24105,24108,24114,24115,24116,24119,24121,24128,24139,24141,24142,24144,24147,24153,24158,24159,24164,24166,24168,24169,24173,24174,24175,24176,24184,24186,24193,24195,24202,24204,24209,24212,24215,24223,24228,24239,24240,24245,24260,24274,24284,24285,24288,24289,24293,24296,24298,24303,24310,24311,24312,24313,24315,24316,24317,24321,24322,24330,24331,24332,24334,24343,24344,24349,24350,24352,24362,24367,24374,24375,24376,24383,24385,24390,24391,24395,24401,24402,24407,24416,24424,24426,24428,24432,24440,24442,24443,24450,24455,24457,24459,24461,24462,24463,24467,24474,24475,24480,24484 50 | Class#50:24500,24508,24517,24533,24534,24536,24537,24551,24556,24560,24565,24566,24572,24588,24595,24603,24604,24612,24614,24615,24616,24618,24619,24622,24626,24633,24637,24642,24644,24650,24656,24657,24670,24671,24682,24684,24685,24692,24695,24696,24705,24706,24709,24710,24711,24713,24716,24721,24726,24727,24737,24739,24741,24744,24745,24752,24758,24760,24762,24768,24771,24784,24785,24794,24799,24816,24817,24818,24819,24822,24828,24831,24833,24834,24837,24838,24845,24846,24847,24848,24851,24855,24860,24866,24868,24869,24876,24881,24883,24884,24885,24892,24893,24902,24907,24916,24919,24921,24922,24932,24937,24938,24956,24957,24964,24966,24972,24977,24984,24985,24991 51 | Class#51:25005,25006,25009,25013,25015,25017,25018,25019,25021,25022,25023,25029,25032,25035,25037,25040,25054,25061,25062,25067,25076,25084,25087,25088,25090,25093,25098,25101,25106,25116,25124,25125,25134,25135,25141,25143,25146,25153,25154,25155,25156,25160,25193,25204,25208,25211,25212,25218,25221,25223,25225,25227,25229,25234,25236,25237,25239,25240,25243,25246,25247,25248,25265,25269,25273,25276,25278,25285,25290,25295,25300,25302,25303,25304,25305,25317,25321,25322,25325,25327,25332,25334,25335,25338,25360,25364,25373,25375,25377,25378,25380,25386,25399,25403,25421,25422,25426,25429,25438,25441,25449,25451,25453,25459,25467,25475,25477,25479,25481,25482,25483,25484,25486,25488,25489,25499 52 | Class#52:25501,25506,25513,25519,25526,25528,25532,25553,25559,25561,25585,25586,25590,25591,25597,25599,25603,25605,25610,25611,25613,25619,25624,25630,25631,25634,25643,25645,25681,25686,25690,25695,25709,25730,25736,25739,25746,25753,25767,25784,25791,25792,25806,25811,25822,25825,25835,25843,25854,25855,25866,25868,25869,25871,25873,25881,25894,25901,25902,25909,25910,25922,25935,25936,25944,25954,25955,25961,25962,25974,25980,25983 53 | Class#53:26000,26001,26005,26006,26012,26016,26019,26024,26028,26034,26035,26039,26043,26064,26067,26071,26081,26083,26084,26086,26087,26090,26093,26095,26099,26100,26101,26103,26104,26108,26109,26111,26113,26115,26118,26122,26123,26132,26133,26134,26136,26140,26144,26148,26149,26152,26154,26155,26164,26169,26172,26178,26180,26182,26183,26185,26186,26190,26192,26195,26197,26199,26204,26209,26213,26218,26223,26224,26231,26234,26244,26250,26252,26257,26268,26280,26281,26286,26304,26306,26311,26313,26318,26321,26323,26334,26335,26342,26346,26349,26351,26353,26354,26359,26362,26369,26370,26376,26377,26380,26381,26382,26384,26387,26388,26390,26395,26410,26414,26415,26420,26428,26430,26432,26435,26437,26438,26440,26444,26448,26460,26471,26472,26475,26478,26483,26490,26493,26494 54 | Class#54:26500,26503,26510,26512,26513,26514,26515,26518,26527,26533,26538,26540,26541,26542,26543,26546,26550,26551,26554,26555,26556,26561,26563,26565,26566,26568,26569,26571,26574,26575,26576,26579,26581,26582,26584,26585,26587,26589,26594,26597,26603,26604,26605,26607,26611,26617,26619,26621,26622,26625,26628,26633,26634,26638,26644,26649,26653,26654,26656,26657,26658,26662,26665,26666,26667,26669,26671,26672,26676,26677,26681,26682,26685,26686,26687,26700,26703,26704,26707,26709,26712,26717,26719,26720,26721,26725,26730,26737,26738,26739,26741,26743,26745,26746,26751,26756,26762,26763,26765,26766,26772,26773,26776,26789,26791,26794,26807,26808,26810,26811,26817,26818,26819,26826,26829,26830,26839,26841,26844,26848,26851,26855,26862,26864,26873,26877,26878,26880,26887,26889,26895,26897,26899,26900,26903,26904,26905,26908,26919,26920,26926,26929,26930,26932,26933,26938,26939,26941,26944,26945,26947,26949,26952,26954,26957,26960,26964,26965,26967,26972,26974,26975,26978,26980,26986,26987,26988,26990,26993,26996,26998 55 | Class#55:27001,27010,27011,27018,27019,27023,27024,27026,27029,27033,27036,27038,27040,27042,27049,27050,27052,27054,27055,27056,27065,27073,27074,27078,27081,27087,27088,27089,27095,27096,27098,27102,27111,27115,27121,27126,27127,27130,27132,27135,27138,27139,27145,27146,27156,27160,27164,27176,27179,27182,27183,27185,27198,27203,27211,27224,27229,27234,27235,27237,27239,27240,27242,27248,27267,27273,27280,27281,27286,27288,27289,27291,27292,27294,27301,27307,27308,27309,27311,27314,27315,27334,27336,27341,27342,27346,27347,27350,27351,27355,27358,27368,27369,27370,27371,27375,27378,27383,27385,27390,27398,27399,27406,27407,27411,27413,27416,27420,27422,27423,27425,27430,27434,27442,27444,27453,27456,27460,27465,27470,27475,27478,27495,27496 56 | Class#56:27500,27501,27507,27524,27529,27537,27545,27549,27554,27558,27559,27560,27563,27566,27568,27574,27580,27583,27588,27592,27595,27600,27601,27610,27617,27621,27625,27633,27644,27646,27657,27677,27680,27681,27685,27686,27692,27699,27704,27706,27716,27724,27725,27730,27745,27770,27772,27774,27775,27777,27780,27785,27786,27798,27843,27845,27846,27848,27849,27851,27859,27861,27864,27865,27866,27869,27870,27875,27876,27881,27886,27894,27911,27916,27924,27930,27937,27940,27953,27972,27982,27994,27999 57 | Class#57:28006,28008,28018,28019,28021,28026,28036,28037,28039,28044,28047,28051,28057,28071,28072,28074,28076,28080,28083,28095,28099,28101,28103,28107,28125,28133,28134,28137,28139,28141,28142,28145,28146,28147,28151,28152,28154,28155,28156,28157,28158,28163,28164,28192,28193,28208,28237,28239,28243,28245,28249,28252,28261,28263,28273,28282,28285,28286,28297,28300,28305,28306,28312,28315,28329,28347,28351,28352,28353,28366,28392,28398,28400,28402,28405,28409,28411,28419,28421,28426,28429,28430,28431,28435,28438,28440,28442,28443,28448,28453,28456,28462,28464,28471,28473,28479 58 | Class#58:28502,28505,28506,28508,28515,28522,28523,28524,28533,28535,28539,28541,28542,28545,28546,28554,28555,28556,28562,28563,28569,28570,28578,28581,28583,28589,28592,28595,28602,28605,28607,28609,28614,28618,28619,28622,28639,28640,28642,28645,28652,28655,28657,28664,28668,28670,28678,28686,28690,28693,28700,28701,28703,28706,28708,28712,28714,28716,28721,28723,28728,28730,28733,28735,28739,28741,28742,28744,28745,28747,28751,28752,28763,28769,28775,28778,28782,28783,28786,28787,28790,28796,28797,28802,28803,28805,28812,28814,28817,28818,28820,28823,28828,28831,28832,28834,28836,28838,28849,28850,28851,28856,28858,28861,28864,28869,28881,28885,28889,28890,28891,28892,28894,28904,28905,28909,28912,28914,28915,28917,28924,28934,28936,28937,28939,28940,28944,28947,28950,28953,28955,28958,28960,28961,28965,28969,28979,28980,28984,28985,28987,28993,28995,28997,28998 59 | Class#59:29001,29008,29011,29012,29013,29018,29020,29021,29023,29026,29029,29035,29036,29044,29047,29048,29049,29050,29053,29055,29056,29059,29065,29069,29071,29076,29079,29083,29089,29090,29101,29103,29104,29111,29113,29116,29117,29123,29124,29130,29138,29140,29157,29159,29162,29164,29166,29168,29170,29174,29175,29180,29183,29187,29192,29195,29199,29203,29207,29216,29221,29223,29228,29236,29237,29238,29239,29240,29241,29254,29258,29260,29262,29266,29268,29269,29277,29278,29284,29298,29299,29301,29304,29306,29318,29326,29333,29338,29339,29342,29344,29349,29350,29352,29361,29371,29373,29374,29375,29376,29379,29385,29392,29394,29395,29396,29397,29398,29406,29408,29411,29414,29417,29418,29424,29427,29430,29432,29434,29439,29442,29443,29451,29453,29454,29463,29464,29465,29472,29473,29477,29478,29481,29483,29484,29486,29487,29489 60 | Class#60:29501,29510,29511,29513,29521,29522,29527,29528,29529,29532,29533,29541,29547,29550,29552,29555,29556,29561,29572,29583,29593,29607,29609,29620,29621,29623,29624,29632,29635,29636,29639,29640,29641,29645,29648,29653,29657,29659,29667,29676,29679,29684,29688,29689,29695,29697,29699,29706,29713,29718,29722,29726,29730,29734,29737,29739,29741,29743,29749,29751,29764,29767,29769,29770,29772,29785,29789,29790,29793,29802,29805,29806,29810,29811,29816,29819,29822,29831,29832,29841,29842,29850,29851,29854,29858,29859,29863,29874,29877,29885,29896,29901,29912,29917,29918,29922,29924,29925,29926,29933,29938,29940,29947,29956,29960,29970,29973,29975,29976,29977,29980,29989,29996 61 | Class#61:30007,30008,30019,30021,30026,30027,30044,30046,30047,30048,30054,30058,30059,30061,30067,30069,30074,30076,30077,30078,30079,30085,30095,30103,30106,30120,30125,30127,30128,30129,30141,30143,30149,30151,30156,30168,30171,30175,30176,30177,30179,30184,30186,30187,30188,30190,30193,30197,30202,30206,30207,30209,30223,30230,30234,30238,30239,30245,30246,30250,30251,30252,30255,30260,30261,30268,30271,30275,30276,30277,30279,30285,30288,30297,30301,30307,30309,30310,30311,30313,30314,30332,30343,30345,30347,30349,30352,30353,30354,30357,30360,30362,30365,30368,30370,30374,30378,30379,30383,30384,30385,30387,30390,30391,30397,30399,30405,30408,30411,30414,30419,30420,30425,30426,30428,30435,30445,30453,30456,30458,30463,30464,30467,30472,30475,30478,30485,30493 62 | Class#62:30500,30507,30508,30519,30521,30523,30528,30531,30534,30538,30540,30544,30546,30547,30550,30552,30557,30559,30561,30563,30571,30572,30574,30575,30580,30583,30585,30587,30590,30593,30597,30602,30603,30604,30605,30606,30609,30611,30612,30614,30617,30621,30622,30626,30627,30630,30631,30635,30638,30639,30640,30641,30642,30643,30651,30652,30653,30659,30662,30664,30671,30675,30676,30677,30680,30682,30685,30686,30687,30690,30696,30698,30699,30702,30703,30708,30710,30712,30716,30721,30723,30724,30726,30728,30741,30742,30751,30752,30754,30759,30762,30765,30767,30770,30771,30772,30776,30779,30781,30791,30793,30799,30801,30805,30811,30814,30815,30816,30819,30823,30826,30830,30831,30833,30834,30835,30836,30837,30838,30845,30852,30856,30858,30861,30865,30868,30869,30872,30874,30885,30888,30890,30895,30896,30899,30903,30908,30911,30915,30920,30922,30924,30926,30932,30933,30943,30950,30952,30956,30959,30960,30961,30963,30966,30967,30971,30973,30974,30977,30982,30984,30988,30990,30991,30993,30994 63 | Class#63:31004,31007,31016,31017,31018,31022,31023,31033,31039,31040,31041,31051,31056,31057,31060,31072,31073,31074,31075,31080,31083,31088,31089,31091,31095,31096,31103,31111,31116,31126,31131,31133,31137,31140,31150,31153,31161,31162,31165,31171,31172,31173,31180,31188,31192,31194,31204,31207,31208,31210,31211,31212,31218,31221,31222,31223,31226,31229,31240,31248,31250,31252,31254,31255,31256,31259,31261,31268,31276,31280,31283,31289,31300,31301,31307,31313,31314,31318,31319,31324,31329,31330,31333,31335,31339,31340,31341,31343,31345,31346,31350,31355,31363,31364,31367,31369,31373,31374,31388,31390,31391,31393,31394,31395,31397,31405,31420,31429,31438,31442,31443,31445,31446,31449,31450,31453,31455,31456,31457,31459,31465,31469,31470,31473,31475,31480,31481 64 | Class#64:31500,31507,31508,31518,31524,31529,31531,31533,31540,31542,31543,31545,31546,31547,31556,31567,31569,31574,31578,31580,31585,31586,31588,31594,31598,31602,31605,31606,31607,31610,31617,31618,31620,31626,31629,31632,31642,31655,31666,31675,31676,31681,31688,31699,31701,31706,31713,31716,31728,31729,31731,31736,31738,31742,31753,31755,31756,31757,31770,31777,31788,31790,31799,31804,31805,31808,31810,31812,31813,31814,31821,31823,31827,31835,31840,31843,31844,31855,31861,31862,31865,31876,31879,31880,31881,31884,31890,31891,31892,31897,31904,31912,31914,31917,31924,31925,31933,31936,31947,31951,31952,31953,31960,31964,31965,31975,31976,31978,31980,31982,31985,31987,31991,31997 65 | Class#65:32008,32009,32013,32018,32024,32028,32029,32034,32036,32049,32052,32054,32069,32073,32075,32076,32081,32082,32092,32096,32098,32103,32107,32114,32121,32126,32141,32142,32148,32151,32167,32170,32177,32190,32195,32196,32201,32214,32217,32222,32226,32239,32249,32258,32267,32270,32285,32288,32303,32307,32314,32321,32326,32332,32340,32354,32357,32364,32371,32374,32397,32398,32400,32426,32431,32446,32458,32469,32478,32486,32490,32493,32494 66 | Class#66:32501,32505,32507,32509,32513,32519,32523,32527,32530,32532,32537,32538,32539,32541,32544,32550,32553,32554,32556,32564,32565,32566,32574,32579,32587,32592,32596,32600,32603,32609,32610,32613,32615,32620,32622,32653,32659,32660,32675,32678,32680,32685,32686,32691,32693,32705,32720,32732,32734,32737,32750,32759,32763,32767,32784,32789,32797,32800,32801,32807,32810,32814,32822,32854,32855,32860,32870,32892,32905,32907,32909,32913,32918,32934,32938,32943,32944,32957,32958,32959,32968,32992,32995,32999 67 | Class#67:33003,33005,33054,33062,33066,33069,33074,33076,33082,33088,33092,33094,33109,33112,33117,33144,33146,33169,33177,33179,33192,33193,33194,33197,33206,33215,33220,33222,33228,33229,33240,33241,33245,33248,33255,33266,33269,33276,33292,33293,33308,33310,33313,33316,33317,33323,33325,33327,33343,33345,33347,33348,33353,33360,33362,33366,33372,33375,33384,33392,33394,33410,33434,33435,33437,33445,33448,33465,33475,33484,33494 68 | Class#68:33511,33514,33524,33526,33531,33538,33545,33552,33555,33559,33562,33563,33564,33565,33570,33578,33583,33590,33591,33594,33598,33602,33604,33609,33618,33620,33624,33625,33628,33629,33634,33636,33637,33638,33650,33652,33653,33654,33657,33680,33682,33686,33689,33690,33692,33699,33701,33702,33704,33706,33707,33710,33721,33722,33731,33732,33734,33742,33752,33765,33773,33777,33783,33785,33788,33790,33797,33801,33805,33806,33810,33811,33812,33814,33816,33819,33820,33826,33827,33832,33840,33846,33851,33853,33856,33857,33859,33872,33877,33886,33889,33894,33898,33900,33908,33913,33921,33922,33931,33932,33933,33939,33941,33956,33957,33961,33962,33972,33974,33982,33983,33985,33998 69 | Class#69:34001,34004,34006,34011,34013,34016,34018,34019,34025,34027,34033,34051,34052,34056,34057,34071,34074,34079,34086,34087,34088,34091,34095,34097,34100,34102,34105,34107,34112,34113,34114,34121,34122,34124,34125,34127,34131,34139,34142,34145,34146,34149,34156,34157,34161,34174,34176,34180,34181,34184,34187,34189,34192,34196,34198,34199,34200,34206,34207,34208,34214,34215,34231,34232,34233,34242,34244,34246,34252,34256,34259,34260,34262,34265,34270,34271,34272,34276,34279,34280,34284,34287,34288,34290,34303,34306,34315,34317,34325,34328,34338,34340,34341,34343,34344,34347,34349,34350,34356,34357,34359,34374,34376,34378,34379,34383,34390,34394,34403,34410,34418,34424,34431,34433,34443,34449,34454,34456,34457,34459,34462,34463,34469,34470,34472,34474,34480,34486,34490,34494,34495,34499 70 | Class#70:34504,34509,34513,34516,34521,34525,34526,34527,34528,34535,34537,34540,34549,34555,34564,34565,34569,34576,34577,34583,34585,34589,34591,34597,34598,34600,34602,34605,34606,34608,34609,34612,34614,34617,34627,34628,34641,34645,34649,34654,34662,34664,34666,34667,34671,34675,34679,34681,34684,34688,34689,34690,34691,34692,34694,34701,34702,34705,34710,34712,34715,34716,34717,34719,34723,34728,34737,34739,34741,34745,34747,34750,34753,34754,34758,34762,34764,34765,34776,34779,34785,34786,34789,34793,34796,34805,34819,34823,34824,34825,34830,34838,34840,34847,34853,34856,34861,34867,34869,34870,34871,34876,34882,34884,34886,34887,34892,34893,34894,34901,34906,34909,34910,34911,34912,34917,34920,34922,34926,34927,34939,34946,34951,34954,34956,34963,34968,34973,34975,34976,34977,34984,34988,34992 71 | Class#71:35009,35011,35013,35014,35015,35017,35019,35023,35029,35030,35036,35038,35039,35040,35045,35057,35062,35066,35070,35077,35078,35082,35085,35092,35094,35096,35102,35110,35114,35115,35119,35122,35123,35135,35136,35139,35140,35144,35147,35148,35149,35152,35154,35156,35160,35161,35168,35170,35176,35179,35182,35188,35190,35198,35207,35209,35213,35218,35235,35240,35241,35247,35248,35249,35250,35251,35259,35261,35262,35263,35264,35265,35268,35270,35271,35273,35277,35278,35279,35287,35288,35289,35294,35296,35298,35302,35307,35308,35311,35315,35321,35322,35323,35343,35348,35351,35357,35361,35362,35367,35371,35376,35383,35384,35386,35387,35389,35390,35392,35404,35406,35410,35414,35417,35421,35422,35424,35425,35426,35427,35431,35446,35449,35450,35451,35462,35464,35466,35475,35478,35480,35487,35488,35489,35490,35494,35497 72 | Class#72:35500,35504,35512,35514,35523,35528,35543,35544,35546,35554,35556,35566,35569,35571,35573,35576,35577,35583,35588,35591,35592,35599,35607,35609,35612,35618,35621,35627,35630,35631,35634,35639,35670,35671,35675,35681,35682,35688,35690,35696,35699,35700,35701,35707,35709,35711,35714,35717,35722,35724,35725,35727,35732,35736,35749,35755,35756,35768,35769,35774,35779,35787,35789,35810,35811,35813,35814,35817,35818,35820,35821,35823,35839,35849,35851,35857,35859,35861,35862,35869,35870,35871,35872,35877,35880,35896,35897,35898,35900,35914,35917,35920,35931,35933,35937,35942,35954,35957,35961,35964,35965,35979,35980,35995,35999 73 | Class#73:36002,36004,36006,36012,36014,36016,36018,36019,36022,36023,36024,36028,36031,36034,36039,36042,36053,36057,36068,36069,36071,36081,36082,36084,36091,36100,36103,36104,36108,36111,36132,36135,36137,36142,36144,36148,36154,36158,36159,36162,36165,36168,36169,36175,36188,36190,36192,36193,36194,36196,36199,36201,36209,36210,36217,36225,36238,36239,36245,36247,36256,36266,36268,36288,36290,36294,36303,36307,36319,36341,36349,36356,36373,36388,36396,36399,36405,36411,36415,36429,36433,36442,36445,36446,36448,36450,36453,36454,36459,36470,36472,36480,36484,36487,36490,36498 74 | Class#74:36505,36542,36543,36544,36545,36546,36547,36548,36549,36554,36556,36561,36562,36594,36595,36596,36597,36605,36606,36609,36616,36619,36625,36628,36631,36638,36639,36644,36647,36651,36655,36667,36668,36673,36677,36689,36690,36694,36696,36705,36713,36715,36716,36718,36719,36727,36736,36744,36746,36758,36765,36770,36777,36779,36781,36783,36784,36786,36795,36799,36800,36801,36807,36816,36831,36840,36846,36852,36853,36854,36859,36861,36862,36863,36868,36871,36873,36877,36880,36894,36900,36901,36916,36917,36919,36920,36922,36923,36932,36933,36934,36935,36943,36944,36945,36946,36957,36965,36979,36981,36988,36989,36997,36999 75 | Class#75:37001,37002,37004,37005,37006,37008,37010,37012,37018,37023,37026,37028,37033,37038,37041,37050,37053,37064,37083,37085,37090,37091,37101,37102,37104,37108,37117,37119,37122,37126,37128,37132,37152,37153,37157,37159,37163,37164,37169,37170,37176,37177,37189,37191,37193,37202,37211,37214,37219,37224,37229,37237,37238,37249,37252,37254,37262,37266,37270,37273,37274,37286,37287,37294,37299,37301,37307,37309,37313,37315,37320,37322,37331,37338,37352,37365,37368,37370,37372,37374,37379,37381,37382,37394,37400,37406,37410,37412,37421,37440,37441,37442,37443,37444,37451,37456,37458,37460,37463,37478,37484,37493,37494,37498 76 | Class#76:37504,37509,37510,37521,37524,37530,37539,37541,37547,37551,37558,37560,37562,37563,37566,37578,37589,37596,37600,37601,37603,37606,37609,37618,37622,37629,37632,37633,37635,37641,37645,37646,37651,37652,37659,37660,37663,37664,37667,37670,37676,37682,37683,37693,37698,37700,37703,37709,37710,37719,37720,37725,37727,37733,37738,37740,37748,37749,37752,37753,37757,37759,37767,37769,37775,37777,37779,37787,37791,37794,37795,37797,37800,37804,37805,37807,37810,37816,37824,37829,37836,37840,37842,37843,37857,37863,37868,37872,37876,37878,37879,37888,37889,37891,37894,37902,37912,37917,37918,37923,37931,37935,37939,37940,37941,37942,37946,37947,37950,37959,37960,37965,37966,37969,37972,37977,37980,37984,37994,37998,37999 77 | Class#77:38006,38009,38011,38024,38032,38034,38035,38037,38038,38041,38051,38054,38056,38061,38065,38066,38068,38070,38083,38089,38094,38095,38098,38101,38107,38108,38113,38115,38117,38119,38138,38141,38142,38151,38160,38161,38164,38165,38166,38167,38181,38182,38184,38186,38194,38199,38209,38218,38222,38223,38229,38232,38238,38239,38247,38253,38254,38259,38263,38264,38275,38279,38280,38281,38284,38286,38294,38300,38302,38320,38323,38327,38328,38330,38331,38345,38349,38351,38358,38364,38365,38370,38371,38374,38377,38382,38384,38386,38389,38392,38394,38396,38404,38407,38408,38423,38429,38430,38432,38433,38439,38445,38454,38457,38458,38465,38475,38480,38487,38489,38493 78 | Class#78:38505,38507,38509,38512,38519,38522,38529,38537,38551,38553,38554,38558,38560,38563,38568,38572,38573,38575,38577,38583,38587,38591,38595,38597,38600,38601,38603,38629,38635,38639,38647,38658,38660,38663,38665,38670,38672,38680,38683,38689,38691,38695,38706,38708,38710,38717,38721,38732,38734,38738,38742,38744,38748,38756,38758,38761,38772,38778,38780,38783,38810,38811,38813,38814,38817,38822,38825,38826,38829,38830,38839,38840,38846,38856,38865,38879,38880,38885,38886,38891,38903,38906,38928,38933,38934,38935,38939,38945,38950,38951,38957,38965,38977,38993 79 | Class#79:39011,39020,39031,39033,39047,39050,39070,39084,39085,39093,39100,39106,39110,39114,39118,39119,39128,39132,39134,39136,39159,39160,39174,39178,39184,39186,39191,39196,39201,39202,39212,39214,39215,39225,39231,39234,39235,39238,39263,39274,39278,39283,39290,39310,39313,39320,39321,39334,39345,39362,39368,39381,39386,39390,39392,39393,39401,39404,39407,39415,39420,39433,39462,39464,39465,39475,39477,39480,39481,39485,39494,39495 80 | Class#80:39502,39507,39510,39511,39513,39517,39535,39536,39537,39539,39546,39551,39555,39559,39561,39573,39587,39590,39594,39604,39609,39615,39622,39623,39626,39632,39641,39644,39647,39650,39652,39653,39654,39655,39659,39662,39664,39668,39682,39684,39685,39690,39694,39695,39698,39701,39712,39729,39733,39734,39738,39745,39753,39764,39767,39771,39775,39777,39778,39794,39809,39815,39820,39821,39823,39826,39827,39833,39836,39840,39841,39847,39855,39858,39864,39865,39866,39877,39881,39884,39889,39895,39904,39909,39923,39934,39936,39937,39938,39943,39944,39950,39958,39959,39966,39972,39978,39981,39984,39985,39990,39991,39992,39997,39999 81 | Class#81:40004,40010,40016,40021,40050,40055,40058,40065,40077,40086,40089,40092,40100,40103,40108,40109,40111,40113,40138,40141,40148,40160,40163,40196,40198,40202,40217,40223,40226,40229,40239,40248,40267,40275,40280,40304,40329,40331,40342,40344,40348,40352,40355,40374,40377,40381,40396,40397,40414,40419,40425,40437,40441,40445,40451,40461,40472,40488 82 | Class#82:40505,40516,40525,40526,40550,40551,40554,40557,40560,40568,40572,40576,40579,40581,40589,40591,40593,40604,40608,40615,40618,40620,40630,40631,40640,40641,40663,40677,40679,40683,40690,40703,40707,40708,40716,40723,40731,40744,40749,40751,40768,40770,40773,40775,40777,40789,40794,40797,40798,40800,40802,40810,40814,40817,40819,40832,40840,40841,40843,40844,40850,40855,40860,40862,40866,40867,40871,40876,40889,40893,40901,40903,40910,40919,40923,40924,40941,40942,40947,40948,40950,40956,40958,40964,40968,40976,40979,40984,40985,40992,40993 83 | Class#83:41003,41007,41014,41019,41020,41022,41025,41027,41030,41035,41040,41045,41050,41051,41056,41062,41071,41072,41074,41075,41078,41080,41083,41086,41087,41091,41092,41099,41104,41108,41110,41114,41116,41119,41123,41130,41132,41141,41148,41151,41154,41155,41156,41166,41191,41199,41201,41208,41211,41212,41220,41225,41226,41228,41229,41243,41246,41251,41255,41256,41258,41263,41267,41279,41284,41286,41290,41292,41294,41296,41297,41303,41304,41307,41309,41311,41316,41323,41326,41327,41347,41349,41351,41362,41363,41367,41373,41374,41375,41378,41379,41382,41386,41393,41399,41402,41408,41413,41414,41415,41422,41423,41425,41429,41437,41444,41445,41446,41447,41448,41451,41452,41456,41458,41460,41465,41469,41473,41476,41479,41485,41493 84 | Class#84:41502,41503,41504,41506,41507,41508,41516,41519,41520,41523,41525,41528,41529,41530,41531,41533,41535,41537,41540,41542,41544,41545,41547,41549,41550,41556,41560,41561,41562,41569,41573,41575,41576,41579,41583,41584,41588,41589,41592,41597,41602,41603,41609,41610,41611,41617,41619,41621,41624,41625,41627,41629,41631,41637,41639,41640,41641,41645,41648,41649,41650,41658,41659,41661,41663,41664,41668,41674,41677,41679,41684,41686,41687,41690,41691,41693,41694,41700,41702,41704,41706,41708,41714,41715,41718,41722,41731,41734,41735,41737,41738,41743,41749,41751,41758,41763,41771,41772,41773,41775,41776,41777,41779,41780,41785,41790,41793,41803,41817,41818,41821,41823,41827,41828,41830,41831,41835,41838,41845,41855,41857,41867,41870,41881,41884,41892,41895,41897,41901,41908,41912,41915,41918,41925,41928,41929,41930,41931,41933,41934,41935,41940,41942,41943,41947,41948,41949,41956,41961,41965,41967,41968,41971,41973,41975,41976,41977,41981,41984,41986,41988,41998 85 | Class#85:42002,42005,42006,42009,42010,42011,42012,42019,42023,42027,42030,42031,42042,42048,42051,42060,42069,42070,42074,42076,42080,42082,42089,42096,42110,42112,42117,42122,42131,42132,42140,42152,42157,42160,42175,42180,42188,42191,42193,42196,42205,42206,42214,42215,42216,42217,42223,42225,42228,42235,42236,42245,42246,42256,42257,42260,42270,42273,42277,42284,42285,42288,42299,42303,42304,42306,42309,42313,42325,42329,42333,42336,42340,42348,42351,42352,42354,42359,42363,42377,42378,42379,42380,42382,42389,42392,42395,42407,42418,42420,42422,42426,42427,42433,42436,42437,42441,42443,42444,42458,42467,42471,42474,42476,42478,42479,42480,42481,42488,42489,42493,42495,42499 86 | Class#86:42505,42507,42512,42514,42518,42521,42525,42526,42532,42537,42539,42540,42544,42546,42550,42551,42554,42556,42569,42570,42572,42573,42579,42581,42582,42586,42589,42609,42619,42620,42622,42623,42630,42635,42637,42652,42655,42656,42659,42660,42668,42673,42677,42679,42680,42683,42685,42690,42691,42698,42700,42702,42703,42709,42714,42717,42718,42723,42731,42734,42741,42744,42753,42755,42756,42758,42761,42763,42770,42772,42775,42784,42786,42792,42793,42795,42797,42798,42801,42802,42820,42822,42831,42835,42836,42839,42840,42842,42844,42847,42857,42867,42868,42869,42873,42875,42876,42877,42878,42885,42887,42890,42895,42909,42915,42916,42917,42918,42919,42920,42922,42923,42924,42927,42929,42933,42937,42942,42945,42948,42949,42955,42957,42961,42974,42975,42978,42981,42987,42994 87 | Class#87:43004,43006,43010,43012,43013,43015,43018,43019,43020,43021,43022,43025,43032,43040,43042,43047,43049,43050,43051,43055,43057,43060,43066,43072,43074,43075,43076,43078,43080,43083,43084,43089,43090,43095,43096,43100,43103,43107,43111,43116,43119,43120,43121,43128,43141,43142,43145,43148,43151,43153,43154,43155,43159,43161,43164,43165,43166,43167,43168,43170,43171,43172,43174,43176,43177,43180,43182,43183,43189,43190,43192,43193,43194,43197,43198,43200,43201,43202,43205,43206,43211,43212,43219,43220,43223,43228,43230,43231,43233,43234,43236,43237,43239,43240,43241,43244,43250,43253,43257,43259,43260,43261,43264,43265,43269,43270,43275,43276,43279,43280,43286,43289,43295,43297,43299,43304,43305,43309,43310,43311,43316,43317,43318,43319,43324,43327,43331,43334,43335,43337,43338,43339,43341,43344,43350,43351,43353,43356,43357,43359,43361,43363,43364,43370,43379,43380,43381,43384,43390,43394,43395,43400,43404,43406,43409,43412,43421,43424,43432,43438,43439,43440,43441,43445,43449,43450,43454,43464,43466,43471,43476,43477,43479,43480,43483,43491,43496,43499 88 | Class#88:43500,43504,43505,43508,43513,43514,43524,43533,43535,43539,43541,43546,43549,43556,43557,43561,43576,43581,43584,43585,43588,43592,43596,43599,43600,43603,43605,43609,43613,43614,43615,43618,43619,43620,43631,43637,43654,43656,43658,43661,43662,43665,43674,43676,43683,43684,43685,43688,43690,43696,43708,43720,43723,43724,43726,43728,43734,43741,43742,43743,43749,43752,43755,43756,43758,43762,43767,43768,43769,43770,43771,43772,43774,43777,43779,43782,43789,43790,43797,43801,43809,43811,43813,43814,43819,43825,43827,43835,43836,43838,43839,43843,43846,43848,43849,43853,43858,43859,43860,43863,43866,43869,43871,43872,43874,43877,43880,43882,43885,43889,43902,43909,43911,43919,43921,43925,43926,43931,43932,43937,43939,43951,43953,43959,43963,43964,43969,43970,43974,43977,43982,43984,43989,43994,43998 89 | Class#89:44001,44007,44008,44016,44018,44020,44025,44031,44034,44039,44055,44057,44061,44063,44066,44085,44097,44110,44120,44121,44135,44139,44146,44160,44165,44166,44168,44175,44181,44184,44226,44232,44235,44250,44255,44257,44258,44260,44269,44273,44285,44287,44292,44301,44312,44336,44339,44355,44362,44366,44372,44374,44375,44384,44390,44404,44405,44406,44411,44414,44421,44432,44436,44439,44444,44450,44454,44462,44463,44468,44472,44474,44485,44488,44497 90 | Class#90:44515,44516,44522,44527,44528,44531,44532,44536,44548,44554,44555,44561,44593,44600,44605,44608,44617,44621,44632,44635,44636,44650,44651,44653,44661,44663,44666,44671,44673,44691,44693,44700,44701,44702,44703,44711,44712,44721,44723,44731,44733,44742,44750,44753,44774,44778,44781,44788,44790,44819,44822,44826,44827,44828,44831,44834,44838,44846,44857,44860,44870,44872,44878,44885,44905,44919,44930,44931,44932,44937,44951,44959,44964,44974,44978,44994,44995 91 | Class#91:45008,45010,45018,45019,45020,45023,45029,45035,45039,45047,45050,45051,45053,45057,45072,45074,45075,45078,45080,45084,45088,45093,45104,45108,45115,45120,45123,45126,45127,45130,45141,45142,45144,45150,45151,45152,45157,45162,45165,45167,45172,45177,45179,45182,45183,45185,45189,45190,45192,45193,45199,45201,45207,45208,45219,45220,45222,45224,45227,45256,45265,45266,45268,45270,45271,45276,45290,45291,45299,45304,45306,45308,45310,45314,45316,45321,45326,45329,45330,45334,45349,45364,45368,45371,45377,45379,45380,45383,45388,45393,45406,45407,45411,45412,45416,45420,45426,45430,45431,45434,45440,45445,45446,45449,45454,45456,45460,45467,45485,45488,45497,45498 92 | Class#92:45501,45503,45504,45506,45517,45518,45519,45520,45529,45531,45534,45536,45537,45538,45543,45548,45552,45553,45558,45560,45574,45576,45577,45578,45579,45582,45583,45585,45592,45594,45595,45596,45597,45601,45607,45611,45614,45617,45620,45632,45635,45640,45641,45645,45647,45650,45652,45653,45654,45656,45658,45661,45662,45664,45666,45672,45681,45682,45687,45695,45697,45698,45700,45702,45704,45705,45715,45716,45718,45719,45721,45727,45728,45731,45732,45735,45740,45743,45746,45747,45750,45751,45754,45755,45757,45760,45762,45771,45778,45780,45784,45789,45790,45791,45794,45795,45797,45799,45803,45808,45809,45814,45818,45821,45823,45824,45828,45831,45833,45836,45838,45839,45843,45844,45847,45848,45852,45854,45857,45858,45859,45860,45869,45870,45871,45880,45885,45891,45894,45905,45906,45908,45912,45918,45930,45931,45933,45934,45943,45944,45948,45951,45952,45963,45967,45969,45970,45974,45975,45977,45979,45989,45995,45996 93 | Class#93:46002,46008,46013,46019,46023,46029,46038,46041,46059,46067,46071,46072,46074,46075,46084,46085,46089,46090,46093,46098,46102,46115,46128,46132,46144,46156,46164,46170,46181,46196,46200,46207,46209,46210,46212,46213,46218,46219,46221,46222,46225,46226,46228,46237,46239,46243,46245,46255,46258,46259,46261,46269,46283,46287,46292,46295,46300,46314,46321,46323,46330,46335,46341,46342,46347,46351,46352,46353,46362,46364,46366,46367,46369,46372,46377,46382,46383,46384,46388,46399,46402,46410,46415,46419,46421,46432,46436,46438,46441,46456,46478,46482,46483,46484 94 | Class#94:46500,46504,46505,46506,46508,46509,46510,46512,46514,46515,46518,46520,46521,46525,46526,46533,46534,46537,46540,46542,46545,46547,46548,46551,46553,46556,46559,46561,46563,46567,46569,46574,46575,46584,46585,46598,46603,46604,46611,46613,46615,46620,46622,46623,46629,46634,46636,46638,46641,46643,46644,46656,46664,46671,46678,46679,46681,46682,46691,46693,46706,46707,46710,46711,46712,46717,46721,46727,46733,46740,46743,46746,46747,46748,46750,46754,46759,46761,46770,46772,46779,46780,46783,46787,46791,46792,46795,46798,46799,46808,46811,46816,46817,46827,46831,46833,46838,46844,46845,46847,46852,46854,46855,46868,46869,46875,46876,46881,46883,46884,46889,46894,46906,46908,46911,46918,46925,46928,46929,46934,46936,46942,46947,46948,46954,46957,46958,46963,46965,46966,46971,46977,46981,46982,46987,46989,46994,46997 95 | Class#95:47000,47009,47013,47024,47025,47026,47027,47029,47036,47037,47049,47053,47057,47061,47067,47072,47074,47078,47085,47096,47098,47100,47108,47115,47116,47124,47125,47130,47132,47133,47134,47143,47147,47154,47161,47165,47166,47171,47181,47182,47187,47188,47191,47193,47199,47200,47212,47214,47217,47219,47220,47225,47228,47229,47236,47239,47244,47246,47250,47251,47254,47256,47257,47259,47262,47271,47272,47274,47276,47284,47297,47298,47301,47303,47304,47310,47312,47313,47316,47317,47323,47326,47333,47334,47338,47340,47344,47350,47354,47358,47363,47371,47374,47378,47379,47381,47382,47383,47387,47391,47393,47394,47398,47400,47401,47404,47405,47408,47421,47423,47431,47433,47434,47435,47436,47437,47440,47444,47449,47451,47453,47454,47459,47469,47472,47473,47474,47477,47480,47485,47495,47496,47499 96 | Class#96:47500,47501,47507,47511,47514,47520,47522,47525,47534,47537,47546,47547,47550,47557,47559,47562,47571,47575,47579,47584,47587,47594,47596,47601,47603,47605,47606,47609,47611,47612,47621,47623,47628,47633,47635,47637,47638,47639,47645,47650,47651,47654,47659,47660,47661,47662,47663,47667,47668,47671,47673,47674,47679,47685,47687,47688,47691,47692,47694,47697,47705,47710,47718,47720,47721,47737,47741,47742,47744,47746,47747,47749,47750,47753,47759,47772,47773,47778,47779,47783,47786,47788,47792,47794,47797,47798,47799,47802,47804,47807,47814,47815,47816,47820,47824,47829,47833,47840,47842,47843,47849,47854,47861,47864,47868,47872,47877,47878,47881,47885,47887,47890,47891,47895,47896,47905,47909,47910,47914,47917,47921,47924,47928,47936,47939,47940,47941,47944,47953,47959,47962,47973,47974,47975,47977,47985,47986,47990 97 | Class#97:48000,48006,48007,48010,48021,48028,48029,48035,48038,48040,48042,48048,48052,48055,48059,48060,48066,48071,48073,48075,48076,48083,48085,48086,48103,48105,48107,48121,48127,48134,48138,48139,48143,48151,48165,48167,48169,48175,48182,48186,48189,48190,48192,48209,48218,48219,48224,48231,48241,48247,48250,48253,48255,48263,48267,48272,48276,48277,48279,48280,48298,48301,48306,48307,48308,48321,48324,48328,48329,48332,48333,48338,48347,48349,48360,48363,48368,48376,48383,48384,48385,48390,48396,48401,48403,48416,48417,48427,48431,48432,48438,48443,48457,48459,48461,48463,48466,48468,48472,48478,48481,48484,48495 98 | Class#98:48500,48505,48512,48515,48526,48528,48537,48544,48546,48548,48559,48566,48571,48581,48614,48617,48621,48627,48631,48633,48640,48644,48650,48655,48660,48662,48666,48677,48678,48682,48684,48686,48690,48691,48696,48702,48705,48711,48712,48716,48718,48720,48729,48741,48751,48752,48759,48770,48775,48780,48781,48782,48790,48807,48813,48817,48818,48822,48823,48834,48847,48853,48854,48862,48863,48876,48881,48885,48886,48895,48898,48909,48912,48916,48921,48928,48930,48936,48938,48940,48954,48961,48963,48964,48966,48967,48976,48982,48988,48989,48994,48999 99 | Class#99:49001,49002,49004,49005,49006,49009,49012,49018,49021,49030,49031,49038,49048,49053,49057,49065,49068,49071,49073,49083,49084,49086,49090,49093,49094,49097,49100,49101,49103,49104,49106,49111,49112,49114,49118,49124,49127,49128,49142,49147,49150,49151,49152,49156,49157,49158,49159,49161,49162,49170,49173,49180,49189,49196,49197,49200,49208,49217,49219,49220,49221,49225,49227,49231,49233,49235,49237,49239,49241,49246,49247,49249,49252,49254,49256,49257,49261,49263,49269,49270,49274,49279,49280,49281,49284,49285,49288,49289,49290,49293,49294,49297,49303,49304,49305,49309,49311,49313,49316,49317,49324,49331,49336,49339,49377,49379,49380,49381,49383,49386,49391,49392,49394,49397,49403,49405,49406,49413,49416,49417,49419,49431,49432,49435,49443,49446,49449,49452,49461,49467,49468,49472,49473,49489,49490,49492,49499 100 | Class#100:49502,49505,49506,49507,49513,49520,49522,49523,49525,49530,49532,49533,49534,49537,49538,49540,49541,49542,49543,49551,49554,49555,49556,49559,49560,49563,49566,49570,49573,49574,49579,49583,49584,49591,49602,49603,49605,49606,49607,49608,49610,49618,49620,49621,49623,49628,49631,49637,49645,49646,49649,49650,49651,49652,49657,49660,49661,49663,49664,49669,49674,49677,49679,49683,49685,49686,49690,49691,49700,49705,49706,49708,49709,49713,49714,49715,49722,49724,49725,49726,49730,49733,49734,49735,49736,49741,49742,49746,49749,49750,49751,49753,49754,49759,49761,49767,49768,49769,49770,49779,49787,49790,49794,49796,49800,49802,49804,49805,49807,49808,49809,49810,49814,49817,49818,49820,49826,49828,49829,49831,49838,49839,49840,49842,49847,49849,49852,49853,49854,49857,49863,49867,49873,49877,49879,49880,49881,49884,49888,49889,49890,49891,49893,49894,49895,49896,49899,49901,49902,49903,49904,49905,49906,49910,49912,49913,49914,49916,49918,49919,49920,49921,49922,49923,49925,49927,49928,49932,49933,49934,49936,49939,49951,49954,49956,49958,49959,49964,49965,49966,49969,49970,49971,49972,49976,49979,49981,49982,49985,49986,49987,49988,49990 101 | --------------------------------------------------------------------------------