├── .gitignore ├── .idea ├── deployment.xml ├── inspectionProfiles │ └── Project_Default.xml ├── vcs.xml └── webServers.xml ├── README.md ├── data_handler ├── __init__.py ├── dataset.py ├── dataset_factory.py └── incremental_loader.py ├── experiment ├── __init__.py └── experiment.py ├── images ├── 2step.png ├── bias.png ├── confusion.png ├── protocol.png └── thresholdmoving.png ├── mnist_missing_experiment.py ├── model ├── __init__.py ├── misc_functions.py ├── model_factory.py ├── res_utils.py ├── resnet32.py └── test_model.py ├── plots ├── herding │ └── noherding │ │ └── NOHERDINGJSONDump2.txt ├── lwf │ ├── 1 │ ├── 2 │ ├── 3 │ ├── 4 │ └── 5 └── memorybudget │ └── os │ ├── 500 │ └── 2000 ├── plotter ├── __init__.py └── plotter.py ├── requirements.txt ├── run_experiment.py ├── trainer ├── __init__.py ├── evaluator.py └── trainer.py └── utils ├── Colorer.py ├── __init__.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | 104 | # Pycharm 105 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 106 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 107 | 108 | # User-specific stuff: 109 | .idea/**/workspace.xml 110 | .idea/**/tasks.xml 111 | .idea/dictionaries 112 | 113 | # Sensitive or high-churn files: 114 | .idea/**/dataSources/ 115 | .idea/**/dataSources.ids 116 | .idea/**/dataSources.xml 117 | .idea/**/dataSources.local.xml 118 | .idea/**/sqlDataSources.xml 119 | .idea/**/dynamic.xml 120 | .idea/**/uiDesigner.xml 121 | 122 | # Gradle: 123 | .idea/**/gradle.xml 124 | .idea/**/libraries 125 | 126 | # CMake 127 | cmake-build-debug/ 128 | cmake-build-release/ 129 | 130 | # Mongo Explorer plugin: 131 | .idea/**/mongoSettings.xml 132 | 133 | ## File-based project format: 134 | *.iws 135 | 136 | ## Plugin-specific files: 137 | 138 | # IntelliJ 139 | out/ 140 | 141 | # mpeltonen/sbt-idea plugin 142 | .idea_modules/ 143 | 144 | # JIRA plugin 145 | atlassian-ide-plugin.xml 146 | 147 | # Cursive Clojure plugin 148 | .idea/replstate.xml 149 | 150 | # Crashlytics plugin (for Android Studio and IntelliJ) 151 | com_crashlytics_export_strings.xml 152 | crashlytics.properties 153 | crashlytics-build.properties 154 | fabric.properties 155 | 156 | # Data files 157 | *.data 158 | *.jpg 159 | *.xml 160 | */*.xml 161 | -------------------------------------------------------------------------------- /.idea/deployment.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 15 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 22 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/webServers.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 14 | 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Revisiting Distillation and Incremental Classifier Learning 2 | Accepted at ACCV18. Pre-print is available at : http://arxiv.org/abs/1807.02802 3 | 4 | Citing the paper : 5 | ``` bash 6 | @inproceedings{javed2018revisiting, 7 | title={Revisiting distillation and incremental classifier learning}, 8 | author={Javed, Khurram and Shafait, Faisal}, 9 | booktitle={Asian Conference on Computer Vision}, 10 | pages={3--17}, 11 | year={2018}, 12 | organization={Springer} 13 | } 14 | ``` 15 | ## Interface to Run Experiments 16 | 17 | ``` bash 18 | usage: runExperiment.py [-h] [--batch-size N] [--lr LR] 19 | [--schedule SCHEDULE [SCHEDULE ...]] 20 | [--gammas GAMMAS [GAMMAS ...]] [--momentum M] 21 | [--no-cuda] [--random-init] [--no-distill] 22 | [--distill-only-exemplars] [--no-random] 23 | [--no-herding] [--seeds SEEDS [SEEDS ...]] 24 | [--log-interval N] [--model-type MODEL_TYPE] 25 | [--name NAME] [--outputDir OUTPUTDIR] [--upsampling] 26 | [--pp] [--distill-step] [--hs] 27 | [--unstructured-size UNSTRUCTURED_SIZE] 28 | [--alphas ALPHAS [ALPHAS ...]] [--decay DECAY] 29 | [--alpha-increment ALPHA_INCREMENT] [--l1 L1] 30 | [--step-size STEP_SIZE] [--T T] 31 | [--memory-budgets MEMORY_BUDGETS [MEMORY_BUDGETS ...]] 32 | [--epochs-class EPOCHS_CLASS] [--dataset DATASET] 33 | [--lwf] [--no-nl] [--rand] [--adversarial] 34 | ``` 35 | 36 | Default configurations can be used to run with same parameters as used by iCaRL. 37 | Simply run: 38 | ``` bash 39 | python run_experiment.py 40 | ``` 41 | ## Dependencies 42 | 43 | 1. Pytorch 0.3.0.post4 44 | 2. Python 3.6 45 | 3. torchnet (https://github.com/pytorch/tnt) 46 | 4. tqdm (pip install tqdm) 47 | 48 | Please see requirements.txt for a complete list. 49 | 50 | ## Setting up enviroment 51 | The easiest way to install the required dependencies is to use conda package manager. 52 | 1. Install Anaconda with Python 3 53 | 2. Install pytorch and torchnet 54 | 3. Install tqdm (pip install progressbar2) 55 | Done. 56 | 57 | ## Branches 58 | 1. iCaRL + Dynamic Threshold Moving is implemented in "Autoencoders" branch. 59 | 60 | ======= 61 | ## Selected Results 62 | ### Removing Bias by Dynamic Threshold Moving 63 | ![alt text](https://github.com/Khurramjaved96/incremental-learning/blob/autoencoders/images/thresholdmoving.png "Dynamic Threshold Moving on MNIST") 64 | Result of threshold moving with T = 2 and 5. Note that different scale is used for 65 | the y axis, and using higher temperature in general results in less bias. 66 | 67 | ### Confusion Matrix with and without Dynamic Threshold Moving 68 | ![alt text](https://github.com/Khurramjaved96/incremental-learning/blob/autoencoders/images/confusion.png "Dynamic Threshold Moving Confusion Matrix") 69 | Confusion matrix of results of the classifier with (right) and without (left) threshold 70 | moving with T=2. We removed the first five classes of MNIST from the train set and only 71 | distilled the knowledge of these classes using a network trained on all classes. Without 72 | threshold moving the model struggled on the older classes. With threshold moving, however, 73 | not only was it able to classify unseen classes nearly perfectly, but also its performance did 74 | not deteriorate on new classes 75 | 76 | 77 | ## FAQs 78 | ### How do I implement more models? 79 | A. Add the model in model/ModelFactory and make sure the forward method of the model satisfy the API of model/resnet32.py 80 | ### How do I add a new dataset? 81 | A. Add the new dataset in DatasetFactory and specify the details in the dataHandler/dataset.py class. Make sure the dataset implements all the variables set by other datasets. 82 | 83 | ## References 84 | [1] Geoffrey Hinton, Oriol Vinyals, and Jeff Dean. Distilling the knowledge in a neural network. arXiv preprint arXiv:1503.02531, 2015 85 | 86 | [2] Sylvestre-Alvise Rebuffi, Alexander Kolesnikov, Georg Sperl, and Christoph H Lampert. Icarl: Incremental classifier and representation learning. In Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pages 2001–2010, 2017. 87 | 88 | [3] Zhizhong Li and Derek Hoiem. Learning without forgetting. IEEE Transactions on Pattern Analysis and Machine Intelligence, 2017. 89 | -------------------------------------------------------------------------------- /data_handler/__init__.py: -------------------------------------------------------------------------------- 1 | from data_handler.dataset_factory import * 2 | from data_handler.incremental_loader import * -------------------------------------------------------------------------------- /data_handler/dataset.py: -------------------------------------------------------------------------------- 1 | ''' Incremental-Classifier Learning 2 | Authors : Khurram Javed, Muhammad Talha Paracha 3 | Maintainer : Khurram Javed 4 | Lab : TUKL-SEECS R&D Lab 5 | Email : 14besekjaved@seecs.edu.pk ''' 6 | 7 | from torchvision import datasets, transforms 8 | import torch 9 | import numpy 10 | 11 | # To incdude a new Dataset, inherit from Dataset and add all the Dataset specific parameters here. 12 | # Goal : Remove any data specific parameters from the rest of the code 13 | 14 | class Dataset(): 15 | ''' 16 | Base class to reprenent a Dataset 17 | ''' 18 | 19 | def __init__(self, classes, name, labels_per_class_train, labels_per_class_test): 20 | self.classes = classes 21 | self.name = name 22 | self.train_data = None 23 | self.test_data = None 24 | self.labels_per_class_train = labels_per_class_train 25 | self.labels_per_class_test = labels_per_class_test 26 | 27 | 28 | class MNIST(Dataset): 29 | ''' 30 | Class to include MNIST specific details 31 | ''' 32 | 33 | def __init__(self): 34 | super().__init__(10, "MNIST", 6000, 1000) 35 | 36 | self.train_transform = transforms.Compose( 37 | [transforms.Resize((32,32)), 38 | transforms.ToTensor(), 39 | transforms.Normalize((0.1307,), (0.3081,))]) 40 | 41 | self.test_transform = transforms.Compose( 42 | [transforms.Resize((32,32)), transforms.ToTensor(), 43 | transforms.Normalize((0.1307,), (0.3081,))]) 44 | 45 | self.train_data = datasets.MNIST("data", train=True, transform=self.train_transform, download=True) 46 | 47 | self.test_data = datasets.MNIST("data", train=False, transform=self.test_transform, download=True) 48 | 49 | def get_random_instance(self): 50 | instance = torch.from_numpy(numpy.random.uniform(low=-1, high=1, size=(32, 32))).float() 51 | instance.unsqueeze_(0) 52 | return instance 53 | 54 | 55 | class CIFAR100(Dataset): 56 | def __init__(self): 57 | super().__init__(100, "CIFAR100", 500, 100) 58 | 59 | mean = [0.5071, 0.4867, 0.4408] 60 | std = [0.2675, 0.2565, 0.2761] 61 | 62 | self.train_transform = transforms.Compose([ 63 | transforms.RandomCrop(32, padding=4), 64 | transforms.RandomHorizontalFlip(), 65 | transforms.ToTensor(), 66 | transforms.Normalize(mean, std), 67 | ]) 68 | # DO NOT DO DATA NORMALIZATION; TO IMPLEMENT DATA NORMALIZATION, MAKE SURE THAT DATA NORMALIZATION IS STILL APPLIED IN GET_ITEM FUNCTION OF INCREMENTAL LOADER 69 | self.train_transform = transforms.Compose( 70 | [transforms.RandomHorizontalFlip(), 71 | transforms.RandomCrop(32, padding=4), 72 | transforms.ToTensor(), ]) 73 | 74 | self.test_transform = transforms.Compose( 75 | [transforms.ToTensor(), ]) 76 | 77 | self.train_data = datasets.CIFAR100("data", train=True, transform=self.train_transform, download=True) 78 | 79 | self.test_data = datasets.CIFAR100("data", train=False, transform=self.test_transform, download=True) 80 | 81 | 82 | class CIFAR10(Dataset): 83 | def __init__(self): 84 | super().__init__(10, "CIFAR10", 5000, 1000) 85 | 86 | self.train_transform = transforms.Compose([ 87 | transforms.RandomCrop(32, padding=4), 88 | transforms.RandomHorizontalFlip(), 89 | transforms.ToTensor(), 90 | transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]) 91 | 92 | self.train_transform = transforms.Compose( 93 | [transforms.RandomHorizontalFlip(), 94 | transforms.RandomCrop(32, padding=4), 95 | transforms.ToTensor(), ]) 96 | 97 | self.test_transform = transforms.Compose( 98 | [transforms.ToTensor(), ]) 99 | 100 | self.train_data = datasets.CIFAR10("data", train=True, transform=self.train_transform, download=True) 101 | 102 | self.test_data = datasets.CIFAR10("data", train=False, transform=self.test_transform, download=True) 103 | -------------------------------------------------------------------------------- /data_handler/dataset_factory.py: -------------------------------------------------------------------------------- 1 | ''' Incremental-Classifier Learning 2 | Authors : Khurram Javed, Muhammad Talha Paracha 3 | Maintainer : Khurram Javed 4 | Lab : TUKL-SEECS R&D Lab 5 | Email : 14besekjaved@seecs.edu.pk ''' 6 | 7 | import data_handler.dataset as data 8 | 9 | 10 | class DatasetFactory: 11 | def __init__(self): 12 | pass 13 | 14 | @staticmethod 15 | def get_dataset(name): 16 | if name == "MNIST": 17 | return data.MNIST() 18 | elif name == "CIFAR100": 19 | return data.CIFAR100() 20 | elif name == "CIFAR10": 21 | return data.CIFAR10() 22 | else: 23 | print("Unsupported Dataset") 24 | assert False 25 | -------------------------------------------------------------------------------- /data_handler/incremental_loader.py: -------------------------------------------------------------------------------- 1 | ''' Incremental-Classifier Learning 2 | Authors : Khurram Javed, Muhammad Talha Paracha 3 | Maintainer : Khurram Javed 4 | Lab : TUKL-SEECS R&D Lab 5 | Email : 14besekjaved@seecs.edu.pk ''' 6 | 7 | import copy 8 | import logging 9 | 10 | import numpy as np 11 | import torch 12 | import torch.utils.data as td 13 | from PIL import Image 14 | from torch.autograd import Variable 15 | 16 | logger = logging.getLogger('iCARL') 17 | 18 | 19 | class IncrementalLoader(td.Dataset): 20 | def __init__(self, data, labels, class_size, classes, active_classes, transform=None, cuda=False, 21 | oversampling=True): 22 | oversampling = not oversampling 23 | self.len = class_size * len(active_classes) 24 | sort_index = np.argsort(labels) 25 | self.class_size = class_size 26 | if "torch" in str(type(data)): 27 | data = data.numpy() 28 | self.data = data[sort_index] 29 | labels = np.array(labels) 30 | self.labels = labels[sort_index] 31 | self.labelsNormal = np.copy(self.labels) 32 | self.transform = transform 33 | self.active_classes = active_classes 34 | self.limited_classes = {} 35 | self.total_classes = classes 36 | self.means = {} 37 | self.cuda = cuda 38 | self.weights = np.zeros(self.total_classes * self.class_size) 39 | self.indices = {} 40 | self.__class_indices() 41 | self.over_sampling = oversampling 42 | # f(label) = new_label. We do this to ensure labels are in increasing order. For example, even if first increment chooses class 1,5,6, the training labels will be 0,1,2 43 | self.indexMapper = {} 44 | self.no_transformation = False 45 | self.transformLabels() 46 | 47 | def transformLabels(self): 48 | '''Change labels to one hot coded vectors''' 49 | b = np.zeros((self.labels.size, self.labels.max() + 1)) 50 | b[np.arange(self.labels.size), self.labels] = 1 51 | self.labels = b 52 | 53 | def __class_indices(self): 54 | cur = 0 55 | for temp in range(0, self.total_classes): 56 | cur_len = len(np.nonzero(np.uint8(self.labels == temp))[0]) 57 | self.indices[temp] = (cur, cur + cur_len) 58 | cur += cur_len 59 | 60 | def add_class(self, n): 61 | logger.debug("Adding Class %d", n) 62 | if n in self.active_classes: 63 | return 64 | # Mapping each new added classes to new label in increasing order; we switch the label so that the resulting confusion matrix is always in order 65 | # regardless of order of classes used for incremental training. 66 | indices = len(self.indexMapper) 67 | if not n in self.indexMapper: 68 | self.indexMapper[n] = indices 69 | self.active_classes.append(n) 70 | self.len = self.class_size * len(self.active_classes) 71 | self.__update_length() 72 | 73 | def __update_length(self): 74 | ''' 75 | Function to compute length of the active elements of the data. 76 | :return: 77 | ''' 78 | 79 | len_var = 0 80 | for a in self.active_classes: 81 | len_var += self.indices[a][1] - self.indices[a][0] 82 | self.len = len_var 83 | 84 | return 85 | 86 | def limit_class(self, n, k): 87 | if k == 0: 88 | logging.warning("Removing class %d", n) 89 | self.remove_class(n) 90 | self.__update_length() 91 | return False 92 | if k > self.indices[n][1] - self.indices[n][0]: 93 | k = self.indices[n][1] - self.indices[n][0] 94 | if n in self.limited_classes: 95 | self.limited_classes[n] = k 96 | # Remove this line; this turns off oversampling 97 | if not self.over_sampling: 98 | self.indices[n] = (self.indices[n][0], self.indices[n][0] + k) 99 | self.__update_length() 100 | return False 101 | else: 102 | if not self.over_sampling: 103 | self.indices[n] = (self.indices[n][0], self.indices[n][0] + k) 104 | self.limited_classes[n] = k 105 | self.__update_length() 106 | return True 107 | 108 | def limit_class_and_sort(self, n, k, model): 109 | ''' This function should only be called the first time a class is limited. To change the limitation, 110 | call the limiClass(self, n, k) function 111 | 112 | :param n: Class to limit 113 | :param k: No of exemplars to keep 114 | :param model: Features extracted from this model for sorting. 115 | :return: 116 | ''' 117 | 118 | if self.limit_class(n, k): 119 | start = self.indices[n][0] 120 | end = self.indices[n][1] 121 | buff = np.zeros(self.data[start:end].shape) 122 | images = [] 123 | # Get input features of all the images of the class 124 | for ind in range(start, end): 125 | img = self.data[ind] 126 | if "torch" in str(type(img)): 127 | img = img.numpy() 128 | img = Image.fromarray(img) 129 | 130 | if self.transform is not None: 131 | img = self.transform(img) 132 | images.append(img) 133 | data_tensor = torch.stack(images) 134 | if self.cuda: 135 | data_tensor = data_tensor.cuda() 136 | 137 | # Get features 138 | features = model.forward(Variable(data_tensor), True) 139 | features_copy = copy.deepcopy(features.data) 140 | mean = torch.mean(features, 0, True) 141 | list_of_selected = [] 142 | 143 | # Select exemplars 144 | for exmp_no in range(0, min(k, self.class_size)): 145 | if exmp_no > 0: 146 | to_add = torch.sum(features_copy[0:exmp_no], dim=0).unsqueeze(0) 147 | if self.cuda: 148 | to_add = to_add.cuda() 149 | features_temp = (features + Variable(to_add)) / (exmp_no + 1) - mean 150 | else: 151 | features_temp = features - mean 152 | features_norm = torch.norm(features_temp.data, 2, dim=1) 153 | if self.cuda: 154 | features_norm = features_norm.cpu() 155 | arg_min = np.argmin(features_norm.numpy()) 156 | if arg_min in list_of_selected: 157 | assert (False) 158 | list_of_selected.append(arg_min) 159 | buff[exmp_no] = self.data[start + arg_min] 160 | features_copy[exmp_no] = features.data[arg_min] 161 | features[arg_min] = features[arg_min] + 1000 162 | # logger.debug("Arg Min: %d", arg_min) 163 | print("Exmp shape", buff[0:min(k, self.class_size)].shape) 164 | self.data[start:start + min(k, self.class_size)] = buff[0:min(k, self.class_size)] 165 | 166 | self.__update_length() 167 | 168 | def remove_class(self, n): 169 | while n in self.active_classes: 170 | self.active_classes.remove(n) 171 | self.__update_length() 172 | 173 | def __len__(self): 174 | return self.len 175 | 176 | def get_start_index(self, n): 177 | ''' 178 | 179 | :param n: 180 | :return: Returns starting index of classs n 181 | ''' 182 | return self.indices[n][0] 183 | 184 | def getIndexElem(self, bool): 185 | self.no_transformation = bool 186 | 187 | def __getitem__(self, index): 188 | ''' 189 | Replacing this with a more efficient implemnetation selection; removing c 190 | :param index: 191 | :return: 192 | ''' 193 | assert (index < len(self.data)) 194 | assert (index < self.len) 195 | 196 | length = 0 197 | temp_a = 0 198 | old_len = 0 199 | for a in self.active_classes: 200 | temp_a = a 201 | old_len = length 202 | length += self.indices[a][1] - self.indices[a][0] 203 | if length > index: 204 | break 205 | base = self.indices[temp_a][0] 206 | incre = index - old_len 207 | if temp_a in self.limited_classes: 208 | incre = incre % self.limited_classes[temp_a] 209 | index = base + incre 210 | img = self.data[index] 211 | if "torch" in str(type(img)): 212 | img = img.numpy() 213 | img = Image.fromarray(img) 214 | if self.transform is not None: 215 | img = self.transform(img) 216 | 217 | if not self.labelsNormal[index] in self.active_classes: 218 | print("Active classes", self.active_classes) 219 | print("Label ", self.labelsNormal[index]) 220 | assert (False) 221 | if self.no_transformation: 222 | return img, index, self.labelsNormal[index] 223 | return img, self.labels[index], self.labelsNormal[index] 224 | -------------------------------------------------------------------------------- /experiment/__init__.py: -------------------------------------------------------------------------------- 1 | from experiment.experiment import * -------------------------------------------------------------------------------- /experiment/experiment.py: -------------------------------------------------------------------------------- 1 | ''' Incremental-Classifier Learning 2 | Authors : Khurram Javed, Muhammad Talha Paracha 3 | Maintainer : Khurram Javed 4 | Lab : TUKL-SEECS R&D Lab 5 | Email : 14besekjaved@seecs.edu.pk ''' 6 | 7 | import json 8 | import os 9 | import subprocess 10 | 11 | 12 | class experiment: 13 | ''' 14 | Class to store results of any experiment 15 | ''' 16 | 17 | def __init__(self, name, args, output_dir="../"): 18 | try: 19 | self.gitHash = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode("utf-8") 20 | except: 21 | self.gitHash ="Not a Git Repo" 22 | print(self.gitHash) 23 | if not args is None: 24 | self.name = name 25 | self.params = vars(args) 26 | self.results = {} 27 | self.dir = output_dir 28 | 29 | import datetime 30 | now = datetime.datetime.now() 31 | rootFolder = str(now.day) + str(now.month) + str(now.year) 32 | if not os.path.exists(output_dir + rootFolder): 33 | os.makedirs(output_dir + rootFolder) 34 | self.name = rootFolder + "/" + self.name 35 | ver = 0 36 | 37 | while os.path.exists(output_dir + self.name + "_" + str(ver)): 38 | ver += 1 39 | 40 | os.makedirs(output_dir + self.name + "_" + str(ver)) 41 | self.path = output_dir + self.name + "_" + str(ver) + "/" + name 42 | 43 | self.results["Temp Results"] = [[1, 2, 3, 4], [5, 6, 2, 6]] 44 | 45 | def store_json(self): 46 | with open(self.path + "JSONDump.txt", 'w') as outfile: 47 | json.dump(json.dumps(self.__dict__), outfile) 48 | 49 | 50 | import argparse 51 | 52 | if __name__ == "__main__": 53 | parser = argparse.ArgumentParser(description='iCarl2.0') 54 | args = parser.parse_args() 55 | e = experiment("TestExperiment", args) 56 | e.store_json() 57 | -------------------------------------------------------------------------------- /images/2step.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khurramjaved96/incremental-learning/ff0c48de37a4bded055278186a7dc32f7d7411f6/images/2step.png -------------------------------------------------------------------------------- /images/bias.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khurramjaved96/incremental-learning/ff0c48de37a4bded055278186a7dc32f7d7411f6/images/bias.png -------------------------------------------------------------------------------- /images/confusion.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khurramjaved96/incremental-learning/ff0c48de37a4bded055278186a7dc32f7d7411f6/images/confusion.png -------------------------------------------------------------------------------- /images/protocol.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khurramjaved96/incremental-learning/ff0c48de37a4bded055278186a7dc32f7d7411f6/images/protocol.png -------------------------------------------------------------------------------- /images/thresholdmoving.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khurramjaved96/incremental-learning/ff0c48de37a4bded055278186a7dc32f7d7411f6/images/thresholdmoving.png -------------------------------------------------------------------------------- /mnist_missing_experiment.py: -------------------------------------------------------------------------------- 1 | ''' Incremental-Classifier Learning 2 | Authors : Khurram Javed, Muhammad Talha Paracha 3 | Maintainer : Khurram Javed 4 | Lab : TUKL-SEECS R&D Lab 5 | Email : 14besekjaved@seecs.edu.pk ''' 6 | 7 | from __future__ import print_function 8 | 9 | import argparse 10 | import logging 11 | 12 | import torch 13 | import torch.utils.data as td 14 | from tqdm import tqdm 15 | 16 | import data_handler 17 | import experiment as ex 18 | import model 19 | import plotter as plt 20 | import trainer 21 | 22 | parser = argparse.ArgumentParser(description='iCarl2.0') 23 | 24 | parser.add_argument('--batch-size', type=int, default=32, metavar='N', 25 | help='input batch size for training (default: 35)') 26 | parser.add_argument('--lr', type=float, default=0.1, metavar='LR', 27 | help='learning rate (default: 0.1)') 28 | parser.add_argument('--schedule', type=int, nargs='+', default=[15, 23, 28], 29 | help='Decrease learning rate at these epochs.') 30 | parser.add_argument('--gammas', type=float, nargs='+', default=[0.2, 0.2, 0.2], 31 | help='LR is multiplied by gamma on schedule, number of gammas should be equal to schedule') 32 | parser.add_argument('--momentum', type=float, default=0.9, metavar='M', 33 | help='SGD momentum (default: 0.9)') 34 | parser.add_argument('--no-cuda', action='store_true', default=False, 35 | help='disables CUDA training') 36 | parser.add_argument('--random-init', action='store_true', default=True, 37 | help='To initialize model using previous weights or random weights in each iteration') 38 | parser.add_argument('--no-distill', action='store_true', default=False, 39 | help='disable distillation loss') 40 | parser.add_argument('--no-random', action='store_true', default=False, 41 | help='Disable random shuffling of classes') 42 | parser.add_argument('--no-herding', action='store_true', default=True, 43 | help='Disable herding for NMC') 44 | parser.add_argument('--distill-step', action='store_true', default=False, 45 | help='Should I .') 46 | parser.add_argument('--seeds', type=int, nargs='+', default=[23423], 47 | help='Seeds values to be used') 48 | parser.add_argument('--log-interval', type=int, default=2, metavar='N', 49 | help='how many batches to wait before logging training status') 50 | parser.add_argument('--model-type', default="resnet32", 51 | help='model type to be used. Example : resnet32, resnet20, densenet, test') 52 | parser.add_argument('--name', default="MNIST_SCALE_EXPERIMENT", 53 | help='Name of the experiment') 54 | parser.add_argument('--outputDir', default="../", 55 | help='Directory to store the results; the new folder will be created ' 56 | 'in the specified directory to save the results.') 57 | parser.add_argument('--upsampling', action='store_true', default=False, 58 | help='Do not do upsampling.') 59 | parser.add_argument('--pp', action='store_true', default=False, 60 | help='Privacy perserving') 61 | parser.add_argument('--unstructured-size', type=int, default=0, help='Number of epochs for each increment') 62 | parser.add_argument('--no-nl', action='store_true', default=False, 63 | help='No Normal Loss') 64 | 65 | parser.add_argument('--alphas', type=float, nargs='+', default=[1.0], 66 | help='Weight given to new classes vs old classes in loss') 67 | parser.add_argument('--decay', type=float, default=0.00005, help='Weight decay (L2 penalty).') 68 | parser.add_argument('--alpha-increment', type=float, default=1.0, help='Weight decay (L2 penalty).') 69 | parser.add_argument('--step-size', type=int, default=10, help='How many classes to add in each increment') 70 | parser.add_argument('--T', type=float, default=1, help='Tempreture used for softening the targets') 71 | parser.add_argument('--memory-budgets', type=int, nargs='+', default=[80000], 72 | help='How many images can we store at max. 0 will result in fine-tuning') 73 | parser.add_argument('--epochs-class', type=int, default=30, help='Number of epochs for each increment') 74 | parser.add_argument('--dataset', default="MNIST", help='Dataset to be used; example CIFAR, MNIST') 75 | parser.add_argument('--lwf', action='store_true', default=False, 76 | help='Use learning without forgetting. Ignores memory-budget ' 77 | '("Learning with Forgetting," Zhizhong Li, Derek Hoiem)') 78 | parser.add_argument('--rand', action='store_true', default=False, 79 | help='Replace exemplars with random instances') 80 | parser.add_argument('--adversarial', action='store_true', default=False, 81 | help='Replace exemplars with adversarial instances') 82 | parser.add_argument('--distill-step', action='store_true', default=False, 83 | help='Ignore some logits for computing distillation loss. I believe this should work.') 84 | 85 | args = parser.parse_args() 86 | args.cuda = not args.no_cuda and torch.cuda.is_available() 87 | 88 | dataset = data_handler.DatasetFactory.get_dataset(args.dataset) 89 | 90 | # Checks to make sure parameters are sane 91 | if args.step_size < 2: 92 | logging.error("Step size of less than 2 will result in no-learning;") 93 | assert False 94 | 95 | # Support for running multiple experiments 96 | for seed in args.seeds: 97 | # Support to do hyperparamter search over alphas 98 | for at in args.alphas: 99 | args.alpha = at 100 | # Run experiments on multiple memory budgets. 101 | for m in args.memory_budgets: 102 | args.memory_budget = m 103 | # In case of lwf, memory-budget = 0 104 | if args.lwf: 105 | args.memory_budget = 0 106 | 107 | args.seed = seed 108 | torch.manual_seed(seed) 109 | if args.cuda: 110 | torch.cuda.manual_seed(seed) 111 | 112 | # Loader used for training data 113 | train_dataset_loader = data_handler.IncrementalLoader(dataset.train_data.train_data, 114 | dataset.train_data.train_labels, 115 | dataset.labels_per_class_train, 116 | dataset.classes, [], 117 | transform=dataset.train_transform, 118 | cuda=args.cuda, oversampling=not args.upsampling, 119 | ) 120 | # Special loader use to compute ideal NMC; i.e, NMC that using all the data points 121 | # to compute the mean embedding 122 | train_dataset_loader_nmc = data_handler.IncrementalLoader(dataset.train_data.train_data, 123 | dataset.train_data.train_labels, 124 | dataset.labels_per_class_train, 125 | dataset.classes, [], 126 | transform=dataset.train_transform, 127 | cuda=args.cuda, oversampling=not args.upsampling, 128 | ) 129 | # Loader for test data. 130 | test_dataset_loader = data_handler.IncrementalLoader(dataset.test_data.test_data, 131 | dataset.test_data.test_labels, 132 | dataset.labels_per_class_test, dataset.classes, 133 | [], transform=dataset.test_transform, cuda=args.cuda, 134 | ) 135 | 136 | kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {} 137 | 138 | # Iterator to iterate over training data. 139 | train_iterator = torch.utils.data.DataLoader(train_dataset_loader, 140 | batch_size=args.batch_size, shuffle=True, **kwargs) 141 | # Iterator to iterate over all training data (Equivalent to memory-budget = infitie 142 | train_iterator_nmc = torch.utils.data.DataLoader(train_dataset_loader_nmc, 143 | batch_size=args.batch_size, shuffle=True, **kwargs) 144 | # Iterator to iterate over test data 145 | test_iterator = torch.utils.data.DataLoader( 146 | test_dataset_loader, 147 | batch_size=args.batch_size, shuffle=True, **kwargs) 148 | 149 | # Get the required model 150 | myModel = model.ModelFactory.get_model(args.model_type, args.dataset) 151 | if args.cuda: 152 | myModel.cuda() 153 | 154 | # Define an experiment. 155 | my_experiment = ex.experiment(args.name, args) 156 | 157 | # Set logger parameter. You can get the logger object with name 'iCARL' in any file and it will work. 158 | logger = logging.getLogger('iCARL') 159 | logger.setLevel(logging.DEBUG) 160 | 161 | # Detailed logging in the log file. 162 | fh = logging.FileHandler(my_experiment.path + ".log") 163 | fh.setLevel(logging.DEBUG) 164 | 165 | fh2 = logging.FileHandler("../temp.log") 166 | fh2.setLevel(logging.DEBUG) 167 | 168 | # Info level logging in stdout. (No debug messages here). 169 | ch = logging.StreamHandler() 170 | ch.setLevel(logging.INFO) 171 | 172 | # Format the logging messages 173 | formatter = logging.Formatter('%(asctime)s - {%(filename)s:%(lineno)d} - %(levelname)s - %(message)s') 174 | fh.setFormatter(formatter) 175 | fh2.setFormatter(formatter) 176 | ch.setFormatter(formatter) 177 | 178 | logger.addHandler(fh) 179 | logger.addHandler(fh2) 180 | logger.addHandler(ch) 181 | 182 | # Define the optimizer used in the experiment 183 | optimizer = torch.optim.SGD(myModel.parameters(), args.lr, momentum=args.momentum, 184 | weight_decay=args.decay, nesterov=True) 185 | 186 | # Trainer object used for training 187 | my_trainer = trainer.Trainer(train_iterator, test_iterator, dataset, myModel, args, optimizer, 188 | train_iterator_nmc) 189 | 190 | # Remove this parameters somehow. 191 | x = [] 192 | y = [] 193 | test_set_classifier = [] 194 | train_set_classifier = [] 195 | classifier_scaled = [] 196 | classifier_scaled_grad = [] 197 | nmc_ideal_cum = [] 198 | 199 | # Evaluator objects to measure the accuracy of the model 200 | nmc_ideal = trainer.EvaluatorFactory.get_evaluator("nmc", args.cuda) 201 | 202 | t_classifier = trainer.EvaluatorFactory.get_evaluator("trainedClassifier", args.cuda) 203 | 204 | class_group = 0 205 | # Adding all the classes initially to train the base model 206 | my_trainer.increment_classes(class_group) 207 | my_trainer.update_frozen_model() 208 | epoch = 0 209 | 210 | # Running epochs_class epochs 211 | for epoch in tqdm(range(0, args.epochs_class), desc="Training with all Data"): 212 | my_trainer.update_lr(epoch) 213 | my_trainer.train(epoch) 214 | # print(my_trainer.threshold) 215 | if epoch % args.log_interval == (args.log_interval - 1): 216 | tError = t_classifier.evaluate(my_trainer.model, train_iterator) 217 | logger.debug("********CURRENT EPOCH********* %d", epoch) 218 | logger.debug("Train Classifier: %0.2f", tError) 219 | logger.debug("Test Classifier: %0.2f", t_classifier.evaluate(my_trainer.model, test_iterator)) 220 | logger.debug("Test Classifier Scaled: %0.2f", 221 | t_classifier.evaluate(my_trainer.model, test_iterator, my_trainer.dynamic_threshold, 222 | False, 223 | my_trainer.older_classes, args.step_size)) 224 | 225 | # Compute final accuracies 226 | testError = t_classifier.evaluate(my_trainer.model, test_iterator) 227 | testErrorScaled = t_classifier.evaluate(my_trainer.model, test_iterator, my_trainer.dynamic_threshold, 228 | False, 229 | my_trainer.older_classes, args.step_size) 230 | testErrorGScaled = t_classifier.evaluate(my_trainer.model, test_iterator, 231 | my_trainer.gradient_threshold_unreported_experiment, False, 232 | my_trainer.older_classes, args.step_size) 233 | classifier_scaled_grad.append(testErrorGScaled) 234 | 235 | # Append result for plotting 236 | x.append(0) 237 | test_set_classifier.append(testError) 238 | classifier_scaled.append(testErrorScaled) 239 | # Logging result in the terminal 240 | logger.info("Orig Model Test Error %0.2f", testError) 241 | logger.info("Orig Model Test Scaled Error %0.2f", testErrorScaled) 242 | logger.info("Orig Model Test GScaled Error %0.2f", testErrorGScaled) 243 | my_trainer.update_frozen_model() 244 | # Computing NMC loss 245 | nmc_ideal.update_means(my_trainer.model, train_iterator_nmc, dataset.classes) 246 | testY_ideal = nmc_ideal.evaluate(my_trainer.model, test_iterator) 247 | nmc_ideal_cum.append(testY_ideal) 248 | 249 | # Getting order of class removal for experiments 250 | listOfElem = list(range(10)) 251 | import random 252 | 253 | random.seed(args.seed) 254 | # random.shuffle(listOfElem) 255 | listOfElem.pop() 256 | logger.info("Order of experiment" + ",".join([str(i) for i in listOfElem])) 257 | counter = 0 258 | for xTemp in listOfElem: 259 | counter += 1 260 | logger.info("Removing class %d", xTemp) 261 | # Set up model 262 | my_trainer.reset_dynamic_threshold() 263 | my_trainer.limit_class(xTemp, 0, False) 264 | my_trainer.randomly_init_model() 265 | 266 | # Remove model 267 | for epoch in tqdm(range(0, args.epochs_class), desc="Training without class " + str(xTemp)): 268 | my_trainer.update_lr(epoch) 269 | my_trainer.train(epoch) 270 | # print(my_trainer.threshold) 271 | if epoch % args.log_interval == (args.log_interval - 1): 272 | tError = t_classifier.evaluate(my_trainer.model, train_iterator) 273 | logger.info("********CURRENT EPOCH********* %0.2f", epoch) 274 | logger.info("Train Classifier: %0.2f", tError) 275 | logger.info("Test Classifier: %0.2f", t_classifier.evaluate(my_trainer.model, test_iterator)) 276 | logger.info("Test Classifier Scaled: %0.2f", 277 | t_classifier.evaluate(my_trainer.model, test_iterator, my_trainer.dynamic_threshold, 278 | False, 279 | my_trainer.older_classes, args.step_size)) 280 | 281 | # Evaluate the learned classifier 282 | img = None 283 | 284 | logger.info("Test Classifier Final: %0.2f", t_classifier.evaluate(my_trainer.model, test_iterator)) 285 | logger.info("Test Classifier Final Scaled: %0.2f", 286 | t_classifier.evaluate(my_trainer.model, test_iterator, my_trainer.dynamic_threshold, False, 287 | my_trainer.older_classes, args.step_size)) 288 | 289 | classifier_scaled.append( 290 | t_classifier.evaluate(my_trainer.model, test_iterator, my_trainer.dynamic_threshold, False, 291 | my_trainer.older_classes, args.step_size)) 292 | test_set_classifier.append(t_classifier.evaluate(my_trainer.model, test_iterator)) 293 | 294 | testErrorGScaled = t_classifier.evaluate(my_trainer.model, test_iterator, 295 | my_trainer.gradient_threshold_unreported_experiment, False, 296 | my_trainer.older_classes, args.step_size) 297 | logger.info("Test Classifier Final GScaled: %0.2f", testErrorGScaled) 298 | 299 | classifier_scaled_grad.append(testErrorGScaled) 300 | 301 | # Compute the the nmc based classification results 302 | tempTrain = t_classifier.evaluate(my_trainer.model, train_iterator) 303 | train_set_classifier.append(tempTrain) 304 | 305 | nmc_ideal.update_means(my_trainer.model, train_iterator_nmc, dataset.classes) 306 | testY_ideal = nmc_ideal.evaluate(my_trainer.model, test_iterator) 307 | 308 | nmc_ideal_cum.append(testY_ideal) 309 | 310 | # Compute confusion matrices of all three cases (Learned classifier, iCaRL, and ideal NMC) 311 | tcMatrix = t_classifier.get_confusion_matrix(my_trainer.model, test_iterator, dataset.classes) 312 | tcMatrix_scaled = t_classifier.get_confusion_matrix(my_trainer.model, test_iterator, dataset.classes, 313 | my_trainer.dynamic_threshold, 314 | my_trainer.older_classes, 315 | args.step_size) 316 | nmcMatrixIdeal = nmc_ideal.get_confusion_matrix(my_trainer.model, test_iterator, dataset.classes) 317 | 318 | print("Train Claissifier", tempTrain) 319 | 320 | # Store the resutls in the my_experiment object; this object should contain all the 321 | # information required to reproduce the results. 322 | x.append(counter) 323 | 324 | my_experiment.results["NMC"] = [x, y] 325 | my_experiment.results["Trained Classifier"] = [x, test_set_classifier] 326 | my_experiment.results["Trained Classifier Scaled"] = [x, classifier_scaled] 327 | my_experiment.results["Train Error Classifier"] = [x, train_set_classifier] 328 | my_experiment.results["Ideal NMC"] = [x, nmc_ideal_cum] 329 | my_experiment.store_json() 330 | 331 | # Finally, plotting the results; 332 | my_plotter = plt.Plotter() 333 | 334 | # Plotting the confusion matrices 335 | my_plotter.plotMatrix(int(class_group / args.step_size) * args.epochs_class + epoch, 336 | my_experiment.path + "tcMatrix" + str(xTemp), tcMatrix) 337 | my_plotter.plotMatrix(int(class_group / args.step_size) * args.epochs_class + epoch, 338 | my_experiment.path + "tcMatrix_scaled" + str(xTemp), tcMatrix_scaled) 339 | my_plotter.plotMatrix(int(class_group / args.step_size) * args.epochs_class + epoch, 340 | my_experiment.path + "nmcMatrixIdeal" + str(xTemp), 341 | nmcMatrixIdeal) 342 | 343 | # Plotting the line diagrams of all the possible cases 344 | 345 | my_plotter.plot(x, classifier_scaled, title=args.name, legend="Trained Classifier Scaled") 346 | my_plotter.plot(x, nmc_ideal_cum, title=args.name, legend="Ideal NMC") 347 | my_plotter.plot(x, test_set_classifier, title=args.name, legend="Trained Classifier") 348 | 349 | # Saving the line plot 350 | my_plotter.save_fig(my_experiment.path, dataset.classes + 1) 351 | -------------------------------------------------------------------------------- /model/__init__.py: -------------------------------------------------------------------------------- 1 | from model.model_factory import * -------------------------------------------------------------------------------- /model/misc_functions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Thu Oct 21 11:09:09 2017 3 | @author: Utku Ozbulak - github.com/utkuozbulak 4 | """ 5 | import copy 6 | 7 | import cv2 8 | import numpy as np 9 | import torch 10 | from torch.autograd import Variable 11 | from torchvision import models 12 | 13 | 14 | def preprocess_image(cv2im, resize_im=True): 15 | """ 16 | Processes image for CNNs 17 | Args: 18 | PIL_img (PIL_img): Image to process 19 | resize_im (bool): Resize to 224 or not 20 | returns: 21 | im_as_var (Pytorch variable): Variable that contains processed float tensor 22 | """ 23 | # mean and std list for channels (Imagenet) 24 | mean = [0.485, 0.456, 0.406] 25 | std = [0.229, 0.224, 0.225] 26 | # Resize image 27 | if resize_im: 28 | cv2im = cv2.resize(cv2im, (224, 224)) 29 | im_as_arr = np.float32(cv2im) 30 | im_as_arr = np.ascontiguousarray(im_as_arr[..., ::-1]) 31 | im_as_arr = im_as_arr.transpose(2, 0, 1) # Convert array to D,W,H 32 | # Normalize the channels 33 | for channel, _ in enumerate(im_as_arr): 34 | im_as_arr[channel] /= 255 35 | im_as_arr[channel] -= mean[channel] 36 | im_as_arr[channel] /= std[channel] 37 | # Convert to float tensor 38 | im_as_ten = torch.from_numpy(im_as_arr).float() 39 | # Add one more channel to the beginning. Tensor shape = 1,3,224,224 40 | im_as_ten.unsqueeze_(0) 41 | # Convert to Pytorch variable 42 | im_as_var = Variable(im_as_ten, requires_grad=True) 43 | return im_as_var 44 | 45 | 46 | def recreate_image(im_as_var): 47 | """ 48 | Recreates images from a torch variable, sort of reverse preprocessing 49 | Args: 50 | im_as_var (torch variable): Image to recreate 51 | returns: 52 | recreated_im (numpy arr): Recreated image in array 53 | """ 54 | reverse_mean = [-0.485, -0.456, -0.406] 55 | reverse_std = [1 / 0.229, 1 / 0.224, 1 / 0.225] 56 | recreated_im = copy.copy(im_as_var.data.numpy()[0]) 57 | for c in range(3): 58 | recreated_im[c] /= reverse_std[c] 59 | recreated_im[c] -= reverse_mean[c] 60 | recreated_im[recreated_im > 1] = 1 61 | recreated_im[recreated_im < 0] = 0 62 | recreated_im = np.round(recreated_im * 255) 63 | 64 | recreated_im = np.uint8(recreated_im).transpose(1, 2, 0) 65 | # Convert RBG to GBR 66 | recreated_im = recreated_im[..., ::-1] 67 | return recreated_im 68 | 69 | 70 | def get_params(example_index): 71 | """ 72 | Gets used variables for almost all visualizations, like the image, model etc. 73 | Args: 74 | example_index (int): Image id to use from examples 75 | returns: 76 | original_image (numpy arr): Original image read from the file 77 | prep_img (numpy_arr): Processed image 78 | target_class (int): Target class for the image 79 | file_name_to_export (string): File name to export the visualizations 80 | pretrained_model(Pytorch model): Model to use for the operations 81 | """ 82 | # Pick one of the examples 83 | example_list = [['../input_images/apple.JPEG', 948], 84 | ['../input_images/eel.JPEG', 390], 85 | ['../input_images/bird.JPEG', 13]] 86 | selected_example = example_index 87 | img_path = example_list[selected_example][0] 88 | target_class = example_list[selected_example][1] 89 | file_name_to_export = img_path[img_path.rfind('/') + 1:img_path.rfind('.')] 90 | # Read image 91 | original_image = cv2.imread(img_path, 1) 92 | # Process image 93 | prep_img = preprocess_image(original_image) 94 | # Define model 95 | pretrained_model = models.alexnet(pretrained=True) 96 | return (original_image, 97 | prep_img, 98 | target_class, 99 | file_name_to_export, 100 | pretrained_model) 101 | -------------------------------------------------------------------------------- /model/model_factory.py: -------------------------------------------------------------------------------- 1 | ''' Incremental-Classifier Learning 2 | Authors : Khurram Javed, Muhammad Talha Paracha 3 | Maintainer : Khurram Javed 4 | Lab : TUKL-SEECS R&D Lab 5 | Email : 14besekjaved@seecs.edu.pk ''' 6 | 7 | import model.resnet32 as res 8 | import model.test_model as tm 9 | 10 | 11 | class ModelFactory(): 12 | def __init__(self): 13 | pass 14 | 15 | @staticmethod 16 | def get_model(model_type, dataset="CIFAR100"): 17 | 18 | if model_type == "resnet32": 19 | if dataset == "MNIST": 20 | return res.resnet32mnist(10) 21 | elif dataset == "CIFAR10": 22 | return res.resnet32(10) 23 | return res.resnet32(100) 24 | 25 | 26 | elif model_type == "resnet20": 27 | if dataset == "MNIST": 28 | return res.resnet20mnist(10) 29 | elif dataset == "CIFAR10": 30 | return res.resnet20(10) 31 | return res.resnet20(100) 32 | 33 | elif model_type == "resnet10": 34 | if dataset == "MNIST": 35 | return res.resnet10mnist(10) 36 | elif dataset == "CIFAR10": 37 | return res.resnet20(10) 38 | return res.resnet20(100) 39 | 40 | 41 | elif model_type == "resnet44": 42 | if dataset == "MNIST": 43 | print("MNIST Dataset not supported in this model. Try resnet20 or 32") 44 | assert (False) 45 | elif dataset == "CIFAR10": 46 | return res.resnet44(10) 47 | return res.resnet44(100) 48 | 49 | 50 | elif model_type == "test": 51 | if dataset == "MNIST": 52 | return tm.Net(10, 1) 53 | elif dataset == "CIFAR10": 54 | return tm.Net(10) 55 | return tm.Net(100) 56 | else: 57 | print("Unsupported model; either implement the model in model/ModelFactory or choose a different model") 58 | assert (False) 59 | -------------------------------------------------------------------------------- /model/res_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | 5 | class DownsampleA(nn.Module): 6 | def __init__(self, nIn, nOut, stride): 7 | super(DownsampleA, self).__init__() 8 | assert stride == 2 9 | self.avg = nn.AvgPool2d(kernel_size=1, stride=stride) 10 | 11 | def forward(self, x): 12 | x = self.avg(x) 13 | return torch.cat((x, x.mul(0)), 1) 14 | 15 | 16 | class DownsampleC(nn.Module): 17 | def __init__(self, nIn, nOut, stride): 18 | super(DownsampleC, self).__init__() 19 | assert stride != 1 or nIn != nOut 20 | self.conv = nn.Conv2d(nIn, nOut, kernel_size=1, stride=stride, padding=0, bias=False) 21 | 22 | def forward(self, x): 23 | x = self.conv(x) 24 | return x 25 | 26 | 27 | class DownsampleD(nn.Module): 28 | def __init__(self, nIn, nOut, stride): 29 | super(DownsampleD, self).__init__() 30 | assert stride == 2 31 | self.conv = nn.Conv2d(nIn, nOut, kernel_size=2, stride=stride, padding=0, bias=False) 32 | self.bn = nn.BatchNorm2d(nOut) 33 | 34 | def forward(self, x): 35 | x = self.conv(x) 36 | x = self.bn(x) 37 | return x 38 | -------------------------------------------------------------------------------- /model/resnet32.py: -------------------------------------------------------------------------------- 1 | ''' Incremental-Classifier Learning 2 | Authors : Khurram Javed, Muhammad Talha Paracha 3 | Maintainer : Khurram Javed 4 | Lab : TUKL-SEECS R&D Lab 5 | Email : 14besekjaved@seecs.edu.pk ''' 6 | 7 | import math 8 | 9 | import torch 10 | import torch.nn as nn 11 | import torch.nn.functional as F 12 | from torch.nn import init 13 | 14 | from .res_utils import DownsampleA 15 | 16 | 17 | class ResNetBasicblock(nn.Module): 18 | expansion = 1 19 | """ 20 | RexNet basicblock (https://github.com/facebook/fb.resnet.torch/blob/master/models/resnet.lua) 21 | """ 22 | 23 | def __init__(self, inplanes, planes, stride=1, downsample=None): 24 | super(ResNetBasicblock, self).__init__() 25 | 26 | self.conv_a = nn.Conv2d(inplanes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 27 | self.bn_a = nn.BatchNorm2d(planes) 28 | 29 | self.conv_b = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) 30 | self.bn_b = nn.BatchNorm2d(planes) 31 | 32 | self.downsample = downsample 33 | self.featureSize = 64 34 | 35 | def forward(self, x): 36 | residual = x 37 | 38 | basicblock = self.conv_a(x) 39 | basicblock = self.bn_a(basicblock) 40 | basicblock = F.relu(basicblock, inplace=True) 41 | 42 | basicblock = self.conv_b(basicblock) 43 | basicblock = self.bn_b(basicblock) 44 | 45 | if self.downsample is not None: 46 | residual = self.downsample(x) 47 | 48 | return F.relu(residual + basicblock, inplace=True) 49 | 50 | 51 | class CifarResNet(nn.Module): 52 | """ 53 | ResNet optimized for the Cifar Dataset, as specified in 54 | https://arxiv.org/abs/1512.03385.pdf 55 | """ 56 | 57 | def __init__(self, block, depth, num_classes, channels=3): 58 | """ Constructor 59 | Args: 60 | depth: number of layers. 61 | num_classes: number of classes 62 | base_width: base width 63 | """ 64 | super(CifarResNet, self).__init__() 65 | 66 | self.featureSize = 64 67 | # Model type specifies number of layers for CIFAR-10 and CIFAR-100 model 68 | assert (depth - 2) % 6 == 0, 'depth should be one of 20, 32, 44, 56, 110' 69 | layer_blocks = (depth - 2) // 6 70 | 71 | self.num_classes = num_classes 72 | 73 | self.conv_1_3x3 = nn.Conv2d(channels, 16, kernel_size=3, stride=1, padding=1, bias=False) 74 | self.bn_1 = nn.BatchNorm2d(16) 75 | 76 | self.inplanes = 16 77 | self.stage_1 = self._make_layer(block, 16, layer_blocks, 1) 78 | self.stage_2 = self._make_layer(block, 32, layer_blocks, 2) 79 | self.stage_3 = self._make_layer(block, 64, layer_blocks, 2) 80 | self.avgpool = nn.AvgPool2d(8) 81 | self.fc = nn.Linear(64 * block.expansion, num_classes) 82 | self.fc2 = nn.Linear(64 * block.expansion, num_classes) 83 | 84 | for m in self.modules(): 85 | if isinstance(m, nn.Conv2d): 86 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 87 | m.weight.data.normal_(0, math.sqrt(2. / n)) 88 | # m.bias.data.zero_() 89 | elif isinstance(m, nn.BatchNorm2d): 90 | m.weight.data.fill_(1) 91 | m.bias.data.zero_() 92 | elif isinstance(m, nn.Linear): 93 | init.kaiming_normal(m.weight) 94 | m.bias.data.zero_() 95 | 96 | def _make_layer(self, block, planes, blocks, stride=1): 97 | downsample = None 98 | if stride != 1 or self.inplanes != planes * block.expansion: 99 | downsample = DownsampleA(self.inplanes, planes * block.expansion, stride) 100 | 101 | layers = [] 102 | layers.append(block(self.inplanes, planes, stride, downsample)) 103 | self.inplanes = planes * block.expansion 104 | for i in range(1, blocks): 105 | layers.append(block(self.inplanes, planes)) 106 | 107 | return nn.Sequential(*layers) 108 | 109 | def forward(self, x, feature=False, T=1, labels=False, scale=None, keep=None): 110 | 111 | x = self.conv_1_3x3(x) 112 | x = F.relu(self.bn_1(x), inplace=True) 113 | x = self.stage_1(x) 114 | x = self.stage_2(x) 115 | x = self.stage_3(x) 116 | x = self.avgpool(x) 117 | x = x.view(x.size(0), -1) 118 | if feature: 119 | return x / torch.norm(x, 2, 1).unsqueeze(1) 120 | x = self.fc(x) / T 121 | if keep is not None: 122 | x = x[:, keep[0]:keep[1]] 123 | if labels: 124 | return F.softmax(x, dim=1) 125 | 126 | if scale is not None: 127 | temp = F.softmax(x, dim=1) 128 | temp = temp * scale 129 | return temp 130 | 131 | return F.log_softmax(x, dim=1) 132 | 133 | def forwardFeature(self, x): 134 | pass 135 | 136 | 137 | def resnet20(num_classes=10): 138 | """Constructs a ResNet-20 model for CIFAR-10 (by default) 139 | Args: 140 | num_classes (uint): number of classes 141 | """ 142 | model = CifarResNet(ResNetBasicblock, 20, num_classes) 143 | return model 144 | 145 | 146 | def resnet10mnist(num_classes=10): 147 | """Constructs a ResNet-20 model for CIFAR-10 (by default) 148 | Args: 149 | num_classes (uint): number of classes 150 | """ 151 | model = CifarResNet(ResNetBasicblock, 10, num_classes, 1) 152 | return model 153 | 154 | 155 | def resnet20mnist(num_classes=10): 156 | """Constructs a ResNet-20 model for CIFAR-10 (by default) 157 | Args: 158 | num_classes (uint): number of classes 159 | """ 160 | model = CifarResNet(ResNetBasicblock, 20, num_classes, 1) 161 | return model 162 | 163 | 164 | def resnet32mnist(num_classes=10, channels=1): 165 | model = CifarResNet(ResNetBasicblock, 32, num_classes, channels) 166 | return model 167 | 168 | 169 | def resnet32(num_classes=10): 170 | """Constructs a ResNet-32 model for CIFAR-10 (by default) 171 | Args: 172 | num_classes (uint): number of classes 173 | """ 174 | model = CifarResNet(ResNetBasicblock, 32, num_classes) 175 | return model 176 | 177 | 178 | def resnet44(num_classes=10): 179 | """Constructs a ResNet-44 model for CIFAR-10 (by default) 180 | Args: 181 | num_classes (uint): number of classes 182 | """ 183 | model = CifarResNet(ResNetBasicblock, 44, num_classes) 184 | return model 185 | 186 | 187 | def resnet56(num_classes=10): 188 | """Constructs a ResNet-56 model for CIFAR-10 (by default) 189 | Args: 190 | num_classes (uint): number of classes 191 | """ 192 | model = CifarResNet(ResNetBasicblock, 56, num_classes) 193 | return model 194 | 195 | 196 | def resnet110(num_classes=10): 197 | """Constructs a ResNet-110 model for CIFAR-10 (by default) 198 | Args: 199 | num_classes (uint): number of classes 200 | """ 201 | model = CifarResNet(ResNetBasicblock, 110, num_classes) 202 | return model 203 | -------------------------------------------------------------------------------- /model/test_model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | 6 | class Net(nn.Module): 7 | def __init__(self, noClasses, channels=3): 8 | super(Net, self).__init__() 9 | self.conv1 = nn.Conv2d(channels, 4, kernel_size=5, padding=(2, 2)) 10 | self.conv2 = nn.Conv2d(4, 6, kernel_size=5, padding=(2, 2)) 11 | self.conv2_bn1 = nn.BatchNorm2d(6) 12 | self.conv3 = nn.Conv2d(6, 8, kernel_size=5, padding=(2, 2)) 13 | self.conv2_bn2 = nn.BatchNorm2d(8) 14 | self.conv4 = nn.Conv2d(8, 10, kernel_size=5, padding=(2, 2)) 15 | self.conv2_bn3 = nn.BatchNorm2d(10) 16 | self.conv5 = nn.Conv2d(10, 12, kernel_size=5, padding=(2, 2)) 17 | self.conv5_bn3 = nn.BatchNorm2d(12) 18 | self.conv5_drop = nn.Dropout2d() 19 | self.fc1 = nn.Linear(48, 100) 20 | self.fc = nn.Linear(100, noClasses) 21 | self.featureSize = 48 22 | 23 | def forward(self, x, feature=False, T=1, labels=False, scale=None, predictClass=False): 24 | 25 | x = F.relu(F.max_pool2d(self.conv1(x), 2)) 26 | x = self.conv2_bn1(self.conv2(x)) 27 | x = F.relu(F.max_pool2d(self.conv2_bn2(self.conv3(x)), 2)) 28 | x = F.relu(F.max_pool2d(self.conv2_bn3(self.conv4(x)), 2)) 29 | x = F.relu(F.max_pool2d(self.conv5_drop(self.conv5_bn3(self.conv5(x))), 2)) 30 | x = x.view(x.size(0), -1) 31 | if feature: 32 | return x / torch.norm(x, 2, 1).unsqueeze(1) 33 | x = F.relu(self.fc1(x)) 34 | x = F.dropout(x, training=self.training) 35 | 36 | if labels: 37 | if predictClass: 38 | return F.softmax(self.fc(x) / T), F.softmax(self.fc2(x) / T) 39 | return F.softmax(self.fc(x) / T) 40 | 41 | if scale is not None: 42 | x = self.fc(x) 43 | temp = F.softmax(x / T) 44 | temp = temp * scale 45 | return temp 46 | 47 | if predictClass: 48 | return F.log_softmax(self.fc(x) / T), F.log_softmax(self.fc2(x) / T) 49 | return F.log_softmax(self.fc(x) / T) 50 | -------------------------------------------------------------------------------- /plots/herding/noherding/NOHERDINGJSONDump2.txt: -------------------------------------------------------------------------------- 1 | "{\"gitHash\": \"bc2372b91db23992eaa2f4b6079081c3d8857569\\n\", \"name\": \"752018/NOHERDING\", \"params\": {\"batch_size\": 64, \"lr\": 2.0, \"schedule\": [45, 60, 68], \"gammas\": [0.2, 0.2, 0.2], \"momentum\": 0.9, \"no_cuda\": false, \"random_init\": false, \"no_distill\": false, \"distill_only_exemplars\": false, \"no_random\": false, \"no_herding\": true, \"seeds\": [400, 500, 600, 700], \"log_interval\": 5, \"model_type\": \"resnet32\", \"name\": \"NOHERDING\", \"outputDir\": \"../\", \"upsampling\": false, \"pp\": false, \"distill_step\": false, \"hs\": false, \"unstructured_size\": 0, \"alphas\": [1.0], \"decay\": 5e-05, \"alpha_increment\": 1.0, \"l1\": 0.0, \"step_size\": 10, \"T\": 1, \"memory_budgets\": [2000], \"epochs_class\": 70, \"dataset\": \"CIFAR100\", \"lwf\": false, \"no_nl\": false, \"rand\": false, \"adversarial\": false, \"cuda\": true, \"alpha\": 1.0, \"memory_budget\": 2000, \"seed\": 400}, \"results\": {\"Temp Results\": [[1, 2, 3, 4], [5, 6, 2, 6]], \"NMC\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [86.2, 78.35, 72.66666666666666, 68.9, 64.7, 63.366666666666666, 55.357142857142854, 53.7125, 51.31111111111111, 44.83]], \"Trained Classifier\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [89.1, 72.3, 71.46666666666667, 66.675, 61.58, 58.36666666666667, 54.957142857142856, 51.125, 48.17777777777778, 43.21]], \"Trained Classifier Scaled\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [89.1, 78.65, 74.06666666666666, 66.725, 61.9, 57.233333333333334, 53.34285714285714, 50.9625, 48.233333333333334, 44.49]], \"Trained Classifier Grad Scaled\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [88.6, 77.55, 73.73333333333333, 67.175, 62.5, 58.1, 54.542857142857144, 51.7125, 49.15555555555556, 45.27]], \"Train Error Classifier\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [99.34, 79.8, 88.17142857142858, 89.5702005730659, 91.02857142857142, 90.1, 90.08595988538681, 90.28735632183908, 90.42857142857143, 88.39541547277938]], \"Ideal NMC\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [89.3, 80.65, 74.33333333333333, 68.25, 63.16, 58.75, 55.128571428571426, 52.3, 49.77777777777778, 45.84]]}, \"dir\": \"../\", \"path\": \"../752018/NOHERDING_0/NOHERDING\"}" -------------------------------------------------------------------------------- /plots/lwf/1: -------------------------------------------------------------------------------- 1 | "{\"name\": \"iCaRLCIFARlwfmc\", \"params\": {\"batch_size\": 64, \"test_batch_size\": 128, \"epochs\": 200, \"lr\": 2.0, \"schedule\": [42, 57, 68], \"gammas\": [0.2, 0.2, 0.2], \"momentum\": 0.9, \"no_cuda\": false, \"no_distill\": false, \"distill_only_exemplars\": false, \"decayed\": false, \"no_random\": false, \"no_herding\": false, \"seeds\": [25, 67, 85, 3], \"log_interval\": 2, \"model_type\": \"resnet44\", \"name\": \"iCaRLCIFARlwfmc\", \"sortby\": \"none\", \"outputDir\": \"../\", \"no_upsampling\": false, \"decay\": 5e-05, \"distill_decay\": 0.5, \"step_size\": 10, \"memory_budgets\": [2000], \"epochs_class\": 70, \"dataset\": \"CIFAR100\", \"lwf\": true, \"cuda\": true, \"memory_budget\": 0, \"seed\": 85}, \"results\": {\"Temp Results\": [[1, 2, 3, 4], [5, 6, 2, 6]], \"NCM\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [86.7, 44.4, 28.8, 23.125, 17.52, 13.933333333333334, 12.614285714285714, 11.8625, 9.977777777777778, 8.88]], \"Trained Classifier\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [86.5, 67.1, 55.833333333333336, 48.425, 40.1, 30.283333333333335, 27.37142857142857, 27.625, 25.533333333333335, 22.2]], \"Train Error Classifier\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [99.34, 99.44, 99.62, 99.56, 99.48, 97.88, 99.08, 99.56, 98.9, 98.28]]}, \"dir\": \"../\", \"path\": \"../iCaRLCIFARlwfmc_5/iCaRLCIFARlwfmc\"}" -------------------------------------------------------------------------------- /plots/lwf/2: -------------------------------------------------------------------------------- 1 | "{\"name\": \"iCaRLCIFARlwfmc\", \"params\": {\"batch_size\": 64, \"test_batch_size\": 128, \"epochs\": 200, \"lr\": 2.0, \"schedule\": [42, 57, 68], \"gammas\": [0.2, 0.2, 0.2], \"momentum\": 0.9, \"no_cuda\": false, \"no_distill\": false, \"distill_only_exemplars\": false, \"decayed\": false, \"no_random\": false, \"no_herding\": false, \"seeds\": [25, 67, 85, 3], \"log_interval\": 2, \"model_type\": \"resnet44\", \"name\": \"iCaRLCIFARlwfmc\", \"sortby\": \"none\", \"outputDir\": \"../\", \"no_upsampling\": false, \"decay\": 5e-05, \"distill_decay\": 0.5, \"step_size\": 10, \"memory_budgets\": [2000], \"epochs_class\": 70, \"dataset\": \"CIFAR100\", \"lwf\": true, \"cuda\": true, \"memory_budget\": 0, \"seed\": 67}, \"results\": {\"Temp Results\": [[1, 2, 3, 4], [5, 6, 2, 6]], \"NCM\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [88.5, 43.05, 30.366666666666667, 22.375, 17.16, 14.516666666666667, 12.085714285714285, 11.5625, 9.966666666666667, 8.64]], \"Trained Classifier\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [88.3, 65.75, 56.666666666666664, 50.575, 38.74, 33.416666666666664, 28.414285714285715, 25.1125, 22.68888888888889, 20.95]], \"Train Error Classifier\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [99.56, 99.46, 99.48, 99.62, 99.2, 98.96, 98.74, 99.24, 97.94, 98.18]]}, \"dir\": \"../\", \"path\": \"../iCaRLCIFARlwfmc_4/iCaRLCIFARlwfmc\"}" -------------------------------------------------------------------------------- /plots/lwf/3: -------------------------------------------------------------------------------- 1 | "{\"name\": \"iCaRLCIFARlwfmc\", \"params\": {\"batch_size\": 64, \"test_batch_size\": 128, \"epochs\": 200, \"lr\": 2.0, \"schedule\": [42, 57, 68], \"gammas\": [0.2, 0.2, 0.2], \"momentum\": 0.9, \"no_cuda\": false, \"no_distill\": false, \"distill_only_exemplars\": false, \"decayed\": false, \"no_random\": false, \"no_herding\": false, \"seeds\": [25, 67, 85, 3], \"log_interval\": 2, \"model_type\": \"resnet44\", \"name\": \"iCaRLCIFARlwfmc\", \"sortby\": \"none\", \"outputDir\": \"../\", \"no_upsampling\": false, \"decay\": 5e-05, \"distill_decay\": 0.5, \"step_size\": 10, \"memory_budgets\": [2000], \"epochs_class\": 70, \"dataset\": \"CIFAR100\", \"lwf\": true, \"cuda\": true, \"memory_budget\": 0, \"seed\": 25}, \"results\": {\"Temp Results\": [[1, 2, 3, 4], [5, 6, 2, 6]], \"NCM\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [86.1, 40.0, 30.7, 22.35, 18.1, 14.466666666666667, 12.942857142857143, 10.95, 9.933333333333334, 9.01]], \"Trained Classifier\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [86.0, 66.2, 57.03333333333333, 50.325, 42.02, 34.166666666666664, 28.285714285714285, 22.6125, 23.233333333333334, 20.96]], \"Train Error Classifier\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [98.78, 99.18, 99.76, 99.58, 99.6, 99.34, 98.86, 98.52, 98.86, 98.58]]}, \"dir\": \"../\", \"path\": \"../iCaRLCIFARlwfmc_3/iCaRLCIFARlwfmc\"}" -------------------------------------------------------------------------------- /plots/lwf/4: -------------------------------------------------------------------------------- 1 | "{\"name\": \"iCaRLCIFARlwfmc\", \"params\": {\"batch_size\": 64, \"test_batch_size\": 128, \"epochs\": 200, \"lr\": 2.0, \"schedule\": [42, 57, 68], \"gammas\": [0.2, 0.2, 0.2], \"momentum\": 0.9, \"no_cuda\": false, \"no_distill\": false, \"distill_only_exemplars\": false, \"decayed\": false, \"no_random\": false, \"no_herding\": false, \"seeds\": [25, 67, 85, 3], \"log_interval\": 2, \"model_type\": \"resnet44\", \"name\": \"iCaRLCIFARlwfmc\", \"sortby\": \"none\", \"outputDir\": \"../\", \"no_upsampling\": false, \"decay\": 5e-05, \"distill_decay\": 0.5, \"step_size\": 10, \"memory_budgets\": [2000], \"epochs_class\": 70, \"dataset\": \"CIFAR100\", \"lwf\": true, \"cuda\": true, \"memory_budget\": 0, \"seed\": 25}, \"results\": {\"Temp Results\": [[1, 2, 3, 4], [5, 6, 2, 6]], \"NCM\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [86.1, 40.0, 30.7, 22.35, 18.1, 14.466666666666667, 12.942857142857143, 10.95, 9.933333333333334, 9.01]], \"Trained Classifier\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [86.0, 66.2, 57.03333333333333, 50.325, 42.02, 34.166666666666664, 28.285714285714285, 22.6125, 23.233333333333334, 20.96]], \"Train Error Classifier\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [98.78, 99.18, 99.76, 99.58, 99.6, 99.34, 98.86, 98.52, 98.86, 98.58]]}, \"dir\": \"../\", \"path\": \"../iCaRLCIFARlwfmc_3/iCaRLCIFARlwfmc\"}" -------------------------------------------------------------------------------- /plots/lwf/5: -------------------------------------------------------------------------------- 1 | "{\"name\": \"iCaRLCIFARlwfmc\", \"params\": {\"batch_size\": 64, \"test_batch_size\": 128, \"epochs\": 200, \"lr\": 2.0, \"schedule\": [42, 57, 68], \"gammas\": [0.2, 0.2, 0.2], \"momentum\": 0.9, \"no_cuda\": false, \"no_distill\": false, \"distill_only_exemplars\": false, \"decayed\": false, \"no_random\": false, \"no_herding\": false, \"seeds\": [24], \"log_interval\": 2, \"model_type\": \"resnet32\", \"name\": \"iCaRLCIFARlwfmc\", \"sortby\": \"none\", \"outputDir\": \"../\", \"no_upsampling\": false, \"decay\": 5e-05, \"distill_decay\": 0.5, \"step_size\": 10, \"memory_budgets\": [2000], \"epochs_class\": 70, \"dataset\": \"CIFAR100\", \"lwf\": true, \"cuda\": true, \"memory_budget\": 0, \"seed\": 24}, \"results\": {\"Temp Results\": [[1, 2, 3, 4], [5, 6, 2, 6]], \"NCM\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [84.9, 43.95, 30.933333333333334, 22.675, 18.78, 14.816666666666666, 13.114285714285714, 10.2875, 10.066666666666666, 8.44]], \"Trained Classifier\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [85.2, 67.0, 57.2, 46.0, 39.74, 32.81666666666667, 31.82857142857143, 24.625, 23.41111111111111, 20.61]], \"Train Error Classifier\": [[10, 20, 30, 40, 50, 60, 70, 80, 90, 100], [99.18, 99.44, 99.68, 99.78, 99.64, 99.0, 99.2, 98.4, 98.94, 97.7]]}, \"dir\": \"../\", \"path\": \"../iCaRLCIFARlwfmc_2/iCaRLCIFARlwfmc\"}" -------------------------------------------------------------------------------- /plots/memorybudget/os/2000: -------------------------------------------------------------------------------- 1 | "{\"name\": \"MNIST_multi_run_memory_noumsample\", \"params\": {\"batch_size\": 32, \"test_batch_size\": 128, \"epochs\": 200, \"lr\": 0.001, \"schedule\": [5, 7], \"gammas\": [0.4, 0.4], \"momentum\": 0.9, \"no_cuda\": false, \"no_distill\": false, \"no_random\": false, \"no_herding\": false, \"seeds\": [40], \"log_interval\": 2, \"model_type\": \"resnet20\", \"name\": \"MNIST_multi_run_memory_noumsample\", \"sortby\": \"none\", \"outputDir\": \"../\", \"no_upsampling\": false, \"decay\": 1e-05, \"step_size\": 2, \"memory_budgets\": [2000, 4000], \"epochs_class\": 10, \"dataset\": \"MNIST\", \"lwf\": false, \"cuda\": true, \"memory_budget\": 2000, \"seed\": 40}, \"results\": {\"Temp Results\": [[1, 2, 3, 4], [5, 6, 2, 6]], \"NCM\": [[2, 4, 6, 8], [99.44623904014767, 98.53012048192771, 97.99612152553328, 97.5367006718089, 97.1537736894251]], \"Trained Classifier\": [[2, 4, 6, 8], [99.53853253345639, 98.6987951807229, 98.28700711053652, 97.84772331425728, 97.42763465236569]], \"Train Error Classifier\": [[2, 4, 6, 8], [98.8740157480315, 98.37959183673469, 98.21660884648742, 98.23157458277184,98.23577732584233]]}, \"dir\": \"../\", \"path\": \"../MNIST_multi_run_memory_noumsample_8/MNIST_multi_run_memory_noumsample\"}" -------------------------------------------------------------------------------- /plots/memorybudget/os/500: -------------------------------------------------------------------------------- 1 | "{\"name\": \"MNIST_multi_run_memory_noumsample\", \"params\": {\"batch_size\": 32, \"test_batch_size\": 128, \"epochs\": 200, \"lr\": 0.001, \"schedule\": [5, 7], \"gammas\": [0.4, 0.4], \"momentum\": 0.9, \"no_cuda\": false, \"no_distill\": false, \"no_random\": false, \"no_herding\": false, \"seeds\": [40], \"log_interval\": 2, \"model_type\": \"resnet20\", \"name\": \"MNIST_multi_run_memory_noumsample\", \"sortby\": \"none\", \"outputDir\": \"../\", \"no_upsampling\": false, \"decay\": 1e-05, \"step_size\": 2, \"memory_budgets\": [500, 1000], \"epochs_class\": 10, \"dataset\": \"MNIST\", \"lwf\": false, \"cuda\": true, \"memory_budget\": 500, \"seed\": 40}, \"results\": {\"Temp Results\": [[1, 2, 3, 4], [5, 6, 2, 6]], \"NCM\": [[2, 4, 6, 8], [99.58467928011075, 98.23686746987952, 97.33354880413704, 96.79489176412043, 96.34]], \"Trained Classifier\": [[2, 4, 6, 8], [99.54853253345639, 98.7987951807229, 97.85700711053652, 97.22772331425728, 97.02763465236569]], \"Train Error Classifier\": [[2, 4, 6, 8], [99.72311952007384, 98.48192771084338, 97.40122818358113, 96.84000995272456, 96.03]]}, \"dir\": \"../\", \"path\": \"../MNIST_multi_run_memory_noumsample_8/MNIST_multi_run_memory_noumsample\"}" -------------------------------------------------------------------------------- /plotter/__init__.py: -------------------------------------------------------------------------------- 1 | from plotter.plotter import * -------------------------------------------------------------------------------- /plotter/plotter.py: -------------------------------------------------------------------------------- 1 | ''' Incremental-Classifier Learning 2 | Authors : Khurram Javed, Muhammad Talha Paracha 3 | Maintainer : Khurram Javed 4 | Lab : TUKL-SEECS R&D Lab 5 | Email : 14besekjaved@seecs.edu.pk ''' 6 | 7 | import matplotlib 8 | import matplotlib.pyplot as plt 9 | 10 | plt.switch_backend('agg') 11 | 12 | MEDIUM_SIZE = 18 13 | 14 | font = {'family': 'sans-serif', 15 | 'weight': 'bold'} 16 | 17 | matplotlib.rc('xtick', labelsize=MEDIUM_SIZE) 18 | matplotlib.rc('ytick', labelsize=MEDIUM_SIZE) 19 | plt.rc('axes', labelsize=MEDIUM_SIZE) # fontsize of the x and y labels 20 | 21 | # matplotlib.rc('font', **font) 22 | from matplotlib import rcParams 23 | 24 | rcParams.update({'figure.autolayout': True}) 25 | 26 | 27 | class Plotter(): 28 | def __init__(self): 29 | import itertools 30 | # plt.figure(figsize=(12, 9)) 31 | self.marker = itertools.cycle(('o', '+', "v", "^", "8", '.', '*')) 32 | self.handles = [] 33 | self.lines = itertools.cycle(('--', '-.', '-', ':')) 34 | 35 | def plot(self, x, y, xLabel="Number of Classes", yLabel="Accuracy %", legend="none", title=None, error=None): 36 | self.x = x 37 | self.y = y 38 | plt.grid(color='0.89', linestyle='--', linewidth=1.0) 39 | if error is None: 40 | l, = plt.plot(x, y, linestyle=next(self.lines), marker=next(self.marker), label=legend, linewidth=3.0) 41 | else: 42 | l = plt.errorbar(x, y, yerr=error, capsize=4.0, capthick=2.0, linestyle=next(self.lines), 43 | marker=next(self.marker), label=legend, linewidth=3.0) 44 | 45 | self.handles.append(l) 46 | self.x_label = xLabel 47 | self.y_label = yLabel 48 | if title is not None: 49 | plt.title(title) 50 | 51 | def save_fig(self, path, xticks=105, title=None, yStart=0, xRange=0, yRange=10): 52 | if title is not None: 53 | plt.title(title) 54 | plt.legend(handles=self.handles) 55 | plt.ylim((yStart, 100 + 0.2)) 56 | plt.xlim((0, xticks + .2)) 57 | plt.ylabel(self.y_label) 58 | plt.xlabel(self.x_label) 59 | plt.yticks(list(range(yStart, 101, yRange))) 60 | print(list(range(yStart, 105, yRange))) 61 | plt.xticks(list(range(0, xticks + 1, xRange + int(xticks / 10)))) 62 | plt.savefig(path + ".eps", format='eps') 63 | plt.gcf().clear() 64 | 65 | def save_fig2(self, path, xticks=105): 66 | plt.legend(handles=self.handles) 67 | plt.xlabel("Memory Budget") 68 | plt.ylabel("Average Incremental Accuracy") 69 | plt.savefig(path + ".jpg") 70 | plt.gcf().clear() 71 | 72 | def plotMatrix(self, epoch, path, img): 73 | 74 | plt.imshow(img, cmap='plasma', interpolation='nearest') 75 | plt.colorbar() 76 | plt.savefig(path + str(epoch) + ".svg", format='svg') 77 | plt.gcf().clear() 78 | 79 | def saveImage(self, img, path, epoch): 80 | from PIL import Image 81 | im = Image.fromarray(img) 82 | im.save(path + str(epoch) + ".jpg") 83 | 84 | 85 | if __name__ == "__main__": 86 | pl = Plotter() 87 | pl.plot([1, 2, 3, 4], [2, 3, 6, 2]) 88 | pl.save_fig("test.jpg") 89 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib==2.1.2 2 | numpy==1.14.1 3 | numpydoc==0.7.0 4 | progressbar2==3.35.2 5 | torch==0.3.1.post2 6 | torchfile==0.1.0 7 | torchnet==0.0.1 8 | torchvision==0.2.0 9 | tqdm==4.19.5 10 | -e git+https://github.com/pytorch/tnt.git@master 11 | -------------------------------------------------------------------------------- /run_experiment.py: -------------------------------------------------------------------------------- 1 | ''' Incremental-Classifier Learning 2 | Authors : Khurram Javed, Muhammad Talha Paracha 3 | Maintainer : Khurram Javed 4 | Lab : TUKL-SEECS R&D Lab 5 | Email : 14besekjaved@seecs.edu.pk ''' 6 | 7 | from __future__ import print_function 8 | 9 | import argparse 10 | import logging 11 | 12 | import torch 13 | import torch.utils.data as td 14 | 15 | import data_handler 16 | import experiment as ex 17 | import model 18 | import plotter as plt 19 | import trainer 20 | 21 | logger = logging.getLogger('iCARL') 22 | 23 | parser = argparse.ArgumentParser(description='iCarl2.0') 24 | parser.add_argument('--batch-size', type=int, default=64, metavar='N', 25 | help='input batch size for training (default: 64)') 26 | parser.add_argument('--lr', type=float, default=2.0, metavar='LR', 27 | help='learning rate (default: 2.0). Note that lr is decayed by args.gamma parameter args.schedule ') 28 | parser.add_argument('--schedule', type=int, nargs='+', default=[45, 60, 68], 29 | help='Decrease learning rate at these epochs.') 30 | parser.add_argument('--gammas', type=float, nargs='+', default=[0.2, 0.2, 0.2], 31 | help='LR is multiplied by gamma on schedule, number of gammas should be equal to schedule') 32 | parser.add_argument('--momentum', type=float, default=0.9, metavar='M', 33 | help='SGD momentum (default: 0.9)') 34 | parser.add_argument('--no-cuda', action='store_true', default=False, 35 | help='disables CUDA training') 36 | parser.add_argument('--random-init', action='store_true', default=False, 37 | help='Initialize model for next increment using previous weights if false and random weights otherwise') 38 | parser.add_argument('--no-distill', action='store_true', default=False, 39 | help='disable distillation loss and only uses the cross entropy loss. See "Distilling Knowledge in Neural Networks" by Hinton et.al for details') 40 | parser.add_argument('--no-random', action='store_true', default=False, 41 | help='Disable random shuffling of classes') 42 | parser.add_argument('--no-herding', action='store_true', default=False, 43 | help='Disable herding algorithm and do random instance selection instead') 44 | parser.add_argument('--seeds', type=int, nargs='+', default=[23423], 45 | help='Seeds values to be used; seed introduces randomness by changing order of classes') 46 | parser.add_argument('--log-interval', type=int, default=5, metavar='N', 47 | help='how many batches to wait before logging training status') 48 | parser.add_argument('--model-type', default="resnet32", 49 | help='model type to be used. Example : resnet32, resnet20, test') 50 | parser.add_argument('--name', default="noname", 51 | help='Name of the experiment') 52 | parser.add_argument('--outputDir', default="../", 53 | help='Directory to store the results; a new folder "DDMMYYYY" will be created ' 54 | 'in the specified directory to save the results.') 55 | parser.add_argument('--upsampling', action='store_true', default=False, 56 | help='Do not do upsampling.') 57 | parser.add_argument('--unstructured-size', type=int, default=0, 58 | help='Leftover parameter of an unreported experiment; leave it at 0') 59 | parser.add_argument('--alphas', type=float, nargs='+', default=[1.0], 60 | help='Weight given to new classes vs old classes in the loss; high value of alpha will increase perfomance on new classes at the expense of older classes. Dynamic threshold moving makes the system more robust to changes in this parameter') 61 | parser.add_argument('--decay', type=float, default=0.00005, help='Weight decay (L2 penalty).') 62 | parser.add_argument('--step-size', type=int, default=10, help='How many classes to add in each increment') 63 | parser.add_argument('--T', type=float, default=1, help='Tempreture used for softening the targets') 64 | parser.add_argument('--memory-budgets', type=int, nargs='+', default=[2000], 65 | help='How many images can we store at max. 0 will result in fine-tuning') 66 | parser.add_argument('--epochs-class', type=int, default=70, help='Number of epochs for each increment') 67 | parser.add_argument('--dataset', default="CIFAR100", help='Dataset to be used; example CIFAR, MNIST') 68 | parser.add_argument('--lwf', action='store_true', default=False, 69 | help='Use learning without forgetting. Ignores memory-budget ' 70 | '("Learning with Forgetting," Zhizhong Li, Derek Hoiem)') 71 | parser.add_argument('--no-nl', action='store_true', default=False, 72 | help='No Normal Loss. Only uses the distillation loss to train the new model on old classes (Normal loss is used for new classes however') 73 | 74 | args = parser.parse_args() 75 | args.cuda = not args.no_cuda and torch.cuda.is_available() 76 | 77 | dataset = data_handler.DatasetFactory.get_dataset(args.dataset) 78 | 79 | # Checks to make sure parameters are sane 80 | if args.step_size < 2: 81 | print("Step size of 1 will result in no learning;") 82 | assert False 83 | 84 | # Run an experiment corresponding to every seed value 85 | for seed in args.seeds: 86 | # Run an experiment corresponding to every alpha value 87 | for at in args.alphas: 88 | args.alpha = at 89 | # Run an experiment corresponding to every memory budget 90 | for m in args.memory_budgets: 91 | args.memory_budget = m 92 | # In LwF, memory_budget is 0 (See the paper "Learning without Forgetting" for details). 93 | if args.lwf: 94 | args.memory_budget = 0 95 | 96 | # Fix the seed. 97 | args.seed = seed 98 | torch.manual_seed(seed) 99 | if args.cuda: 100 | torch.cuda.manual_seed(seed) 101 | 102 | # Loader used for training data 103 | train_dataset_loader = data_handler.IncrementalLoader(dataset.train_data.train_data, 104 | dataset.train_data.train_labels, 105 | dataset.labels_per_class_train, 106 | dataset.classes, [1,2], 107 | transform=dataset.train_transform, 108 | cuda=args.cuda, oversampling=not args.upsampling, 109 | ) 110 | # Special loader use to compute ideal NMC; i.e, NMC that using all the data points to compute the mean embedding 111 | train_dataset_loader_nmc = data_handler.IncrementalLoader(dataset.train_data.train_data, 112 | dataset.train_data.train_labels, 113 | dataset.labels_per_class_train, 114 | dataset.classes, [1,2], 115 | transform=dataset.train_transform, 116 | cuda=args.cuda, oversampling=not args.upsampling, 117 | ) 118 | # Loader for test data. 119 | test_dataset_loader = data_handler.IncrementalLoader(dataset.test_data.test_data, 120 | dataset.test_data.test_labels, 121 | dataset.labels_per_class_test, dataset.classes, 122 | [1,2], transform=dataset.test_transform, cuda=args.cuda, 123 | ) 124 | 125 | kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {} 126 | 127 | # Iterator to iterate over training data. 128 | train_iterator = torch.utils.data.DataLoader(train_dataset_loader, 129 | batch_size=args.batch_size, shuffle=True, **kwargs) 130 | # Iterator to iterate over all training data (Equivalent to memory-budget = infitie 131 | train_iterator_nmc = torch.utils.data.DataLoader(train_dataset_loader_nmc, 132 | batch_size=args.batch_size, shuffle=True, **kwargs) 133 | # Iterator to iterate over test data 134 | test_iterator = torch.utils.data.DataLoader( 135 | test_dataset_loader, 136 | batch_size=args.batch_size, shuffle=True, **kwargs) 137 | 138 | # Get the required model 139 | myModel = model.ModelFactory.get_model(args.model_type, args.dataset) 140 | if args.cuda: 141 | myModel.cuda() 142 | 143 | # Define an experiment. 144 | my_experiment = ex.experiment(args.name, args) 145 | 146 | # Adding support for logging. A .log is generated with all the logs. Logs are also stored in a temp file one directory 147 | # before the code repository 148 | logger = logging.getLogger('iCARL') 149 | logger.setLevel(logging.DEBUG) 150 | 151 | fh = logging.FileHandler(my_experiment.path + ".log") 152 | fh.setLevel(logging.DEBUG) 153 | 154 | fh2 = logging.FileHandler("../temp.log") 155 | fh2.setLevel(logging.DEBUG) 156 | 157 | ch = logging.StreamHandler() 158 | ch.setLevel(logging.DEBUG) 159 | 160 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 161 | fh.setFormatter(formatter) 162 | fh2.setFormatter(formatter) 163 | 164 | logger.addHandler(fh) 165 | logger.addHandler(fh2) 166 | logger.addHandler(ch) 167 | 168 | # Define the optimizer used in the experiment 169 | optimizer = torch.optim.SGD(myModel.parameters(), args.lr, momentum=args.momentum, 170 | weight_decay=args.decay, nesterov=True) 171 | 172 | # Trainer object used for training 173 | my_trainer = trainer.Trainer(train_iterator, test_iterator, dataset, myModel, args, optimizer, 174 | train_iterator_nmc) 175 | 176 | # Parameters for storing the results 177 | x = [] 178 | y = [] 179 | y1 = [] 180 | train_y = [] 181 | higher_y = [] 182 | y_scaled = [] 183 | y_grad_scaled = [] 184 | nmc_ideal_cum = [] 185 | 186 | # Initilize the evaluators used to measure the performance of the system. 187 | nmc = trainer.EvaluatorFactory.get_evaluator("nmc", args.cuda) 188 | nmc_ideal = trainer.EvaluatorFactory.get_evaluator("nmc", args.cuda) 189 | t_classifier = trainer.EvaluatorFactory.get_evaluator("trainedClassifier", args.cuda) 190 | 191 | # Loop that incrementally adds more and more classes 192 | for class_group in range(0, dataset.classes, args.step_size): 193 | print("SEED:", seed, "MEMORY_BUDGET:", m, "CLASS_GROUP:", class_group) 194 | # Add new classes to the train, train_nmc, and test iterator 195 | my_trainer.increment_classes(class_group) 196 | my_trainer.update_frozen_model() 197 | epoch = 0 198 | 199 | # Running epochs_class epochs 200 | for epoch in range(0, args.epochs_class): 201 | my_trainer.update_lr(epoch) 202 | my_trainer.train(epoch) 203 | # print(my_trainer.threshold) 204 | if epoch % args.log_interval == (args.log_interval - 1): 205 | tError = t_classifier.evaluate(my_trainer.model, train_iterator) 206 | logger.debug("*********CURRENT EPOCH********** : %d", epoch) 207 | logger.debug("Train Classifier: %0.2f", tError) 208 | logger.debug("Test Classifier: %0.2f", t_classifier.evaluate(my_trainer.model, test_iterator)) 209 | logger.debug("Test Classifier Scaled: %0.2f", 210 | t_classifier.evaluate(my_trainer.model, test_iterator, my_trainer.dynamic_threshold, False, 211 | my_trainer.older_classes, args.step_size)) 212 | logger.info("Test Classifier Grad Scaled: %0.2f", 213 | t_classifier.evaluate(my_trainer.model, test_iterator, my_trainer.gradient_threshold_unreported_experiment, False, 214 | my_trainer.older_classes, args.step_size)) 215 | 216 | # Evaluate the learned classifier 217 | img = None 218 | 219 | logger.info("Test Classifier Final: %0.2f", t_classifier.evaluate(my_trainer.model, test_iterator)) 220 | logger.info("Test Classifier Final Scaled: %0.2f", 221 | t_classifier.evaluate(my_trainer.model, test_iterator, my_trainer.dynamic_threshold, False, 222 | my_trainer.older_classes, args.step_size)) 223 | logger.info("Test Classifier Final Grad Scaled: %0.2f", 224 | t_classifier.evaluate(my_trainer.model, test_iterator, my_trainer.gradient_threshold_unreported_experiment, False, 225 | my_trainer.older_classes, args.step_size)) 226 | 227 | higher_y.append(t_classifier.evaluate(my_trainer.model, test_iterator, higher=True)) 228 | 229 | y_grad_scaled.append( 230 | t_classifier.evaluate(my_trainer.model, test_iterator, my_trainer.gradient_threshold_unreported_experiment, False, 231 | my_trainer.older_classes, args.step_size)) 232 | y_scaled.append(t_classifier.evaluate(my_trainer.model, test_iterator, my_trainer.dynamic_threshold, False, 233 | my_trainer.older_classes, args.step_size)) 234 | y1.append(t_classifier.evaluate(my_trainer.model, test_iterator)) 235 | 236 | # Update means using the train iterator; this is iCaRL case 237 | nmc.update_means(my_trainer.model, train_iterator, dataset.classes) 238 | # Update mean using all the data. This is equivalent to memory_budget = infinity 239 | nmc_ideal.update_means(my_trainer.model, train_iterator_nmc, dataset.classes) 240 | # Compute the the nmc based classification results 241 | tempTrain = t_classifier.evaluate(my_trainer.model, train_iterator) 242 | train_y.append(tempTrain) 243 | 244 | testY1 = nmc.evaluate(my_trainer.model, test_iterator, step_size=args.step_size, kMean=True) 245 | testY = nmc.evaluate(my_trainer.model, test_iterator) 246 | testY_ideal = nmc_ideal.evaluate(my_trainer.model, test_iterator) 247 | y.append(testY) 248 | nmc_ideal_cum.append(testY_ideal) 249 | 250 | # Compute confusion matrices of all three cases (Learned classifier, iCaRL, and ideal NMC) 251 | tcMatrix = t_classifier.get_confusion_matrix(my_trainer.model, test_iterator, dataset.classes) 252 | tcMatrix_scaled = t_classifier.get_confusion_matrix(my_trainer.model, test_iterator, dataset.classes, 253 | my_trainer.dynamic_threshold, my_trainer.older_classes, 254 | args.step_size) 255 | tcMatrix_grad_scaled = t_classifier.get_confusion_matrix(my_trainer.model, test_iterator, 256 | dataset.classes, 257 | my_trainer.gradient_threshold_unreported_experiment, 258 | my_trainer.older_classes, 259 | args.step_size) 260 | nmcMatrix = nmc.get_confusion_matrix(my_trainer.model, test_iterator, dataset.classes) 261 | nmcMatrixIdeal = nmc_ideal.get_confusion_matrix(my_trainer.model, test_iterator, dataset.classes) 262 | tcMatrix_scaled_binning = t_classifier.get_confusion_matrix(my_trainer.model, test_iterator, 263 | dataset.classes, 264 | my_trainer.dynamic_threshold, 265 | my_trainer.older_classes, 266 | args.step_size, True) 267 | 268 | my_trainer.setup_training() 269 | 270 | # Store the resutls in the my_experiment object; this object should contain all the information required to reproduce the results. 271 | x.append(class_group + args.step_size) 272 | 273 | my_experiment.results["NMC"] = [x, [float(p) for p in y]] 274 | my_experiment.results["Trained Classifier"] = [x, [float(p) for p in y1]] 275 | my_experiment.results["Trained Classifier Scaled"] = [x, [float(p) for p in y_scaled]] 276 | my_experiment.results["Trained Classifier Grad Scaled"] = [x, [float(p) for p in y_grad_scaled]] 277 | my_experiment.results["Train Error Classifier"] = [x, [float(p) for p in train_y]] 278 | my_experiment.results["Ideal NMC"] = [x, [float(p) for p in nmc_ideal_cum]] 279 | my_experiment.store_json() 280 | 281 | # Finally, plotting the results; 282 | my_plotter = plt.Plotter() 283 | 284 | # Plotting the confusion matrices 285 | my_plotter.plotMatrix(int(class_group / args.step_size) * args.epochs_class + epoch, 286 | my_experiment.path + "tcMatrix", tcMatrix) 287 | my_plotter.plotMatrix(int(class_group / args.step_size) * args.epochs_class + epoch, 288 | my_experiment.path + "tcMatrix_scaled", tcMatrix_scaled) 289 | my_plotter.plotMatrix(int(class_group / args.step_size) * args.epochs_class + epoch, 290 | my_experiment.path + "tcMatrix_scaled_binning", tcMatrix_scaled_binning) 291 | my_plotter.plotMatrix(int(class_group / args.step_size) * args.epochs_class + epoch, 292 | my_experiment.path + "nmcMatrix", 293 | nmcMatrix) 294 | my_plotter.plotMatrix(int(class_group / args.step_size) * args.epochs_class + epoch, 295 | my_experiment.path + "nmcMatrixIdeal", 296 | nmcMatrixIdeal) 297 | 298 | # Plotting the line diagrams of all the possible cases 299 | my_plotter.plot(x, y, title=args.name, legend="NMC") 300 | my_plotter.plot(x, higher_y, title=args.name, legend="Higher Model") 301 | my_plotter.plot(x, y_scaled, title=args.name, legend="Trained Classifier Scaled") 302 | my_plotter.plot(x, y_grad_scaled, title=args.name, legend="Trained Classifier Grad Scaled") 303 | my_plotter.plot(x, nmc_ideal_cum, title=args.name, legend="Ideal NMC") 304 | my_plotter.plot(x, y1, title=args.name, legend="Trained Classifier") 305 | my_plotter.plot(x, train_y, title=args.name, legend="Trained Classifier Train Set") 306 | 307 | # Saving the line plot 308 | my_plotter.save_fig(my_experiment.path, dataset.classes + 1) 309 | -------------------------------------------------------------------------------- /trainer/__init__.py: -------------------------------------------------------------------------------- 1 | from trainer.evaluator import * 2 | from trainer.trainer import * -------------------------------------------------------------------------------- /trainer/evaluator.py: -------------------------------------------------------------------------------- 1 | ''' Incremental-Classifier Learning 2 | Authors : Khurram Javed, Muhammad Talha Paracha 3 | Maintainer : Khurram Javed 4 | Lab : TUKL-SEECS R&D Lab 5 | Email : 14besekjaved@seecs.edu.pk ''' 6 | 7 | import logging 8 | 9 | import numpy as np 10 | import torch 11 | import torch.nn.functional as F 12 | from torch.autograd import Variable 13 | from torchnet.meter import confusionmeter 14 | 15 | logger = logging.getLogger('iCARL') 16 | 17 | 18 | class EvaluatorFactory(): 19 | ''' 20 | This class is used to get different versions of evaluators 21 | ''' 22 | 23 | def __init__(self): 24 | pass 25 | 26 | @staticmethod 27 | def get_evaluator(testType="nmc", cuda=True): 28 | if testType == "nmc": 29 | return NearestMeanEvaluator(cuda) 30 | if testType == "trainedClassifier": 31 | return softmax_evaluator(cuda) 32 | 33 | 34 | class NearestMeanEvaluator(): 35 | ''' 36 | Nearest Class Mean based classifier. Mean embedding is computed and stored; at classification time, the embedding closest to the 37 | input embedding corresponds to the predicted class. 38 | ''' 39 | 40 | def __init__(self, cuda): 41 | self.cuda = cuda 42 | self.means = None 43 | self.totalFeatures = np.zeros((100, 1)) 44 | 45 | def evaluate(self, model, loader, step_size=10, kMean=False): 46 | ''' 47 | :param model: Train model 48 | :param loader: Data loader 49 | :param step_size: Step size for incremental learning 50 | :param kMean: Doesn't work very well so don't use; Will be removed in future versions 51 | :return: 52 | ''' 53 | model.eval() 54 | if self.means is None: 55 | self.means = np.zeros((100, model.featureSize)) 56 | correct = 0 57 | 58 | for data, y, target in loader: 59 | if self.cuda: 60 | data, target = data.cuda(), target.cuda() 61 | self.means = self.means.cuda() 62 | data, target = Variable(data, volatile=True), Variable(target) 63 | output = model(data, True).unsqueeze(1) 64 | result = (output.data - self.means.float()) 65 | 66 | result = torch.norm(result, 2, 2) 67 | if kMean: 68 | result = result.cpu().numpy() 69 | tempClassifier = np.zeros((len(result), int(100 / step_size))) 70 | for outer in range(0, len(result)): 71 | for tempCounter in range(0, int(100 / step_size)): 72 | tempClassifier[outer, tempCounter] = np.sum( 73 | result[tempCounter * step_size:(tempCounter * step_size) + step_size]) 74 | for outer in range(0, len(result)): 75 | minClass = np.argmin(tempClassifier[outer, :]) 76 | result[outer, 0:minClass * step_size] += 300000 77 | result[outer, minClass * step_size:(minClass + 1) * step_size] += 300000 78 | result = torch.from_numpy(result) 79 | if self.cuda: 80 | result = result.cuda() 81 | _, pred = torch.min(result, 1) 82 | correct += pred.eq(target.data.view_as(pred)).cpu().sum() 83 | 84 | return 100. * correct / len(loader.dataset) 85 | 86 | def get_confusion_matrix(self, model, loader, size): 87 | ''' 88 | 89 | :param model: Trained model 90 | :param loader: Data iterator 91 | :param size: Size of confusion matrix (Equal to largest possible label predicted by the model) 92 | :return: 93 | ''' 94 | model.eval() 95 | test_loss = 0 96 | correct = 0 97 | # Get the confusion matrix object 98 | cMatrix = confusionmeter.ConfusionMeter(size, True) 99 | 100 | for data, y, target in loader: 101 | if self.cuda: 102 | data, target = data.cuda(), target.cuda() 103 | self.means = self.means.cuda() 104 | data, target = Variable(data, volatile=True), Variable(target) 105 | output = model(data, True).unsqueeze(1) 106 | result = (output.data - self.means.float()) 107 | 108 | result = torch.norm(result, 2, 2) 109 | # NMC for classification 110 | _, pred = torch.min(result, 1) 111 | # Evaluate results 112 | correct += pred.eq(target.data.view_as(pred)).cpu().sum() 113 | # Add the results in appropriate places in the matrix. 114 | cMatrix.add(pred, target.data.view_as(pred)) 115 | 116 | test_loss /= len(loader.dataset) 117 | # Get 2d numpy matrix to remove the dependency of other code on confusionmeter 118 | img = cMatrix.value() 119 | return img 120 | 121 | def update_means(self, model, train_loader, classes=100): 122 | ''' 123 | This method updates the mean embedding using the train data; DO NOT pass test data iterator to this. 124 | :param model: Trained model 125 | :param train_loader: data iterator 126 | :param classes: Total number of classes 127 | :return: 128 | ''' 129 | # Set the mean to zero 130 | if self.means is None: 131 | self.means = np.zeros((classes, model.featureSize)) 132 | self.means *= 0 133 | self.classes = classes 134 | self.means = np.zeros((classes, model.featureSize)) 135 | self.totalFeatures = np.zeros((classes, 1)) + .001 136 | logger.debug("Computing means") 137 | # Iterate over all train Dataset 138 | for batch_id, (data, y, target) in enumerate(train_loader): 139 | # Get features for a minibactch 140 | if self.cuda: 141 | data = data.cuda() 142 | features = model.forward(Variable(data), True) 143 | # Convert result to a numpy array 144 | featuresNp = features.data.cpu().numpy() 145 | # Accumulate the results in the means array 146 | # print (self.means.shape,featuresNp.shape) 147 | np.add.at(self.means, target, featuresNp) 148 | # Keep track of how many instances of a class have been seen. This should be an array with all elements = classSize 149 | np.add.at(self.totalFeatures, target, 1) 150 | 151 | # Divide the means array with total number of instan ces to get the average 152 | # print ("Total instances", self.totalFeatures) 153 | self.means = self.means / self.totalFeatures 154 | self.means = torch.from_numpy(self.means) 155 | # Normalize the mean vector 156 | self.means = self.means / torch.norm(self.means, 2, 1).unsqueeze(1) 157 | self.means[self.means != self.means] = 0 158 | self.means = self.means.unsqueeze(0) 159 | 160 | logger.debug("Mean vectors computed") 161 | # Return 162 | return 163 | 164 | 165 | class softmax_evaluator(): 166 | ''' 167 | Evaluator class for softmax classification 168 | ''' 169 | 170 | def __init__(self, cuda): 171 | self.cuda = cuda 172 | self.means = None 173 | self.totalFeatures = np.zeros((100, 1)) 174 | 175 | def evaluate(self, model, loader, scale=None, thres=False, older_classes=None, step_size=10, descriptor=False, 176 | falseDec=False, higher=False): 177 | ''' 178 | :param model: Trained model 179 | :param loader: Data iterator 180 | :param scale: Scale vector computed by dynamic threshold moving 181 | :param thres: If true, use scaling 182 | :param older_classes: Will be removed in next iteration 183 | :param step_size: Step size for incremental learning 184 | :param descriptor: Will be removed in next iteration; used to compare the results with a recent paper by Facebook. 185 | :param falseDec: Will be removed in the next iteration. 186 | :param higher: Hierarchical softmax support 187 | :return: 188 | ''' 189 | model.eval() 190 | correct = 0 191 | if scale is not None: 192 | scale = np.copy(scale) 193 | scale = scale / np.max(scale) 194 | # print ("Gets here") 195 | scaleTemp = np.copy(scale) 196 | if thres: 197 | for x in range(0, len(scale)): 198 | temp = 0 199 | for y in range(0, len(scale)): 200 | if x == y: 201 | pass 202 | else: 203 | temp = temp + (scale[y] / scale[x]) 204 | scaleTemp[x] = temp 205 | scale = scaleTemp 206 | else: 207 | scale = 1 / scale 208 | 209 | scale = scale / np.linalg.norm(scale, 1) 210 | scale = torch.from_numpy(scale).unsqueeze(0) 211 | if self.cuda: 212 | scale = scale.cuda() 213 | tempCounter = 0 214 | for data, y, target in loader: 215 | if self.cuda: 216 | data, target = data.cuda(), target.cuda() 217 | data, target = Variable(data, volatile=True), Variable(target) 218 | if thres: 219 | output = model(data) 220 | output = output * Variable(scale.float()) 221 | elif scale is not None: 222 | # print("Gets here, getting outputs") 223 | output = model(data, scale=Variable(scale.float())) 224 | else: 225 | output = model(data) 226 | if descriptor: 227 | # To compare with FB paper 228 | # output = output/Variable(scale.float()) 229 | outputTemp = output.data.cpu().numpy() 230 | targetTemp = target.data.cpu().numpy() 231 | if falseDec: 232 | for a in range(0, len(targetTemp)): 233 | random = np.random.choice(len(older_classes) + step_size, step_size, replace=False).tolist() 234 | if targetTemp[a] in random: 235 | pass 236 | else: 237 | random[0] = targetTemp[a] 238 | for b in random: 239 | outputTemp[a, b] += 20 240 | else: 241 | for a in range(0, len(targetTemp)): 242 | outputTemp[a, int(float(targetTemp[a]) / step_size) * step_size:(int( 243 | float(targetTemp[a]) / step_size) * step_size) + step_size] += 20 244 | if tempCounter == 0: 245 | print(int(float(targetTemp[a]) / step_size) * step_size, 246 | (int(float(targetTemp[a]) / step_size) * step_size) + step_size) 247 | tempCounter += 1 248 | 249 | output = torch.from_numpy(outputTemp) 250 | if self.cuda: 251 | output = output.cuda() 252 | output = Variable(output) 253 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 254 | correct += pred.eq(target.data.view_as(pred)).cpu().sum() 255 | 256 | return 100. * correct / len(loader.dataset) 257 | 258 | def get_confusion_matrix(self, model, loader, size, scale=None, older_classes=None, step_size=10, descriptor=False): 259 | ''' 260 | :return: Returns the confusion matrix on the data given by loader 261 | ''' 262 | model.eval() 263 | test_loss = 0 264 | correct = 0 265 | # Initialize confusion matrix 266 | cMatrix = confusionmeter.ConfusionMeter(size, True) 267 | 268 | # Checks is threshold moving should be used 269 | if scale is not None: 270 | scale = np.copy(scale) 271 | scale = scale / np.max(scale) 272 | scale = 1 / scale 273 | scale = torch.from_numpy(scale).unsqueeze(0) 274 | if self.cuda: 275 | scale = scale.cuda() 276 | 277 | # Iterate over the data and stores the results in the confusion matrix 278 | for data, y, target in loader: 279 | if self.cuda: 280 | data, target = data.cuda(), target.cuda() 281 | data, target = Variable(data, volatile=True), Variable(target) 282 | if scale is not None: 283 | output = model(data, scale=Variable(scale.float())) 284 | else: 285 | output = model(data) 286 | 287 | if descriptor: 288 | # To compare with FB paper 289 | outputTemp = output.data.cpu().numpy() 290 | targetTemp = target.data.cpu().numpy() 291 | for a in range(0, len(targetTemp)): 292 | outputTemp[a, int(float(targetTemp[a]) / step_size) * step_size:(int( 293 | float(targetTemp[a]) / step_size) * step_size) + step_size] += 20 294 | output = torch.from_numpy(outputTemp) 295 | if self.cuda: 296 | output = output.cuda() 297 | output = Variable(output) 298 | 299 | test_loss += F.nll_loss(output, target, size_average=False).data[0] # sum up batch loss 300 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 301 | correct += pred.eq(target.data.view_as(pred)).cpu().sum() 302 | cMatrix.add(pred.squeeze(), target.data.view_as(pred).squeeze()) 303 | 304 | # Returns normalized matrix. 305 | test_loss /= len(loader.dataset) 306 | img = cMatrix.value() 307 | return img 308 | -------------------------------------------------------------------------------- /trainer/trainer.py: -------------------------------------------------------------------------------- 1 | ''' Incremental-Classifier Learning 2 | Authors : Khurram Javed, Muhammad Talha Paracha 3 | Maintainer : Khurram Javed 4 | Lab : TUKL-SEECS R&D Lab 5 | Email : 14besekjaved@seecs.edu.pk ''' 6 | 7 | from __future__ import print_function 8 | 9 | import copy 10 | import logging 11 | 12 | import numpy as np 13 | import torch 14 | import torch.nn.functional as F 15 | from torch.autograd import Variable 16 | from tqdm import tqdm 17 | 18 | import model 19 | 20 | logger = logging.getLogger('iCARL') 21 | 22 | 23 | class GenericTrainer: 24 | ''' 25 | Base class for trainer; to implement a new training routine, inherit from this. 26 | ''' 27 | 28 | def __init__(self, trainDataIterator, testDataIterator, dataset, model, args, optimizer, ideal_iterator=None): 29 | self.train_data_iterator = trainDataIterator 30 | self.test_data_iterator = testDataIterator 31 | self.model = model 32 | self.args = args 33 | self.dataset = dataset 34 | self.train_loader = self.train_data_iterator.dataset 35 | self.older_classes = [] 36 | self.optimizer = optimizer 37 | self.model_fixed = copy.deepcopy(self.model) 38 | self.active_classes = [] 39 | for param in self.model_fixed.parameters(): 40 | param.requires_grad = False 41 | self.models = [] 42 | self.current_lr = args.lr 43 | self.all_classes = list(range(dataset.classes)) 44 | self.all_classes.sort(reverse=True) 45 | self.left_over = [] 46 | self.ideal_iterator = ideal_iterator 47 | self.model_single = copy.deepcopy(self.model) 48 | self.optimizer_single = None 49 | 50 | logger.warning("Shuffling turned off for debugging") 51 | # random.seed(args.seed) 52 | # random.shuffle(self.all_classes) 53 | 54 | 55 | class Trainer(GenericTrainer): 56 | def __init__(self, trainDataIterator, testDataIterator, dataset, model, args, optimizer, ideal_iterator=None): 57 | super().__init__(trainDataIterator, testDataIterator, dataset, model, args, optimizer, ideal_iterator) 58 | self.dynamic_threshold = np.ones(self.dataset.classes, dtype=np.float64) 59 | self.gradient_threshold_unreported_experiment = np.ones(self.dataset.classes, dtype=np.float64) 60 | 61 | def update_lr(self, epoch): 62 | for temp in range(0, len(self.args.schedule)): 63 | if self.args.schedule[temp] == epoch: 64 | for param_group in self.optimizer.param_groups: 65 | self.current_lr = param_group['lr'] 66 | param_group['lr'] = self.current_lr * self.args.gammas[temp] 67 | logger.debug("Changing learning rate from %0.2f to %0.2f", self.current_lr, 68 | self.current_lr * self.args.gammas[temp]) 69 | self.current_lr *= self.args.gammas[temp] 70 | 71 | def increment_classes(self, class_group): 72 | ''' 73 | Add classes starting from class_group to class_group + step_size 74 | :param class_group: 75 | :return: N/A. Only has side-affects 76 | ''' 77 | for temp in range(class_group, class_group + self.args.step_size): 78 | pop_val = self.all_classes.pop() 79 | self.train_data_iterator.dataset.add_class(pop_val) 80 | self.ideal_iterator.dataset.add_class(pop_val) 81 | self.test_data_iterator.dataset.add_class(pop_val) 82 | self.left_over.append(pop_val) 83 | 84 | def limit_class(self, n, k, herding=True): 85 | if not herding: 86 | self.train_loader.limit_class(n, k) 87 | else: 88 | self.train_loader.limit_class_and_sort(n, k, self.model_fixed) 89 | if n not in self.older_classes: 90 | self.older_classes.append(n) 91 | 92 | def reset_dynamic_threshold(self): 93 | ''' 94 | Reset the threshold vector maintaining the scale factor. 95 | Important to set this to zero before every increment. 96 | setupTraining() also does this so not necessary to call both. 97 | :return: 98 | ''' 99 | threshTemp = self.dynamic_threshold / np.max(self.dynamic_threshold) 100 | threshTemp = ['{0:.4f}'.format(i) for i in threshTemp] 101 | 102 | threshTemp2 = self.gradient_threshold_unreported_experiment / np.max( 103 | self.gradient_threshold_unreported_experiment) 104 | threshTemp2 = ['{0:.4f}'.format(i) for i in threshTemp2] 105 | 106 | logger.debug("Scale Factor" + ",".join(threshTemp)) 107 | logger.debug("Scale GFactor" + ",".join(threshTemp2)) 108 | 109 | self.dynamic_threshold = np.ones(self.dataset.classes, dtype=np.float64) 110 | self.gradient_threshold_unreported_experiment = np.ones(self.dataset.classes, dtype=np.float64) 111 | 112 | def setup_training(self): 113 | self.reset_dynamic_threshold() 114 | 115 | for param_group in self.optimizer.param_groups: 116 | logger.debug("Setting LR to %0.2f", self.args.lr) 117 | param_group['lr'] = self.args.lr 118 | self.current_lr = self.args.lr 119 | for val in self.left_over: 120 | self.limit_class(val, int(self.args.memory_budget / len(self.left_over)), not self.args.no_herding) 121 | 122 | def update_frozen_model(self): 123 | self.model.eval() 124 | self.model_fixed = copy.deepcopy(self.model) 125 | self.model_fixed.eval() 126 | for param in self.model_fixed.parameters(): 127 | param.requires_grad = False 128 | self.models.append(self.model_fixed) 129 | 130 | if self.args.random_init: 131 | logger.warning("Random Initilization of weights at each increment") 132 | myModel = model.ModelFactory.get_model(self.args.model_type, self.args.dataset) 133 | if self.args.cuda: 134 | myModel.cuda() 135 | self.model = myModel 136 | self.optimizer = torch.optim.SGD(self.model.parameters(), self.args.lr, momentum=self.args.momentum, 137 | weight_decay=self.args.decay, nesterov=True) 138 | self.model.eval() 139 | 140 | def randomly_init_model(self): 141 | logger.info("Randomly initilizaing model") 142 | myModel = model.ModelFactory.get_model(self.args.model_type, self.args.dataset) 143 | if self.args.cuda: 144 | myModel.cuda() 145 | self.model = myModel 146 | self.optimizer = torch.optim.SGD(self.model.parameters(), self.args.lr, momentum=self.args.momentum, 147 | weight_decay=self.args.decay, nesterov=True) 148 | self.model.eval() 149 | 150 | def get_model(self): 151 | myModel = model.ModelFactory.get_model(self.args.model_type, self.args.dataset) 152 | if self.args.cuda: 153 | myModel.cuda() 154 | optimizer = torch.optim.SGD(myModel.parameters(), self.args.lr, momentum=self.args.momentum, 155 | weight_decay=self.args.decay, nesterov=True) 156 | myModel.eval() 157 | 158 | self.current_lr = self.args.lr 159 | 160 | self.model_single = myModel 161 | self.optimizer_single = optimizer 162 | 163 | def train(self, epoch): 164 | 165 | self.model.train() 166 | logger.info("Epochs %d", epoch) 167 | for data, y, target in tqdm(self.train_data_iterator): 168 | if self.args.cuda: 169 | data, target = data.cuda(), target.cuda() 170 | y = y.cuda() 171 | oldClassesIndices = (target * 0).int() 172 | for elem in range(0, self.args.unstructured_size): 173 | oldClassesIndices = oldClassesIndices + (target == elem).int() 174 | 175 | old_classes_indices = torch.squeeze(torch.nonzero((oldClassesIndices > 0)).long()) 176 | new_classes_indices = torch.squeeze(torch.nonzero((oldClassesIndices == 0)).long()) 177 | 178 | self.optimizer.zero_grad() 179 | 180 | target_normal_loss = target[new_classes_indices] 181 | data_normal_loss = data[new_classes_indices] 182 | 183 | target_distillation_loss = y.float() 184 | data_distillation_loss = data 185 | 186 | y_onehot = torch.FloatTensor(len(target_normal_loss), self.dataset.classes) 187 | if self.args.cuda: 188 | y_onehot = y_onehot.cuda() 189 | 190 | y_onehot.zero_() 191 | target_normal_loss.unsqueeze_(1) 192 | y_onehot.scatter_(1, target_normal_loss, 1) 193 | 194 | output = self.model(Variable(data_normal_loss)) 195 | self.dynamic_threshold += np.sum(y_onehot.cpu().numpy(), 0) 196 | loss = F.kl_div(output, Variable(y_onehot)) 197 | 198 | myT = self.args.T 199 | 200 | if self.args.no_distill: 201 | pass 202 | 203 | elif len(self.older_classes) > 0: 204 | # Get softened labels of the model from a previous version of the model. 205 | pred2 = self.model_fixed(Variable(data_distillation_loss), T=myT, labels=True).data 206 | # Softened output of the model 207 | if myT > 1: 208 | output2 = self.model(Variable(data_distillation_loss), T=myT) 209 | else: 210 | output2 = output 211 | 212 | self.dynamic_threshold += (np.sum(pred2.cpu().numpy(), 0)) * ( 213 | myT * myT) * self.args.alpha 214 | loss2 = F.kl_div(output2, Variable(pred2)) 215 | 216 | loss2.backward(retain_graph=True) 217 | 218 | # Scale gradient by a factor of square of T. See Distilling Knowledge in Neural Networks by Hinton et.al. for details. 219 | for param in self.model.parameters(): 220 | if param.grad is not None: 221 | param.grad = param.grad * (myT * myT) * self.args.alpha 222 | 223 | if len(self.older_classes) == 0 or not self.args.no_nl: 224 | loss.backward() 225 | 226 | for param in self.model.named_parameters(): 227 | if "fc.weight" in param[0]: 228 | self.gradient_threshold_unreported_experiment *= 0.99 229 | self.gradient_threshold_unreported_experiment += np.sum(np.abs(param[1].grad.data.cpu().numpy()), 1) 230 | 231 | self.optimizer.step() 232 | 233 | if self.args.no_nl: 234 | self.dynamic_threshold[len(self.older_classes):len(self.dynamic_threshold)] = np.max(self.dynamic_threshold) 235 | self.gradient_threshold_unreported_experiment[ 236 | len(self.older_classes):len(self.gradient_threshold_unreported_experiment)] = np.max( 237 | self.gradient_threshold_unreported_experiment) 238 | else: 239 | self.dynamic_threshold[0:self.args.unstructured_size] = np.max(self.dynamic_threshold) 240 | self.gradient_threshold_unreported_experiment[0:self.args.unstructured_size] = np.max( 241 | self.gradient_threshold_unreported_experiment) 242 | 243 | self.dynamic_threshold[self.args.unstructured_size + len( 244 | self.older_classes) + self.args.step_size: len(self.dynamic_threshold)] = np.max( 245 | self.dynamic_threshold) 246 | self.gradient_threshold_unreported_experiment[self.args.unstructured_size + len( 247 | self.older_classes) + self.args.step_size: len(self.gradient_threshold_unreported_experiment)] = np.max( 248 | self.gradient_threshold_unreported_experiment) 249 | 250 | def add_model(self): 251 | model = copy.deepcopy(self.model_single) 252 | model.eval() 253 | for param in model.parameters(): 254 | param.requires_grad = False 255 | self.models.append(model) 256 | logger.debug("Total Models %d", len(self.models)) 257 | -------------------------------------------------------------------------------- /utils/Colorer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | import logging 4 | 5 | # Source : # http://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output 6 | 7 | # now we patch Python code to add color support to logging.StreamHandler 8 | def add_coloring_to_emit_windows(fn): 9 | # add methods we need to the class 10 | def _out_handle(self): 11 | import ctypes 12 | return ctypes.windll.kernel32.GetStdHandle(self.STD_OUTPUT_HANDLE) 13 | 14 | out_handle = property(_out_handle) 15 | 16 | def _set_color(self, code): 17 | import ctypes 18 | # Constants from the Windows API 19 | self.STD_OUTPUT_HANDLE = -11 20 | hdl = ctypes.windll.kernel32.GetStdHandle(self.STD_OUTPUT_HANDLE) 21 | ctypes.windll.kernel32.SetConsoleTextAttribute(hdl, code) 22 | 23 | setattr(logging.StreamHandler, '_set_color', _set_color) 24 | 25 | def new(*args): 26 | FOREGROUND_BLUE = 0x0001 # text color contains blue. 27 | FOREGROUND_GREEN = 0x0002 # text color contains green. 28 | FOREGROUND_RED = 0x0004 # text color contains red. 29 | FOREGROUND_INTENSITY = 0x0008 # text color is intensified. 30 | FOREGROUND_WHITE = FOREGROUND_BLUE | FOREGROUND_GREEN | FOREGROUND_RED 31 | # winbase.h 32 | STD_INPUT_HANDLE = -10 33 | STD_OUTPUT_HANDLE = -11 34 | STD_ERROR_HANDLE = -12 35 | 36 | # wincon.h 37 | FOREGROUND_BLACK = 0x0000 38 | FOREGROUND_BLUE = 0x0001 39 | FOREGROUND_GREEN = 0x0002 40 | FOREGROUND_CYAN = 0x0003 41 | FOREGROUND_RED = 0x0004 42 | FOREGROUND_MAGENTA = 0x0005 43 | FOREGROUND_YELLOW = 0x0006 44 | FOREGROUND_GREY = 0x0007 45 | FOREGROUND_INTENSITY = 0x0008 # foreground color is intensified. 46 | 47 | BACKGROUND_BLACK = 0x0000 48 | BACKGROUND_BLUE = 0x0010 49 | BACKGROUND_GREEN = 0x0020 50 | BACKGROUND_CYAN = 0x0030 51 | BACKGROUND_RED = 0x0040 52 | BACKGROUND_MAGENTA = 0x0050 53 | BACKGROUND_YELLOW = 0x0060 54 | BACKGROUND_GREY = 0x0070 55 | BACKGROUND_INTENSITY = 0x0080 # background color is intensified. 56 | 57 | levelno = args[1].levelno 58 | if (levelno >= 50): 59 | color = BACKGROUND_YELLOW | FOREGROUND_RED | FOREGROUND_INTENSITY | BACKGROUND_INTENSITY 60 | elif (levelno >= 40): 61 | color = FOREGROUND_RED | FOREGROUND_INTENSITY 62 | elif (levelno >= 30): 63 | color = FOREGROUND_YELLOW | FOREGROUND_INTENSITY 64 | elif (levelno >= 20): 65 | color = FOREGROUND_GREEN 66 | elif (levelno >= 10): 67 | color = FOREGROUND_MAGENTA 68 | else: 69 | color = FOREGROUND_WHITE 70 | args[0]._set_color(color) 71 | 72 | ret = fn(*args) 73 | args[0]._set_color(FOREGROUND_WHITE) 74 | # print "after" 75 | return ret 76 | 77 | return new 78 | 79 | 80 | def add_coloring_to_emit_ansi(fn): 81 | # add methods we need to the class 82 | def new(*args): 83 | levelno = args[1].levelno 84 | if (levelno >= 50): 85 | color = '\x1b[31m' # red 86 | elif (levelno >= 40): 87 | color = '\x1b[31m' # red 88 | elif (levelno >= 30): 89 | color = '\x1b[33m' # yellow 90 | elif (levelno >= 20): 91 | color = '\x1b[32m' # green 92 | elif (levelno >= 10): 93 | color = '\x1b[35m' # pink 94 | else: 95 | color = '\x1b[0m' # normal 96 | args[1].msg = color + args[1].msg + '\x1b[0m' # normal 97 | # print "after" 98 | return fn(*args) 99 | 100 | return new 101 | 102 | 103 | import platform 104 | 105 | if platform.system() == 'Windows': 106 | # Windows does not support ANSI escapes and we are using API calls to set the console color 107 | logging.StreamHandler.emit = add_coloring_to_emit_windows(logging.StreamHandler.emit) 108 | else: 109 | # all non-Windows platforms are supporting ANSI escapes so we use them 110 | logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit) 111 | # log = logging.getLogger() 112 | # log.addFilter(log_filter()) 113 | # //hdlr = logging.StreamHandler() 114 | # //hdlr.setFormatter(formatter()) 115 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | from utils.utils import * -------------------------------------------------------------------------------- /utils/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch.nn.functional as F 3 | from torch.autograd import Variable 4 | from torchnet.meter import confusionmeter 5 | 6 | 7 | def resize_image(img, factor): 8 | ''' 9 | 10 | :param img: 11 | :param factor: 12 | :return: 13 | ''' 14 | img2 = np.zeros(np.array(img.shape) * factor) 15 | 16 | for a in range(0, img.shape[0]): 17 | for b in range(0, img.shape[1]): 18 | img2[a * factor:(a + 1) * factor, b * factor:(b + 1) * factor] = img[a, b] 19 | return img2 20 | 21 | 22 | def save_confusion_matrix(epoch, path, model, args, dataset, test_loader): 23 | model.eval() 24 | test_loss = 0 25 | correct = 0 26 | cMatrix = confusionmeter.ConfusionMeter(dataset.classes, True) 27 | 28 | for data, target in test_loader: 29 | if args.cuda: 30 | data, target = data.cuda(), target.cuda() 31 | data, target = Variable(data, volatile=True), Variable(target) 32 | output = model(data) 33 | test_loss += F.nll_loss(output, target, size_average=False).data[0] # sum up batch loss 34 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 35 | correct += pred.eq(target.data.view_as(pred)).cpu().sum() 36 | if epoch > 0: 37 | cMatrix.add(pred, target.data.view_as(pred)) 38 | 39 | test_loss /= len(test_loader.dataset) 40 | img = cMatrix.value() 41 | 42 | plt.imshow(img, cmap='plasma', interpolation='nearest') 43 | plt.colorbar() 44 | plt.savefig(path + str(epoch) + ".jpg") 45 | plt.gcf().clear() 46 | return 100. * correct / len(test_loader.dataset) 47 | 48 | 49 | import matplotlib.pyplot as plt 50 | 51 | cur = 1 52 | 53 | 54 | # Function to plot images; 55 | 56 | 57 | 58 | def plot(img, title, g=True): 59 | global cur, fig 60 | p = fig.add_subplot(10, 10, cur) 61 | p.set_title(title) 62 | cur += 1 63 | if g: 64 | plt.imshow(img, cmap='gray') 65 | else: 66 | plt.imshow(img) 67 | 68 | 69 | def visualizeTensor(t, path): 70 | global cur, fig 71 | cur = 1 72 | # Function to plot images; 73 | fig = plt.figure(figsize=(10, 10)) 74 | for a in t: 75 | img = a.cpu().numpy() 76 | img = np.swapaxes(img, 0, 2) 77 | imgMin = np.min(img) 78 | 79 | # img = img-np.min(img) 80 | # img = img/np.max(img) 81 | plot(img, str(cur), False) 82 | plt.savefig(path) 83 | plt.gcf().clear() 84 | plt.close() 85 | --------------------------------------------------------------------------------