├── data ├── .gitkeep ├── imagenet_classes.pkl └── imagenet_class_index.json ├── networks ├── results │ ├── .gitkeep │ ├── targeted_results.pkl │ └── untargeted_results.pkl ├── models │ ├── lenet.h5 │ └── resnet.h5 ├── train_plot.py ├── capsnet.py ├── vgg16.py ├── capsulenet │ ├── helper_function.py │ ├── capsule_net.py │ ├── capsule_layers.py │ ├── capsulenet.py │ └── capsulelayers.py ├── lenet.py ├── pure_cnn.py ├── network_in_network.py ├── wide_resnet.py ├── resnet.py └── densenet.py ├── images ├── pred.png ├── Ackley.gif ├── pred2.png └── who-would-win.jpg ├── requirements.txt ├── CONTRIBUTING.md ├── LICENSE ├── train.py ├── .gitignore ├── helper.py ├── attack.py ├── README.md └── differential_evolution.py /data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /networks/results/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/pred.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hyperparticle/one-pixel-attack-keras/HEAD/images/pred.png -------------------------------------------------------------------------------- /images/Ackley.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hyperparticle/one-pixel-attack-keras/HEAD/images/Ackley.gif -------------------------------------------------------------------------------- /images/pred2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hyperparticle/one-pixel-attack-keras/HEAD/images/pred2.png -------------------------------------------------------------------------------- /data/imagenet_classes.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hyperparticle/one-pixel-attack-keras/HEAD/data/imagenet_classes.pkl -------------------------------------------------------------------------------- /images/who-would-win.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hyperparticle/one-pixel-attack-keras/HEAD/images/who-would-win.jpg -------------------------------------------------------------------------------- /networks/models/lenet.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hyperparticle/one-pixel-attack-keras/HEAD/networks/models/lenet.h5 -------------------------------------------------------------------------------- /networks/models/resnet.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hyperparticle/one-pixel-attack-keras/HEAD/networks/models/resnet.h5 -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | jupyter 2 | keras 3 | tensorflow-gpu 4 | pandas 5 | numpy 6 | matplotlib 7 | scipy 8 | tqdm 9 | requests -------------------------------------------------------------------------------- /networks/results/targeted_results.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hyperparticle/one-pixel-attack-keras/HEAD/networks/results/targeted_results.pkl -------------------------------------------------------------------------------- /networks/results/untargeted_results.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hyperparticle/one-pixel-attack-keras/HEAD/networks/results/untargeted_results.pkl -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Any contributions are welcome. 2 | 3 | - If you have any questions, feel free to open an issue. 4 | - If you want to contribute your code, fork this repository and open a pull request. 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Dan Kondratyuk 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import argparse 5 | 6 | from networks.lenet import LeNet 7 | from networks.pure_cnn import PureCnn 8 | from networks.network_in_network import NetworkInNetwork 9 | from networks.resnet import ResNet 10 | from networks.densenet import DenseNet 11 | from networks.wide_resnet import WideResNet 12 | from networks.capsnet import CapsNet 13 | 14 | if __name__ == '__main__': 15 | models = { 16 | 'lenet': LeNet, 17 | 'pure_cnn': PureCnn, 18 | 'net_in_net': NetworkInNetwork, 19 | 'resnet': ResNet, 20 | 'densenet': DenseNet, 21 | 'wide_resnet': WideResNet, 22 | 'capsnet': CapsNet 23 | } 24 | 25 | parser = argparse.ArgumentParser(description='Train models on Cifar10') 26 | parser.add_argument('--model', choices=models.keys(), required=True, help='Specify a model by name to train.') 27 | 28 | parser.add_argument('--epochs', default=None, type=int) 29 | parser.add_argument('--batch_size', default=None, type=int) 30 | 31 | args = parser.parse_args() 32 | model_name = args.model 33 | 34 | args = {k: v for k, v in vars(args).items() if v != None} 35 | del args['model'] 36 | 37 | model = models[model_name](**args, load_weights=False) 38 | 39 | model.train() 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | *.h5 104 | 105 | networks/models 106 | !networks/models/lenet.h5 107 | !networks/models/resnet.h5 108 | data/*.jpg 109 | 110 | /.idea/ 111 | -------------------------------------------------------------------------------- /networks/train_plot.py: -------------------------------------------------------------------------------- 1 | from matplotlib import pyplot as plt 2 | from IPython.display import clear_output 3 | import keras 4 | 5 | # Live loss plot as the network trains 6 | # https://gist.github.com/stared/dfb4dfaf6d9a8501cd1cc8b8cb806d2e 7 | class PlotLearning(keras.callbacks.Callback): 8 | def __init__(self, clear_on_begin=False): 9 | self.clear_on_begin = clear_on_begin 10 | self.reset() 11 | 12 | def on_train_begin(self, logs={}): 13 | if (self.clear_on_begin): 14 | self.reset() 15 | 16 | def reset(self): 17 | self.i = 0 18 | self.x = [] 19 | self.losses = [] 20 | self.val_losses = [] 21 | self.acc = [] 22 | self.val_acc = [] 23 | self.fig = plt.figure() 24 | 25 | self.logs = [] 26 | 27 | def on_epoch_end(self, epoch, logs={}): 28 | self.logs.append(logs) 29 | self.x.append(self.i) 30 | self.losses.append(logs.get('loss')) 31 | self.val_losses.append(logs.get('val_loss')) 32 | self.acc.append(logs.get('acc')) 33 | self.val_acc.append(logs.get('val_acc')) 34 | self.i += 1 35 | 36 | if (self.i < 3): 37 | return 38 | 39 | f, (ax1, ax2) = plt.subplots(1, 2, figsize=(15,5)) 40 | 41 | clear_output(wait=True) 42 | 43 | ax1.plot(self.x, self.losses, label="loss") 44 | ax1.plot(self.x, self.val_losses, label="val_loss") 45 | ax1.set_title('Model Loss') 46 | ax1.set_ylabel('Loss') 47 | ax1.set_xlabel('Epoch') 48 | ax1.legend(['train', 'val'], loc='best') 49 | 50 | ax2.plot(self.x, self.acc, label="accuracy") 51 | ax2.plot(self.x, self.val_acc, label="validation accuracy") 52 | ax2.set_title('Model Accuracy') 53 | ax2.set_ylabel('Accuracy') 54 | ax2.set_xlabel('Epoch') 55 | ax2.legend(['train', 'val'], loc='best') 56 | 57 | plt.show() -------------------------------------------------------------------------------- /networks/capsnet.py: -------------------------------------------------------------------------------- 1 | from keras.utils import to_categorical 2 | import numpy as np 3 | 4 | from networks.capsulenet.capsule_net import CapsNetv1, train as train_net 5 | 6 | # Capsule Network taken from https://github.com/theblackcat102/dynamic-routing-capsule-cifar 7 | class CapsNet: 8 | def __init__(self, epochs=200, batch_size=128, load_weights=True): 9 | self.name = 'capsnet' 10 | self.model_filename = 'networks/models/capsnet.h5' 11 | self.num_classes = 10 12 | self.input_shape = 32, 32, 3 13 | self.num_routes = 3 14 | self.batch_size = batch_size 15 | self.epochs = epochs 16 | 17 | self._model = CapsNetv1(input_shape=self.input_shape, 18 | n_class=self.num_classes, 19 | n_route=self.num_routes) 20 | 21 | if load_weights: 22 | try: 23 | self._model.load_weights(self.model_filename) 24 | print('Successfully loaded', self.name) 25 | except (ImportError, ValueError, OSError): 26 | print('Failed to load', self.name) 27 | 28 | def count_params(self): 29 | return self._model.count_params() 30 | 31 | def train(self): 32 | self._model = train_net() 33 | 34 | def color_process(self, imgs): 35 | if imgs.ndim < 4: 36 | imgs = np.array([imgs]) 37 | imgs = imgs.astype('float32') 38 | mean = [125.307, 122.95, 113.865] 39 | std = [62.9932, 62.0887, 66.7048] 40 | for img in imgs: 41 | for i in range(3): 42 | img[:,:,i] = (img[:,:,i] - mean[i]) / std[i] 43 | return imgs 44 | 45 | def predict(self, img): 46 | label = to_categorical(np.repeat([0], len(img)), self.num_classes) # Don't care what label it is, just needs to be fed into the network 47 | processed = self.color_process(img) 48 | input_ = [processed, label] 49 | pred, _ = self._model.predict(input_, batch_size=self.batch_size) 50 | return pred 51 | 52 | def predict_one(self, img): 53 | return self.predict(img)[0] 54 | -------------------------------------------------------------------------------- /networks/vgg16.py: -------------------------------------------------------------------------------- 1 | # Very deep convolutional model (VGG16) from https://arxiv.org/pdf/1409.1556.pdf 2 | # Code taken from https://github.com/geifmany/cifar-vgg 3 | 4 | def vgg16_model(input_shape=INPUT_SHAPE): 5 | weight_decay = 0.0005 6 | 7 | model = Sequential() 8 | 9 | model.add(Conv2D(64, (3, 3), padding='same', 10 | input_shape=input_shape,kernel_regularizer=regularizers.l2(weight_decay))) 11 | model.add(Activation('relu')) 12 | model.add(BatchNormalization()) 13 | model.add(Dropout(0.3)) 14 | 15 | model.add(Conv2D(64, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 16 | model.add(Activation('relu')) 17 | model.add(BatchNormalization()) 18 | 19 | model.add(MaxPooling2D(pool_size=(2, 2))) 20 | 21 | model.add(Conv2D(128, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 22 | model.add(Activation('relu')) 23 | model.add(BatchNormalization()) 24 | model.add(Dropout(0.4)) 25 | 26 | model.add(Conv2D(128, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 27 | model.add(Activation('relu')) 28 | model.add(BatchNormalization()) 29 | 30 | model.add(MaxPooling2D(pool_size=(2, 2))) 31 | 32 | model.add(Conv2D(256, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 33 | model.add(Activation('relu')) 34 | model.add(BatchNormalization()) 35 | model.add(Dropout(0.4)) 36 | 37 | model.add(Conv2D(256, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 38 | model.add(Activation('relu')) 39 | model.add(BatchNormalization()) 40 | model.add(Dropout(0.4)) 41 | 42 | model.add(Conv2D(256, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 43 | model.add(Activation('relu')) 44 | model.add(BatchNormalization()) 45 | 46 | model.add(MaxPooling2D(pool_size=(2, 2))) 47 | 48 | 49 | model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 50 | model.add(Activation('relu')) 51 | model.add(BatchNormalization()) 52 | model.add(Dropout(0.4)) 53 | 54 | model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 55 | model.add(Activation('relu')) 56 | model.add(BatchNormalization()) 57 | model.add(Dropout(0.4)) 58 | 59 | model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 60 | model.add(Activation('relu')) 61 | model.add(BatchNormalization()) 62 | 63 | model.add(MaxPooling2D(pool_size=(2, 2))) 64 | 65 | 66 | model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 67 | model.add(Activation('relu')) 68 | model.add(BatchNormalization()) 69 | model.add(Dropout(0.4)) 70 | 71 | model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 72 | model.add(Activation('relu')) 73 | model.add(BatchNormalization()) 74 | model.add(Dropout(0.4)) 75 | 76 | model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 77 | model.add(Activation('relu')) 78 | model.add(BatchNormalization()) 79 | 80 | model.add(MaxPooling2D(pool_size=(2, 2))) 81 | model.add(Dropout(0.5)) 82 | 83 | model.add(Flatten()) 84 | model.add(Dense(512,kernel_regularizer=regularizers.l2(weight_decay))) 85 | model.add(Activation('relu')) 86 | model.add(BatchNormalization()) 87 | 88 | model.add(Dropout(0.5)) 89 | model.add(Dense(num_classes)) 90 | model.add(Activation('softmax')) 91 | 92 | # model.summary() 93 | 94 | return model -------------------------------------------------------------------------------- /networks/capsulenet/helper_function.py: -------------------------------------------------------------------------------- 1 | import os, math, csv 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | from keras.utils import to_categorical 5 | from pandas import read_csv 6 | 7 | def load_cifar_10(): 8 | from keras.datasets import cifar10 9 | num_classes = 10 10 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 11 | x_train = x_train.astype('float32') 12 | x_test = x_test.astype('float32') 13 | x_train /= 255.0 14 | x_test /= 255.0 15 | y_train = to_categorical(y_train, num_classes) 16 | y_test = to_categorical(y_test, num_classes) 17 | 18 | return (x_train,y_train),(x_test,y_test) 19 | 20 | def load_cifar_100(): 21 | from keras.datasets import cifar100 22 | num_classes = 100 23 | (x_train, y_train), (x_test, y_test) = cifar100.load_data() 24 | x_train = x_train.astype('float32') 25 | x_test = x_test.astype('float32') 26 | x_train /= 255.0 27 | x_test /= 255.0 28 | y_train = to_categorical(y_train, num_classes) 29 | y_test = to_categorical(y_test, num_classes) 30 | 31 | return (x_train,y_train),(x_test,y_test) 32 | 33 | def combine_images(generated_images): 34 | num = generated_images.shape[0] 35 | width = int(math.sqrt(num)) 36 | height = int(math.ceil(float(num)/width)) 37 | shape = generated_images.shape[1:3] 38 | image = np.zeros((height*shape[0], width*shape[1]), 39 | dtype=generated_images.dtype) 40 | for index, img in enumerate(generated_images): 41 | i = int(index/width) 42 | j = index % width 43 | image[i*shape[0]:(i+1)*shape[0], j*shape[1]:(j+1)*shape[1]] = \ 44 | img[:, :, 0] 45 | return image 46 | 47 | def initializer(): 48 | if not os.path.exists('results/'): 49 | os.mkdir('results') 50 | if not os.path.exists('weights/'): 51 | os.mkdir('weights') 52 | 53 | def plot_log(filename, show=True): 54 | # load data 55 | log_df = read_csv(filename) 56 | # epoch_list = [i for i in range(len(values[:,0]))] 57 | fig = plt.figure(figsize=(4,6)) 58 | fig.subplots_adjust(top=0.95, bottom=0.05, right=0.95) 59 | fig.add_subplot(211) 60 | for column in list(log_df): 61 | if 'loss' in column and 'val' in column: 62 | plt.plot(log_df['epoch'].tolist(),log_df[column].tolist(), label=column) 63 | plt.legend() 64 | plt.title('Training loss') 65 | 66 | fig.add_subplot(212) 67 | for column in list(log_df): 68 | if 'acc' in column : 69 | plt.plot(log_df['epoch'].tolist(),log_df[column].tolist(), label=column) 70 | plt.legend() 71 | plt.title('Training and validation accuracy') 72 | 73 | # fig.savefig('result/log.png') 74 | if show: 75 | plt.show() 76 | 77 | 78 | def data_generator(x,y,batch_size): 79 | x_train,y_train = x,y 80 | from keras.preprocessing.image import ImageDataGenerator 81 | datagen = ImageDataGenerator( 82 | featurewise_center=False, # set input mean to 0 over the dataset 83 | samplewise_center=False, # set each sample mean to 0 84 | featurewise_std_normalization=False, # divide inputs by std of the dataset 85 | samplewise_std_normalization=False, # divide each input by its std 86 | zca_whitening=False, # apply ZCA whitening 87 | rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180) 88 | width_shift_range=0.1, # randomly shift images horizontally (fraction of total width) 89 | height_shift_range=0.1, # randomly shift images vertically (fraction of total height) 90 | horizontal_flip=True, # randomly flip images 91 | vertical_flip=False) # randomly flip images 92 | 93 | # Compute quantities required for featurewise normalization 94 | # (std, mean, and principal components if ZCA whitening is applied). 95 | datagen.fit(x_train) 96 | generator = datagen.flow(x_train,y_train,batch_size=batch_size) 97 | while True: 98 | x,y = generator.next() 99 | yield ([x,y],[y,x]) -------------------------------------------------------------------------------- /networks/lenet.py: -------------------------------------------------------------------------------- 1 | import keras 2 | import numpy as np 3 | from keras import optimizers 4 | from keras.datasets import cifar10 5 | from keras.models import Sequential, load_model 6 | from keras.layers import Conv2D, Dense, Flatten, MaxPooling2D 7 | from keras.callbacks import LearningRateScheduler, TensorBoard, ModelCheckpoint 8 | from keras.preprocessing.image import ImageDataGenerator 9 | from keras.regularizers import l2 10 | 11 | from networks.train_plot import PlotLearning 12 | 13 | # Code taken from https://github.com/BIGBALLON/cifar-10-cnn 14 | class LeNet: 15 | def __init__(self, epochs=200, batch_size=128, load_weights=True): 16 | self.name = 'lenet' 17 | self.model_filename = 'networks/models/lenet.h5' 18 | self.num_classes = 10 19 | self.input_shape = 32, 32, 3 20 | self.batch_size = batch_size 21 | self.epochs = epochs 22 | self.iterations = 391 23 | self.weight_decay = 0.0001 24 | self.log_filepath = r'networks/models/lenet/' 25 | 26 | if load_weights: 27 | try: 28 | self._model = load_model(self.model_filename) 29 | print('Successfully loaded', self.name) 30 | except (ImportError, ValueError, OSError): 31 | print('Failed to load', self.name) 32 | 33 | def count_params(self): 34 | return self._model.count_params() 35 | 36 | def color_preprocessing(self, x_train, x_test): 37 | x_train = x_train.astype('float32') 38 | x_test = x_test.astype('float32') 39 | mean = [125.307, 122.95, 113.865] 40 | std = [62.9932, 62.0887, 66.7048] 41 | for i in range(3): 42 | x_train[:,:,:,i] = (x_train[:,:,:,i] - mean[i]) / std[i] 43 | x_test[:,:,:,i] = (x_test[:,:,:,i] - mean[i]) / std[i] 44 | return x_train, x_test 45 | 46 | def build_model(self): 47 | model = Sequential() 48 | model.add(Conv2D(6, (5, 5), padding='valid', activation = 'relu', kernel_initializer='he_normal', kernel_regularizer=l2(self.weight_decay), input_shape=self.input_shape)) 49 | model.add(MaxPooling2D((2, 2), strides=(2, 2))) 50 | model.add(Conv2D(16, (5, 5), padding='valid', activation = 'relu', kernel_initializer='he_normal', kernel_regularizer=l2(self.weight_decay))) 51 | model.add(MaxPooling2D((2, 2), strides=(2, 2))) 52 | model.add(Flatten()) 53 | model.add(Dense(120, activation = 'relu', kernel_initializer='he_normal', kernel_regularizer=l2(self.weight_decay) )) 54 | model.add(Dense(84, activation = 'relu', kernel_initializer='he_normal', kernel_regularizer=l2(self.weight_decay) )) 55 | model.add(Dense(10, activation = 'softmax', kernel_initializer='he_normal', kernel_regularizer=l2(self.weight_decay) )) 56 | sgd = optimizers.SGD(lr=.1, momentum=0.9, nesterov=True) 57 | model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) 58 | return model 59 | 60 | def scheduler(self, epoch): 61 | if epoch <= 60: 62 | return 0.05 63 | if epoch <= 120: 64 | return 0.01 65 | if epoch <= 160: 66 | return 0.002 67 | return 0.0004 68 | 69 | def train(self): 70 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 71 | y_train = keras.utils.to_categorical(y_train, self.num_classes) 72 | y_test = keras.utils.to_categorical(y_test, self.num_classes) 73 | 74 | # color preprocessing 75 | x_train, x_test = self.color_preprocessing(x_train, x_test) 76 | 77 | # build network 78 | model = self.build_model() 79 | model.summary() 80 | 81 | # Save the best model during each training checkpoint 82 | checkpoint = ModelCheckpoint(self.model_filename, 83 | monitor='val_loss', 84 | verbose=0, 85 | save_best_only= True, 86 | mode='auto') 87 | plot_callback = PlotLearning() 88 | tb_cb = TensorBoard(log_dir=self.log_filepath, histogram_freq=0) 89 | 90 | cbks = [checkpoint, plot_callback, tb_cb] 91 | 92 | # using real-time data augmentation 93 | print('Using real-time data augmentation.') 94 | datagen = ImageDataGenerator(horizontal_flip=True, 95 | width_shift_range=0.125,height_shift_range=0.125,fill_mode='constant',cval=0.) 96 | 97 | datagen.fit(x_train) 98 | 99 | # start traing 100 | model.fit_generator(datagen.flow(x_train, y_train,batch_size=self.batch_size), 101 | steps_per_epoch=self.iterations, 102 | epochs=self.epochs, 103 | callbacks=cbks, 104 | validation_data=(x_test, y_test)) 105 | # save model 106 | model.save(self.model_filename) 107 | 108 | self._model = model 109 | 110 | def color_process(self, imgs): 111 | if imgs.ndim < 4: 112 | imgs = np.array([imgs]) 113 | imgs = imgs.astype('float32') 114 | mean = [125.307, 122.95, 113.865] 115 | std = [62.9932, 62.0887, 66.7048] 116 | for img in imgs: 117 | for i in range(3): 118 | img[:,:,i] = (img[:,:,i] - mean[i]) / std[i] 119 | return imgs 120 | 121 | def predict(self, img): 122 | processed = self.color_process(img) 123 | return self._model.predict(processed, batch_size=self.batch_size) 124 | 125 | def predict_one(self, img): 126 | return self.predict(img)[0] 127 | 128 | def accuracy(self): 129 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 130 | y_train = keras.utils.to_categorical(y_train, self.num_classes) 131 | y_test = keras.utils.to_categorical(y_test, self.num_classes) 132 | 133 | # color preprocessing 134 | x_train, x_test = self.color_preprocessing(x_train, x_test) 135 | 136 | return self._model.evaluate(x_test, y_test, verbose=0)[1] 137 | 138 | 139 | 140 | -------------------------------------------------------------------------------- /networks/pure_cnn.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib 3 | from matplotlib import pyplot as plt 4 | import keras 5 | from keras.models import Sequential 6 | from keras.optimizers import Adam, SGD 7 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, TensorBoard 8 | from keras.constraints import maxnorm 9 | from keras.models import load_model 10 | from keras.layers import GlobalAveragePooling2D, Lambda, Conv2D, MaxPooling2D, Dropout, Dense, Flatten, Activation 11 | from keras.preprocessing.image import ImageDataGenerator 12 | from keras.datasets import cifar10 13 | 14 | from networks.train_plot import PlotLearning 15 | 16 | # A pure CNN model from https://arxiv.org/pdf/1412.6806.pdf 17 | # Code taken from https://github.com/09rohanchopra/cifar10 18 | class PureCnn: 19 | def __init__(self, epochs=350, batch_size=128, load_weights=True): 20 | self.name = 'pure_cnn' 21 | self.model_filename = 'networks/models/pure_cnn.h5' 22 | self.num_classes = 10 23 | self.input_shape = 32, 32, 3 24 | self.batch_size = batch_size 25 | self.epochs = epochs 26 | self.learn_rate = 1.0e-4 27 | self.log_filepath = r'networks/models/pure_cnn/' 28 | 29 | if load_weights: 30 | try: 31 | self._model = load_model(self.model_filename) 32 | print('Successfully loaded', self.name) 33 | except (ImportError, ValueError, OSError): 34 | print('Failed to load', self.name) 35 | 36 | def count_params(self): 37 | return self._model.count_params() 38 | 39 | def color_preprocessing(self, x_train, x_test): 40 | x_train = x_train.astype('float32') 41 | x_test = x_test.astype('float32') 42 | mean = [125.307, 122.95, 113.865] 43 | std = [62.9932, 62.0887, 66.7048] 44 | for i in range(3): 45 | x_train[:,:,:,i] = (x_train[:,:,:,i] - mean[i]) / std[i] 46 | x_test[:,:,:,i] = (x_test[:,:,:,i] - mean[i]) / std[i] 47 | return x_train, x_test 48 | 49 | def pure_cnn_network(self, input_shape): 50 | model = Sequential() 51 | 52 | model.add(Conv2D(96, (3, 3), activation='relu', padding = 'same', input_shape=input_shape)) 53 | model.add(Dropout(0.2)) 54 | 55 | model.add(Conv2D(96, (3, 3), activation='relu', padding = 'same')) 56 | model.add(Conv2D(96, (3, 3), activation='relu', padding = 'same', strides = 2)) 57 | model.add(Dropout(0.5)) 58 | 59 | model.add(Conv2D(192, (3, 3), activation='relu', padding = 'same')) 60 | model.add(Conv2D(192, (3, 3), activation='relu', padding = 'same')) 61 | model.add(Conv2D(192, (3, 3), activation='relu', padding = 'same', strides = 2)) 62 | model.add(Dropout(0.5)) 63 | 64 | model.add(Conv2D(192, (3, 3), padding = 'same')) 65 | model.add(Activation('relu')) 66 | model.add(Conv2D(192, (1, 1),padding='valid')) 67 | model.add(Activation('relu')) 68 | model.add(Conv2D(10, (1, 1), padding='valid')) 69 | 70 | model.add(GlobalAveragePooling2D()) 71 | 72 | model.add(Activation('softmax')) 73 | 74 | return model 75 | 76 | def train(self): 77 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 78 | y_train = keras.utils.to_categorical(y_train, self.num_classes) 79 | y_test = keras.utils.to_categorical(y_test, self.num_classes) 80 | 81 | # color preprocessing 82 | x_train, x_test = self.color_preprocessing(x_train, x_test) 83 | 84 | model = self.pure_cnn_network(self.input_shape) 85 | model.summary() 86 | 87 | # Save the best model during each training checkpoint 88 | checkpoint = ModelCheckpoint(self.model_filename, 89 | monitor='val_loss', 90 | verbose=0, 91 | save_best_only= True, 92 | mode='auto') 93 | plot_callback = PlotLearning() 94 | tb_cb = TensorBoard(log_dir=self.log_filepath, histogram_freq=0) 95 | 96 | cbks = [checkpoint, plot_callback, tb_cb] 97 | 98 | # set data augmentation 99 | print('Using real-time data augmentation.') 100 | datagen = ImageDataGenerator(horizontal_flip=True, 101 | width_shift_range=0.125,height_shift_range=0.125,fill_mode='constant',cval=0.) 102 | 103 | datagen.fit(x_train) 104 | 105 | model.compile(loss='categorical_crossentropy', # Better loss function for neural networks 106 | optimizer=Adam(lr=self.learn_rate), # Adam optimizer with 1.0e-4 learning rate 107 | metrics = ['accuracy']) # Metrics to be evaluated by the model 108 | 109 | model.fit_generator(datagen.flow(x_train, y_train, batch_size = self.batch_size), 110 | epochs = self.epochs, 111 | validation_data= (x_test, y_test), 112 | callbacks=cbks, 113 | verbose=1) 114 | 115 | model.save(self.model_filename) 116 | 117 | self._model = model 118 | 119 | def color_process(self, imgs): 120 | if imgs.ndim < 4: 121 | imgs = np.array([imgs]) 122 | imgs = imgs.astype('float32') 123 | mean = [125.307, 122.95, 113.865] 124 | std = [62.9932, 62.0887, 66.7048] 125 | for img in imgs: 126 | for i in range(3): 127 | img[:,:,i] = (img[:,:,i] - mean[i]) / std[i] 128 | return imgs 129 | 130 | def predict(self, img): 131 | processed = self.color_process(img) 132 | return self._model.predict(processed, batch_size=self.batch_size) 133 | 134 | def predict_one(self, img): 135 | return self.predict(img)[0] 136 | 137 | def accuracy(self): 138 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 139 | y_train = keras.utils.to_categorical(y_train, self.num_classes) 140 | y_test = keras.utils.to_categorical(y_test, self.num_classes) 141 | 142 | # color preprocessing 143 | x_train, x_test = self.color_preprocessing(x_train, x_test) 144 | 145 | return self._model.evaluate(x_test, y_test, verbose=0)[1] -------------------------------------------------------------------------------- /networks/capsulenet/capsule_net.py: -------------------------------------------------------------------------------- 1 | from keras.layers import ( 2 | Input, 3 | Conv2D, 4 | Activation, 5 | Dense, 6 | Flatten, 7 | Reshape, 8 | Dropout 9 | ) 10 | from keras.layers.merge import add 11 | from keras.regularizers import l2 12 | from keras.models import Model 13 | from keras.layers.normalization import BatchNormalization 14 | import keras.backend as K 15 | from keras import optimizers 16 | import numpy as np 17 | 18 | from networks.capsulenet.capsule_layers import CapsuleLayer, PrimaryCapsule, Length,Mask 19 | from networks.capsulenet.capsulenet import CapsNet as CapsNetv1 20 | from networks.capsulenet.helper_function import load_cifar_10,load_cifar_100 21 | 22 | 23 | def convolution_block(input,kernel_size=8,filters=16,kernel_regularizer=l2(1.e-4)): 24 | conv2 = Conv2D(filters=filters,kernel_size=kernel_size,kernel_regularizer=kernel_regularizer, 25 | kernel_initializer="he_normal",padding="same")(input) 26 | norm = BatchNormalization(axis=3)(conv2) 27 | activation = Activation("relu")(norm) 28 | return activation 29 | 30 | def CapsNet(input_shape,n_class,n_route,n_prime_caps=32,dense_size = (512,1024)): 31 | conv_filter = 256 32 | n_kernel = 24 33 | primary_channel =64 34 | primary_vector = 9 35 | vector_dim = 9 36 | 37 | target_shape = input_shape 38 | 39 | input = Input(shape=input_shape) 40 | 41 | # TODO: try leaky relu next time 42 | conv1 = Conv2D(filters=conv_filter,kernel_size=n_kernel, strides=1, padding='valid', activation='relu',name='conv1',kernel_initializer="he_normal")(input) 43 | 44 | primary_cap = PrimaryCapsule(conv1,dim_vector=8, n_channels=64,kernel_size=9,strides=2,padding='valid') 45 | 46 | routing_layer = CapsuleLayer(num_capsule=n_class, dim_vector=vector_dim, num_routing=n_route,name='routing_layer')(primary_cap) 47 | 48 | output = Length(name='output')(routing_layer) 49 | 50 | y = Input(shape=(n_class,)) 51 | masked = Mask()([routing_layer,y]) 52 | 53 | x_recon = Dense(dense_size[0],activation='relu')(masked) 54 | 55 | for i in range(1,len(dense_size)): 56 | x_recon = Dense(dense_size[i],activation='relu')(x_recon) 57 | # Is there any other way to do 58 | x_recon = Dense(target_shape[0]*target_shape[1]*target_shape[2],activation='relu')(x_recon) 59 | x_recon = Reshape(target_shape=target_shape,name='output_recon')(x_recon) 60 | 61 | return Model([input,y],[output,x_recon]) 62 | 63 | # why using 512, 1024 Maybe to mimic original 10M params? 64 | def CapsNetv2(input_shape,n_class,n_route,n_prime_caps=32,dense_size = (512,1024)): 65 | conv_filter = 64 66 | n_kernel = 16 67 | primary_channel =64 68 | primary_vector = 12 69 | capsule_dim_size = 8 70 | 71 | target_shape = input_shape 72 | 73 | input = Input(shape=input_shape) 74 | 75 | # TODO: try leaky relu next time 76 | conv_block_1 = convolution_block(input,kernel_size=16,filters=64) 77 | primary_cap = PrimaryCapsule(conv_block_1,dim_vector=capsule_dim_size,n_channels=primary_channel,kernel_size=9,strides=2,padding='valid') 78 | # Suppose this act like a max pooling 79 | routing_layer = CapsuleLayer(num_capsule=n_class,dim_vector=capsule_dim_size*2,num_routing=n_route,name='routing_layer_1')(primary_cap) 80 | output = Length(name='output')(routing_layer) 81 | 82 | y = Input(shape=(n_class,)) 83 | masked = Mask()([routing_layer,y]) 84 | 85 | x_recon = Dense(dense_size[0],activation='relu')(masked) 86 | 87 | for i in range(1,len(dense_size)): 88 | x_recon = Dense(dense_size[i],activation='relu')(x_recon) 89 | # Is there any other way to do 90 | x_recon = Dense(np.prod(target_shape),activation='relu')(x_recon) 91 | x_recon = Reshape(target_shape=target_shape,name='output_recon')(x_recon) 92 | 93 | # conv_block_2 = convolution_block(routing_layer) 94 | # b12_sum = add([conv_block_2,conv_block_1]) 95 | 96 | return Model([input,y],[output,x_recon]) 97 | 98 | def margin_loss(y_true, y_pred): 99 | """ 100 | Margin loss for Eq.(4). When y_true[i, :] contains not just one `1`, this loss should work too. Not test it. 101 | :param y_true: [None, n_classes] 102 | :param y_pred: [None, num_capsule] 103 | :return: a scalar loss value. 104 | """ 105 | L = y_true * K.square(K.maximum(0., 0.9 - y_pred)) + \ 106 | 0.5 * (1 - y_true) * K.square(K.maximum(0., y_pred - 0.1)) 107 | 108 | return K.mean(K.sum(L, 1)) 109 | 110 | def train(epochs=50,batch_size=64,mode=1): 111 | import numpy as np 112 | import os 113 | from keras import callbacks 114 | from keras.utils.vis_utils import plot_model 115 | if mode==1: 116 | num_classes = 10 117 | (x_train,y_train),(x_test,y_test) = load_cifar_10() 118 | else: 119 | num_classes = 100 120 | (x_train,y_train),(x_test,y_test) = load_cifar_100() 121 | model = CapsNetv1(input_shape=[32, 32, 3], 122 | n_class=num_classes, 123 | n_route=3) 124 | print('x_train shape:', x_train.shape) 125 | print(x_train.shape[0], 'train samples') 126 | print(x_test.shape[0], 'test samples') 127 | 128 | model.summary() 129 | log = callbacks.CSVLogger('networks/models/results/capsule-cifar-'+str(num_classes)+'-log.csv') 130 | tb = callbacks.TensorBoard(log_dir='networks/models/results/tensorboard-capsule-cifar-'+str(num_classes)+'-logs', 131 | batch_size=batch_size, histogram_freq=True) 132 | checkpoint = callbacks.ModelCheckpoint('networks/models/capsnet.h5', 133 | save_best_only=True, verbose=1) 134 | lr_decay = callbacks.LearningRateScheduler(schedule=lambda epoch: 0.001 * np.exp(-epoch / 10.)) 135 | 136 | # plot_model(model, to_file='models/capsule-cifar-'+str(num_classes)+'.png', show_shapes=True) 137 | 138 | model.compile(optimizer=optimizers.Adam(lr=0.001), 139 | loss=[margin_loss, 'mse'], 140 | loss_weights=[1., 0.1], 141 | metrics={'output_recon':'accuracy','output':'accuracy'}) 142 | from networks.capsulenet.helper_function import data_generator 143 | 144 | generator = data_generator(x_train,y_train,batch_size) 145 | # Image generator significantly increase the accuracy and reduce validation loss 146 | model.fit_generator(generator, 147 | steps_per_epoch=x_train.shape[0] // batch_size, 148 | validation_data=([x_test, y_test], [y_test, x_test]), 149 | epochs=epochs, verbose=1, max_q_size=100, 150 | callbacks=[log,tb,checkpoint,lr_decay]) 151 | 152 | return model 153 | 154 | def test(epoch, mode=1): 155 | import matplotlib.pyplot as plt 156 | from PIL import Image 157 | from networks.capsulenet.helper_function import combine_images 158 | 159 | if mode == 1: 160 | num_classes =10 161 | _,(x_test,y_test) = load_cifar_10() 162 | else: 163 | num_classes = 100 164 | _,(x_test,y_test) = load_cifar_100() 165 | 166 | model = CapsNetv2(input_shape=[32, 32, 3], 167 | n_class=num_classes, 168 | n_route=3) 169 | model.load_weights('weights/capsule_weights/capsule-cifar-'+str(num_classes)+'weights-{:02d}.h5'.format(epoch)) 170 | print("Weights loaded, start validation") 171 | # model.load_weights('weights/capsule-weights-{:02d}.h5'.format(epoch)) 172 | y_pred, x_recon = model.predict([x_test, y_test], batch_size=100) 173 | print('-'*50) 174 | # Test acc: 0.7307 175 | print('Test acc:', np.sum(np.argmax(y_pred, 1) == np.argmax(y_test, 1))/y_test.shape[0]) 176 | 177 | img = combine_images(np.concatenate([x_test[:50],x_recon[:50]])) 178 | image = img*255 179 | Image.fromarray(image.astype(np.uint8)).save("results/real_and_recon.png") 180 | print('Reconstructed images are saved to ./results/real_and_recon.png') 181 | print('-'*50) 182 | plt.imshow(plt.imread("results/real_and_recon.png", )) 183 | plt.show() 184 | -------------------------------------------------------------------------------- /networks/network_in_network.py: -------------------------------------------------------------------------------- 1 | import keras 2 | import numpy as np 3 | from keras.datasets import cifar10 4 | from keras.preprocessing.image import ImageDataGenerator 5 | from keras.models import Sequential, load_model 6 | from keras.layers import Dense, Dropout, Activation, Flatten 7 | from keras.layers import Conv2D, MaxPooling2D, GlobalAveragePooling2D, AveragePooling2D 8 | from keras.initializers import RandomNormal 9 | from keras.layers.normalization import BatchNormalization 10 | from keras import optimizers 11 | from keras.callbacks import LearningRateScheduler, TensorBoard, ModelCheckpoint 12 | 13 | from networks.train_plot import PlotLearning 14 | 15 | # Code taken from https://github.com/BIGBALLON/cifar-10-cnn 16 | class NetworkInNetwork: 17 | def __init__(self, epochs=200, batch_size=128, load_weights=True): 18 | self.name = 'net_in_net' 19 | self.model_filename = 'networks/models/net_in_net.h5' 20 | self.num_classes = 10 21 | self.input_shape = 32, 32, 3 22 | self.batch_size = batch_size 23 | self.epochs = epochs 24 | self.iterations = 391 25 | self.weight_decay = 0.0001 26 | self.dropout = 0.5 27 | self.log_filepath = r'networks/models/net_in_net/' 28 | 29 | if load_weights: 30 | try: 31 | self._model = load_model(self.model_filename) 32 | print('Successfully loaded', self.name) 33 | except (ImportError, ValueError, OSError): 34 | print('Failed to load', self.name) 35 | 36 | def count_params(self): 37 | return self._model.count_params() 38 | 39 | def color_preprocessing(self, x_train,x_test): 40 | x_train = x_train.astype('float32') 41 | x_test = x_test.astype('float32') 42 | mean = [125.307, 122.95, 113.865] 43 | std = [62.9932, 62.0887, 66.7048] 44 | for i in range(3): 45 | x_train[:,:,:,i] = (x_train[:,:,:,i] - mean[i]) / std[i] 46 | x_test[:,:,:,i] = (x_test[:,:,:,i] - mean[i]) / std[i] 47 | return x_train, x_test 48 | 49 | def scheduler(self, epoch): 50 | if epoch <= 60: 51 | return 0.05 52 | if epoch <= 120: 53 | return 0.01 54 | if epoch <= 160: 55 | return 0.002 56 | return 0.0004 57 | 58 | def build_model(self): 59 | model = Sequential() 60 | 61 | model.add(Conv2D(192, (5, 5), padding='same', kernel_regularizer=keras.regularizers.l2(self.weight_decay), kernel_initializer="he_normal", input_shape=self.input_shape)) 62 | model.add(BatchNormalization()) 63 | model.add(Activation('relu')) 64 | model.add(Conv2D(160, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(self.weight_decay), kernel_initializer="he_normal")) 65 | model.add(BatchNormalization()) 66 | model.add(Activation('relu')) 67 | model.add(Conv2D(96, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(self.weight_decay), kernel_initializer="he_normal")) 68 | model.add(BatchNormalization()) 69 | model.add(Activation('relu')) 70 | model.add(MaxPooling2D(pool_size=(3, 3),strides=(2,2),padding = 'same')) 71 | 72 | model.add(Dropout(self.dropout)) 73 | 74 | model.add(Conv2D(192, (5, 5), padding='same', kernel_regularizer=keras.regularizers.l2(self.weight_decay), kernel_initializer="he_normal")) 75 | model.add(BatchNormalization()) 76 | model.add(Activation('relu')) 77 | model.add(Conv2D(192, (1, 1),padding='same', kernel_regularizer=keras.regularizers.l2(self.weight_decay), kernel_initializer="he_normal")) 78 | model.add(BatchNormalization()) 79 | model.add(Activation('relu')) 80 | model.add(Conv2D(192, (1, 1),padding='same', kernel_regularizer=keras.regularizers.l2(self.weight_decay), kernel_initializer="he_normal")) 81 | model.add(BatchNormalization()) 82 | model.add(Activation('relu')) 83 | model.add(MaxPooling2D(pool_size=(3, 3),strides=(2,2),padding = 'same')) 84 | 85 | model.add(Dropout(self.dropout)) 86 | 87 | model.add(Conv2D(192, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(self.weight_decay), kernel_initializer="he_normal")) 88 | model.add(BatchNormalization()) 89 | model.add(Activation('relu')) 90 | model.add(Conv2D(192, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(self.weight_decay), kernel_initializer="he_normal")) 91 | model.add(BatchNormalization()) 92 | model.add(Activation('relu')) 93 | model.add(Conv2D(10, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(self.weight_decay), kernel_initializer="he_normal")) 94 | model.add(BatchNormalization()) 95 | model.add(Activation('relu')) 96 | 97 | model.add(GlobalAveragePooling2D()) 98 | model.add(Activation('softmax')) 99 | 100 | sgd = optimizers.SGD(lr=.1, momentum=0.9, nesterov=True) 101 | model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) 102 | return model 103 | 104 | def train(self): 105 | # load data 106 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 107 | y_train = keras.utils.to_categorical(y_train, self.num_classes) 108 | y_test = keras.utils.to_categorical(y_test, self.num_classes) 109 | 110 | x_train, x_test = self.color_preprocessing(x_train, x_test) 111 | 112 | # build network 113 | model = self.build_model() 114 | model.summary() 115 | 116 | # Save the best model during each training checkpoint 117 | checkpoint = ModelCheckpoint(self.model_filename, 118 | monitor='val_loss', 119 | verbose=0, 120 | save_best_only= True, 121 | mode='auto') 122 | plot_callback = PlotLearning() 123 | tb_cb = TensorBoard(log_dir=self.log_filepath, histogram_freq=0) 124 | 125 | cbks = [checkpoint, plot_callback, tb_cb] 126 | 127 | # set data augmentation 128 | print('Using real-time data augmentation.') 129 | datagen = ImageDataGenerator(horizontal_flip=True,width_shift_range=0.125,height_shift_range=0.125,fill_mode='constant',cval=0.) 130 | datagen.fit(x_train) 131 | 132 | # start training 133 | model.fit_generator(datagen.flow(x_train, y_train,batch_size=self.batch_size),steps_per_epoch=self.iterations,epochs=self.epochs,callbacks=cbks,validation_data=(x_test, y_test)) 134 | 135 | model.save(self.model_filename) 136 | 137 | self._model = model 138 | 139 | def color_process(self, imgs): 140 | if imgs.ndim < 4: 141 | imgs = np.array([imgs]) 142 | imgs = imgs.astype('float32') 143 | mean = [125.307, 122.95, 113.865] 144 | std = [62.9932, 62.0887, 66.7048] 145 | for img in imgs: 146 | for i in range(3): 147 | img[:,:,i] = (img[:,:,i] - mean[i]) / std[i] 148 | return imgs 149 | 150 | def predict(self, img): 151 | processed = self.color_process(img) 152 | return self._model.predict(processed, batch_size=self.batch_size) 153 | 154 | def predict_one(self, img): 155 | return self.predict(img)[0] 156 | 157 | def accuracy(self): 158 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 159 | y_train = keras.utils.to_categorical(y_train, self.num_classes) 160 | y_test = keras.utils.to_categorical(y_test, self.num_classes) 161 | 162 | # color preprocessing 163 | x_train, x_test = self.color_preprocessing(x_train, x_test) 164 | 165 | return self._model.evaluate(x_test, y_test, verbose=0)[1] 166 | -------------------------------------------------------------------------------- /networks/wide_resnet.py: -------------------------------------------------------------------------------- 1 | import keras 2 | import numpy as np 3 | from keras.datasets import cifar10 4 | from keras.preprocessing.image import ImageDataGenerator 5 | from keras.layers.normalization import BatchNormalization 6 | from keras.layers import Conv2D, Dense, Input, add, Activation, GlobalAveragePooling2D 7 | from keras.initializers import he_normal 8 | from keras.callbacks import LearningRateScheduler, TensorBoard, ModelCheckpoint 9 | from keras.models import Model, load_model 10 | from keras import optimizers 11 | from keras import regularizers 12 | 13 | from networks.train_plot import PlotLearning 14 | 15 | # Code taken from https://github.com/BIGBALLON/cifar-10-cnn 16 | class WideResNet: 17 | def __init__(self, epochs=200, batch_size=128, load_weights=True): 18 | self.name = 'wide_resnet' 19 | self.model_filename = 'networks/models/wide_resnet.h5' 20 | 21 | self.depth = 16 22 | self.wide = 8 23 | self.num_classes = 10 24 | self.img_rows, self.img_cols = 32, 32 25 | self.img_channels = 3 26 | self.batch_size = batch_size 27 | self.epochs = epochs 28 | self.iterations = 391 29 | self.weight_decay = 0.0005 30 | self.log_filepath = r'networks/models/wide_resnet/' 31 | 32 | if load_weights: 33 | try: 34 | self._model = load_model(self.model_filename) 35 | print('Successfully loaded', self.name) 36 | except (ImportError, ValueError, OSError): 37 | print('Failed to load', self.name) 38 | 39 | def count_params(self): 40 | return self._model.count_params() 41 | 42 | def scheduler(self, epoch): 43 | if epoch <= 60: 44 | return 0.1 45 | if epoch <= 120: 46 | return 0.02 47 | if epoch <= 160: 48 | return 0.004 49 | return 0.0008 50 | 51 | def color_preprocessing(self, x_train,x_test): 52 | x_train = x_train.astype('float32') 53 | x_test = x_test.astype('float32') 54 | mean = [125.307, 122.95, 113.865] 55 | std = [62.9932, 62.0887, 66.7048] 56 | for i in range(3): 57 | x_train[:,:,:,i] = (x_train[:,:,:,i] - mean[i]) / std[i] 58 | x_test[:,:,:,i] = (x_test[:,:,:,i] - mean[i]) / std[i] 59 | return x_train, x_test 60 | 61 | def wide_residual_network(self, img_input,classes_num,depth,k): 62 | 63 | print('Wide-Resnet %dx%d' %(depth, k)) 64 | n_filters = [16, 16*k, 32*k, 64*k] 65 | n_stack = (depth - 4) / 6 66 | in_filters = 16 67 | 68 | def conv3x3(x,filters): 69 | return Conv2D(filters=filters, kernel_size=(3,3), strides=(1,1), padding='same', 70 | kernel_initializer=he_normal(), 71 | kernel_regularizer=regularizers.l2(self.weight_decay))(x) 72 | 73 | def residual_block(x,out_filters,increase_filter=False): 74 | if increase_filter: 75 | first_stride = (2,2) 76 | else: 77 | first_stride = (1,1) 78 | pre_bn = BatchNormalization()(x) 79 | pre_relu = Activation('relu')(pre_bn) 80 | conv_1 = Conv2D(out_filters,kernel_size=(3,3),strides=first_stride,padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(self.weight_decay))(pre_relu) 81 | bn_1 = BatchNormalization()(conv_1) 82 | relu1 = Activation('relu')(bn_1) 83 | conv_2 = Conv2D(out_filters, kernel_size=(3,3), strides=(1,1), padding='same', kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(self.weight_decay))(relu1) 84 | if increase_filter or in_filters != out_filters: 85 | projection = Conv2D(out_filters,kernel_size=(1,1),strides=first_stride,padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(self.weight_decay))(x) 86 | block = add([conv_2, projection]) 87 | else: 88 | block = add([conv_2,x]) 89 | return block 90 | 91 | def wide_residual_layer(x,out_filters,increase_filter=False): 92 | x = residual_block(x,out_filters,increase_filter) 93 | in_filters = out_filters 94 | for _ in range(1,int(n_stack)): 95 | x = residual_block(x,out_filters) 96 | return x 97 | 98 | x = conv3x3(img_input,n_filters[0]) 99 | x = wide_residual_layer(x,n_filters[1]) 100 | x = wide_residual_layer(x,n_filters[2],increase_filter=True) 101 | x = wide_residual_layer(x,n_filters[3],increase_filter=True) 102 | x = BatchNormalization()(x) 103 | x = Activation('relu')(x) 104 | x = GlobalAveragePooling2D()(x) 105 | x = Dense(classes_num,activation='softmax',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(self.weight_decay))(x) 106 | return x 107 | 108 | def train(self): 109 | # load data 110 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 111 | y_train = keras.utils.to_categorical(y_train, self.num_classes) 112 | y_test = keras.utils.to_categorical(y_test, self.num_classes) 113 | 114 | # color preprocessing 115 | x_train, x_test = self.color_preprocessing(x_train, x_test) 116 | 117 | # build network 118 | img_input = Input(shape=(self.img_rows,self.img_cols,self.img_channels)) 119 | output = self.wide_residual_network(img_input,self.num_classes,self.depth,self.wide) 120 | resnet = Model(img_input, output) 121 | resnet.summary() 122 | 123 | # set optimizer 124 | sgd = optimizers.SGD(lr=.1, momentum=0.9, nesterov=True) 125 | resnet.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) 126 | 127 | # set callback 128 | tb_cb = TensorBoard(log_dir=self.log_filepath, histogram_freq=0) 129 | change_lr = LearningRateScheduler(self.scheduler) 130 | checkpoint = ModelCheckpoint(self.model_filename, 131 | monitor='val_loss', verbose=0, save_best_only= True, mode='auto') 132 | plot_callback = PlotLearning() 133 | cbks = [change_lr,tb_cb,checkpoint,plot_callback] 134 | 135 | # set data augmentation 136 | print('Using real-time data augmentation.') 137 | datagen = ImageDataGenerator(horizontal_flip=True, 138 | width_shift_range=0.125,height_shift_range=0.125,fill_mode='constant',cval=0.) 139 | 140 | datagen.fit(x_train) 141 | 142 | # start training 143 | resnet.fit_generator(datagen.flow(x_train, y_train,batch_size=self.batch_size), 144 | steps_per_epoch=self.iterations, 145 | epochs=self.epochs, 146 | callbacks=cbks, 147 | validation_data=(x_test, y_test)) 148 | resnet.save(self.model_filename) 149 | 150 | self._model = resnet 151 | self.param_count = self._model.count_params() 152 | 153 | def color_process(self, imgs): 154 | if imgs.ndim < 4: 155 | imgs = np.array([imgs]) 156 | imgs = imgs.astype('float32') 157 | mean = [125.307, 122.95, 113.865] 158 | std = [62.9932, 62.0887, 66.7048] 159 | for img in imgs: 160 | for i in range(3): 161 | img[:,:,i] = (img[:,:,i] - mean[i]) / std[i] 162 | return imgs 163 | 164 | def predict(self, img): 165 | processed = self.color_process(img) 166 | return self._model.predict(processed, batch_size=self.batch_size) 167 | 168 | def predict_one(self, img): 169 | return self.predict(img)[0] 170 | 171 | def accuracy(self): 172 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 173 | y_train = keras.utils.to_categorical(y_train, self.num_classes) 174 | y_test = keras.utils.to_categorical(y_test, self.num_classes) 175 | 176 | # color preprocessing 177 | x_train, x_test = self.color_preprocessing(x_train, x_test) 178 | 179 | return self._model.evaluate(x_test, y_test, verbose=0)[1] -------------------------------------------------------------------------------- /networks/resnet.py: -------------------------------------------------------------------------------- 1 | import keras 2 | import numpy as np 3 | from keras.datasets import cifar10 4 | from keras.preprocessing.image import ImageDataGenerator 5 | from keras.layers.normalization import BatchNormalization 6 | from keras.layers import Conv2D, Dense, Input, add, Activation, GlobalAveragePooling2D 7 | from keras.callbacks import LearningRateScheduler, TensorBoard, ModelCheckpoint 8 | from keras.models import Model, load_model 9 | from keras import optimizers, regularizers 10 | 11 | from networks.train_plot import PlotLearning 12 | 13 | # Code taken from https://github.com/BIGBALLON/cifar-10-cnn 14 | class ResNet: 15 | def __init__(self, epochs=200, batch_size=128, load_weights=True): 16 | self.name = 'resnet' 17 | self.model_filename = 'networks/models/resnet.h5' 18 | 19 | self.stack_n = 5 20 | self.num_classes = 10 21 | self.img_rows, self.img_cols = 32, 32 22 | self.img_channels = 3 23 | self.batch_size = batch_size 24 | self.epochs = epochs 25 | self.iterations = 50000 // self.batch_size 26 | self.weight_decay = 0.0001 27 | self.log_filepath = r'networks/models/resnet/' 28 | 29 | if load_weights: 30 | try: 31 | self._model = load_model(self.model_filename) 32 | print('Successfully loaded', self.name) 33 | except (ImportError, ValueError, OSError): 34 | print('Failed to load', self.name) 35 | 36 | def count_params(self): 37 | return self._model.count_params() 38 | 39 | def color_preprocessing(self, x_train,x_test): 40 | x_train = x_train.astype('float32') 41 | x_test = x_test.astype('float32') 42 | mean = [125.307, 122.95, 113.865] 43 | std = [62.9932, 62.0887, 66.7048] 44 | for i in range(3): 45 | x_train[:,:,:,i] = (x_train[:,:,:,i] - mean[i]) / std[i] 46 | x_test[:,:,:,i] = (x_test[:,:,:,i] - mean[i]) / std[i] 47 | return x_train, x_test 48 | 49 | def scheduler(self, epoch): 50 | if epoch < 80: 51 | return 0.1 52 | if epoch < 150: 53 | return 0.01 54 | return 0.001 55 | 56 | def residual_network(self, img_input,classes_num=10,stack_n=5): 57 | def residual_block(intput,out_channel,increase=False): 58 | if increase: 59 | stride = (2,2) 60 | else: 61 | stride = (1,1) 62 | 63 | pre_bn = BatchNormalization()(intput) 64 | pre_relu = Activation('relu')(pre_bn) 65 | 66 | conv_1 = Conv2D(out_channel,kernel_size=(3,3),strides=stride,padding='same', 67 | kernel_initializer="he_normal", 68 | kernel_regularizer=regularizers.l2(self.weight_decay))(pre_relu) 69 | bn_1 = BatchNormalization()(conv_1) 70 | relu1 = Activation('relu')(bn_1) 71 | conv_2 = Conv2D(out_channel,kernel_size=(3,3),strides=(1,1),padding='same', 72 | kernel_initializer="he_normal", 73 | kernel_regularizer=regularizers.l2(self.weight_decay))(relu1) 74 | if increase: 75 | projection = Conv2D(out_channel, 76 | kernel_size=(1,1), 77 | strides=(2,2), 78 | padding='same', 79 | kernel_initializer="he_normal", 80 | kernel_regularizer=regularizers.l2(self.weight_decay))(intput) 81 | block = add([conv_2, projection]) 82 | else: 83 | block = add([intput,conv_2]) 84 | return block 85 | 86 | # build model 87 | # total layers = stack_n * 3 * 2 + 2 88 | # stack_n = 5 by default, total layers = 32 89 | # input: 32x32x3 output: 32x32x16 90 | x = Conv2D(filters=16,kernel_size=(3,3),strides=(1,1),padding='same', 91 | kernel_initializer="he_normal", 92 | kernel_regularizer=regularizers.l2(self.weight_decay))(img_input) 93 | 94 | # input: 32x32x16 output: 32x32x16 95 | for _ in range(stack_n): 96 | x = residual_block(x,16,False) 97 | 98 | # input: 32x32x16 output: 16x16x32 99 | x = residual_block(x,32,True) 100 | for _ in range(1,stack_n): 101 | x = residual_block(x,32,False) 102 | 103 | # input: 16x16x32 output: 8x8x64 104 | x = residual_block(x,64,True) 105 | for _ in range(1,stack_n): 106 | x = residual_block(x,64,False) 107 | 108 | x = BatchNormalization()(x) 109 | x = Activation('relu')(x) 110 | x = GlobalAveragePooling2D()(x) 111 | 112 | # input: 64 output: 10 113 | x = Dense(classes_num,activation='softmax', 114 | kernel_initializer="he_normal", 115 | kernel_regularizer=regularizers.l2(self.weight_decay))(x) 116 | return x 117 | 118 | def train(self): 119 | # load data 120 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 121 | y_train = keras.utils.to_categorical(y_train, self.num_classes) 122 | y_test = keras.utils.to_categorical(y_test, self.num_classes) 123 | 124 | # color preprocessing 125 | x_train, x_test = self.color_preprocessing(x_train, x_test) 126 | 127 | # build network 128 | img_input = Input(shape=(self.img_rows,self.img_cols,self.img_channels)) 129 | output = self.residual_network(img_input,self.num_classes,self.stack_n) 130 | resnet = Model(img_input, output) 131 | resnet.summary() 132 | 133 | # set optimizer 134 | sgd = optimizers.SGD(lr=.1, momentum=0.9, nesterov=True) 135 | resnet.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) 136 | 137 | # set callback 138 | tb_cb = TensorBoard(log_dir=self.log_filepath, histogram_freq=0) 139 | change_lr = LearningRateScheduler(self.scheduler) 140 | checkpoint = ModelCheckpoint(self.model_filename, 141 | monitor='val_loss', verbose=0, save_best_only= True, mode='auto') 142 | plot_callback = PlotLearning() 143 | cbks = [change_lr,tb_cb,checkpoint,plot_callback] 144 | 145 | # set data augmentation 146 | print('Using real-time data augmentation.') 147 | datagen = ImageDataGenerator(horizontal_flip=True, 148 | width_shift_range=0.125, 149 | height_shift_range=0.125, 150 | fill_mode='constant',cval=0.) 151 | 152 | datagen.fit(x_train) 153 | 154 | # start training 155 | resnet.fit_generator(datagen.flow(x_train, y_train,batch_size=self.batch_size), 156 | steps_per_epoch=self.iterations, 157 | epochs=self.epochs, 158 | callbacks=cbks, 159 | validation_data=(x_test, y_test)) 160 | resnet.save(self.model_filename) 161 | 162 | self._model = resnet 163 | self.param_count = self._model.count_params() 164 | 165 | def color_process(self, imgs): 166 | if imgs.ndim < 4: 167 | imgs = np.array([imgs]) 168 | imgs = imgs.astype('float32') 169 | mean = [125.307, 122.95, 113.865] 170 | std = [62.9932, 62.0887, 66.7048] 171 | for img in imgs: 172 | for i in range(3): 173 | img[:,:,i] = (img[:,:,i] - mean[i]) / std[i] 174 | return imgs 175 | 176 | def predict(self, img): 177 | processed = self.color_process(img) 178 | return self._model.predict(processed, batch_size=self.batch_size) 179 | 180 | def predict_one(self, img): 181 | return self.predict(img)[0] 182 | 183 | def accuracy(self): 184 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 185 | y_train = keras.utils.to_categorical(y_train, self.num_classes) 186 | y_test = keras.utils.to_categorical(y_test, self.num_classes) 187 | 188 | # color preprocessing 189 | x_train, x_test = self.color_preprocessing(x_train, x_test) 190 | 191 | return self._model.evaluate(x_test, y_test, verbose=0)[1] -------------------------------------------------------------------------------- /helper.py: -------------------------------------------------------------------------------- 1 | # CIFAR - 10 2 | 3 | import pickle 4 | import numpy as np 5 | from keras.datasets import cifar10 6 | from keras.utils import np_utils 7 | from matplotlib import pyplot as plt 8 | import pandas as pd 9 | import requests 10 | from tqdm import tqdm 11 | 12 | 13 | def perturb_image(xs, img): 14 | # If this function is passed just one perturbation vector, 15 | # pack it in a list to keep the computation the same 16 | if xs.ndim < 2: 17 | xs = np.array([xs]) 18 | 19 | # Copy the image n == len(xs) times so that we can 20 | # create n new perturbed images 21 | tile = [len(xs)] + [1] * (xs.ndim + 1) 22 | imgs = np.tile(img, tile) 23 | 24 | # Make sure to floor the members of xs as int types 25 | xs = xs.astype(int) 26 | 27 | for x, img in zip(xs, imgs): 28 | # Split x into an array of 5-tuples (perturbation pixels) 29 | # i.e., [[x,y,r,g,b], ...] 30 | pixels = np.split(x, len(x) // 5) 31 | for pixel in pixels: 32 | # At each pixel's x,y position, assign its rgb value 33 | x_pos, y_pos, *rgb = pixel 34 | img[x_pos, y_pos] = rgb 35 | 36 | return imgs 37 | 38 | 39 | def plot_image(image, label_true=None, class_names=None, label_pred=None): 40 | if image.ndim == 4 and image.shape[0] == 1: 41 | image = image[0] 42 | 43 | plt.grid() 44 | plt.imshow(image.astype(np.uint8)) 45 | 46 | # Show true and predicted classes 47 | if label_true is not None and class_names is not None: 48 | labels_true_name = class_names[label_true] 49 | if label_pred is None: 50 | xlabel = "True: " + labels_true_name 51 | else: 52 | # Name of the predicted class 53 | labels_pred_name = class_names[label_pred] 54 | 55 | xlabel = "True: " + labels_true_name + "\nPredicted: " + labels_pred_name 56 | 57 | # Show the class on the x-axis 58 | plt.xlabel(xlabel) 59 | 60 | plt.xticks([]) # Remove ticks from the plot 61 | plt.yticks([]) 62 | plt.show() # Show the plot 63 | 64 | 65 | def plot_images(images, labels_true, class_names, labels_pred=None, 66 | confidence=None, titles=None): 67 | assert len(images) == len(labels_true) 68 | 69 | # Create a figure with sub-plots 70 | fig, axes = plt.subplots(3, 3, figsize=(10, 10)) 71 | 72 | # Adjust the vertical spacing 73 | hspace = 0.2 74 | if labels_pred is not None: 75 | hspace += 0.2 76 | if titles is not None: 77 | hspace += 0.2 78 | 79 | fig.subplots_adjust(hspace=hspace, wspace=0.0) 80 | 81 | for i, ax in enumerate(axes.flat): 82 | # Fix crash when less than 9 images 83 | if i < len(images): 84 | # Plot the image 85 | ax.imshow(images[i]) 86 | 87 | # Name of the true class 88 | labels_true_name = class_names[labels_true[i]] 89 | 90 | # Show true and predicted classes 91 | if labels_pred is None: 92 | xlabel = "True: " + labels_true_name 93 | else: 94 | # Name of the predicted class 95 | labels_pred_name = class_names[labels_pred[i]] 96 | 97 | xlabel = "True: " + labels_true_name + "\nPred: " + labels_pred_name 98 | if (confidence is not None): 99 | xlabel += " (" + "{0:.1f}".format(confidence[i] * 100) + "%)" 100 | 101 | # Show the class on the x-axis 102 | ax.set_xlabel(xlabel) 103 | 104 | if titles is not None: 105 | ax.set_title(titles[i]) 106 | 107 | # Remove ticks from the plot 108 | ax.set_xticks([]) 109 | ax.set_yticks([]) 110 | 111 | # Show the plot 112 | plt.show() 113 | 114 | 115 | def plot_model(model_details): 116 | # Create sub-plots 117 | fig, axs = plt.subplots(1, 2, figsize=(15, 5)) 118 | 119 | # Summarize history for accuracy 120 | axs[0].plot(range(1, len(model_details.history['acc']) + 1), model_details.history['acc']) 121 | axs[0].plot(range(1, len(model_details.history['val_acc']) + 1), model_details.history['val_acc']) 122 | axs[0].set_title('Model Accuracy') 123 | axs[0].set_ylabel('Accuracy') 124 | axs[0].set_xlabel('Epoch') 125 | axs[0].set_xticks(np.arange(1, len(model_details.history['acc']) + 1), len(model_details.history['acc']) / 10) 126 | axs[0].legend(['train', 'val'], loc='best') 127 | 128 | # Summarize history for loss 129 | axs[1].plot(range(1, len(model_details.history['loss']) + 1), model_details.history['loss']) 130 | axs[1].plot(range(1, len(model_details.history['val_loss']) + 1), model_details.history['val_loss']) 131 | axs[1].set_title('Model Loss') 132 | axs[1].set_ylabel('Loss') 133 | axs[1].set_xlabel('Epoch') 134 | axs[1].set_xticks(np.arange(1, len(model_details.history['loss']) + 1), len(model_details.history['loss']) / 10) 135 | axs[1].legend(['train', 'val'], loc='best') 136 | 137 | # Show the plot 138 | plt.show() 139 | 140 | 141 | def visualize_attack(df, class_names): 142 | _, (x_test, _) = cifar10.load_data() 143 | 144 | results = df[df.success].sample(9) 145 | 146 | z = zip(results.perturbation, x_test[results.image]) 147 | images = np.array([perturb_image(p, img)[0] 148 | for p, img in z]) 149 | 150 | labels_true = np.array(results.true) 151 | labels_pred = np.array(results.predicted) 152 | titles = np.array(results.model) 153 | 154 | # Plot the first 9 images. 155 | plot_images(images=images, 156 | labels_true=labels_true, 157 | class_names=class_names, 158 | labels_pred=labels_pred, 159 | titles=titles) 160 | 161 | 162 | def attack_stats(df, models, network_stats): 163 | stats = [] 164 | for model in models: 165 | val_accuracy = np.array(network_stats[network_stats.name == model.name].accuracy)[0] 166 | m_result = df[df.model == model.name] 167 | pixels = list(set(m_result.pixels)) 168 | 169 | for pixel in pixels: 170 | p_result = m_result[m_result.pixels == pixel] 171 | success_rate = len(p_result[p_result.success]) / len(p_result) 172 | stats.append([model.name, val_accuracy, pixel, success_rate]) 173 | 174 | return pd.DataFrame(stats, columns=['model', 'accuracy', 'pixels', 'attack_success_rate']) 175 | 176 | 177 | def evaluate_models(models, x_test, y_test): 178 | correct_imgs = [] 179 | network_stats = [] 180 | for model in models: 181 | print('Evaluating', model.name) 182 | 183 | predictions = model.predict(x_test) 184 | 185 | correct = [[model.name, i, label, np.max(pred), pred] 186 | for i, (label, pred) 187 | in enumerate(zip(y_test[:, 0], predictions)) 188 | if label == np.argmax(pred)] 189 | accuracy = len(correct) / len(x_test) 190 | 191 | correct_imgs += correct 192 | network_stats += [[model.name, accuracy, model.count_params()]] 193 | return network_stats, correct_imgs 194 | 195 | 196 | def load_results(): 197 | with open('networks/results/untargeted_results.pkl', 'rb') as file: 198 | untargeted = pickle.load(file) 199 | with open('networks/results/targeted_results.pkl', 'rb') as file: 200 | targeted = pickle.load(file) 201 | return untargeted, targeted 202 | 203 | 204 | def checkpoint(results, targeted=False): 205 | filename = 'targeted' if targeted else 'untargeted' 206 | 207 | with open('networks/results/' + filename + '_results.pkl', 'wb') as file: 208 | pickle.dump(results, file) 209 | 210 | 211 | def download_from_url(url, dst): 212 | """ 213 | @param: url to download file 214 | @param: dst place to put the file 215 | """ 216 | # Streaming, so we can iterate over the response. 217 | r = requests.get(url, stream=True) 218 | 219 | with open(dst, 'wb') as f: 220 | for data in tqdm(r.iter_content(), unit='B', unit_scale=True): 221 | f.write(data) 222 | 223 | # def load_imagenet(): 224 | # with open('data/imagenet_class_index.json', 'r') as f: 225 | # class_names = json.load(f) 226 | # class_names = pd.DataFrame([[i,wid,name] for i,(wid,name) in class_names.items()], columns=['id', 'wid', 'text']) 227 | 228 | # wid_to_id = {wid:int(i) for i,wid in class_names[['id', 'wid']].as_matrix()} 229 | 230 | # imagenet_urls = pd.read_csv('data/imagenet_urls.txt', delimiter='\t', names=['label', 'url']) 231 | # imagenet_urls['label'], imagenet_urls['id'] = zip(*imagenet_urls.label.apply(lambda x: x.split('_'))) 232 | # imagenet_urls.label = imagenet_urls.label.apply(lambda wid: wid_to_id[wid]) 233 | -------------------------------------------------------------------------------- /networks/densenet.py: -------------------------------------------------------------------------------- 1 | import keras 2 | import math 3 | import numpy as np 4 | from keras.datasets import cifar10 5 | from keras.preprocessing.image import ImageDataGenerator 6 | from keras.layers.normalization import BatchNormalization 7 | from keras.layers import Conv2D, Dense, Input, add, Activation, AveragePooling2D, GlobalAveragePooling2D 8 | from keras.layers import Lambda, concatenate 9 | from keras.initializers import he_normal 10 | from keras.layers.merge import Concatenate 11 | from keras.callbacks import LearningRateScheduler, TensorBoard, ModelCheckpoint 12 | from keras.models import Model, load_model 13 | from keras import optimizers 14 | from keras import regularizers 15 | from keras.utils import plot_model 16 | 17 | from networks.train_plot import PlotLearning 18 | 19 | # Code taken from https://github.com/BIGBALLON/cifar-10-cnn 20 | class DenseNet: 21 | def __init__(self, epochs=250, batch_size=64, load_weights=True): 22 | self.name = 'densenet' 23 | self.model_filename = 'networks/models/densenet.h5' 24 | self.growth_rate = 12 25 | self.depth = 100 26 | self.compression = 0.5 27 | self.num_classes = 10 28 | self.img_rows, self.img_cols = 32, 32 29 | self.img_channels = 3 30 | self.batch_size = batch_size 31 | self.epochs = epochs 32 | self.iterations = 782 33 | self.weight_decay = 0.0001 34 | self.log_filepath = r'networks/models/densenet/' 35 | 36 | if load_weights: 37 | try: 38 | self._model = load_model(self.model_filename) 39 | print('Successfully loaded', self.name) 40 | except (ImportError, ValueError, OSError): 41 | print('Failed to load', self.name) 42 | 43 | def count_params(self): 44 | return self._model.count_params() 45 | 46 | def color_preprocessing(self, x_train,x_test): 47 | x_train = x_train.astype('float32') 48 | x_test = x_test.astype('float32') 49 | mean = [125.307, 122.95, 113.865] 50 | std = [62.9932, 62.0887, 66.7048] 51 | for i in range(3): 52 | x_train[:,:,:,i] = (x_train[:,:,:,i] - mean[i]) / std[i] 53 | x_test[:,:,:,i] = (x_test[:,:,:,i] - mean[i]) / std[i] 54 | return x_train, x_test 55 | 56 | def scheduler(self, epoch): 57 | if epoch <= 75: 58 | return 0.1 59 | if epoch <= 150: 60 | return 0.01 61 | if epoch <= 210: 62 | return 0.001 63 | return 0.0005 64 | 65 | def densenet(self, img_input,classes_num): 66 | 67 | def bn_relu(x): 68 | x = BatchNormalization()(x) 69 | x = Activation('relu')(x) 70 | return x 71 | 72 | def bottleneck(x): 73 | channels = self.growth_rate * 4 74 | x = bn_relu(x) 75 | x = Conv2D(channels,kernel_size=(1,1),strides=(1,1),padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(self.weight_decay),use_bias=False)(x) 76 | x = bn_relu(x) 77 | x = Conv2D(self.growth_rate,kernel_size=(3,3),strides=(1,1),padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(self.weight_decay),use_bias=False)(x) 78 | return x 79 | 80 | def single(x): 81 | x = bn_relu(x) 82 | x = Conv2D(self.growth_rate,kernel_size=(3,3),strides=(1,1),padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(self.weight_decay),use_bias=False)(x) 83 | return x 84 | 85 | def transition(x, inchannels): 86 | outchannels = int(inchannels * self.compression) 87 | x = bn_relu(x) 88 | x = Conv2D(outchannels,kernel_size=(1,1),strides=(1,1),padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(self.weight_decay),use_bias=False)(x) 89 | x = AveragePooling2D((2,2), strides=(2, 2))(x) 90 | return x, outchannels 91 | 92 | def dense_block(x,blocks,nchannels): 93 | concat = x 94 | for i in range(blocks): 95 | x = bottleneck(concat) 96 | concat = concatenate([x,concat], axis=-1) 97 | nchannels += self.growth_rate 98 | return concat, nchannels 99 | 100 | def dense_layer(x): 101 | return Dense(classes_num,activation='softmax',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(self.weight_decay))(x) 102 | 103 | 104 | nblocks = (self.depth - 4) // 6 105 | nchannels = self.growth_rate * 2 106 | 107 | x = Conv2D(nchannels,kernel_size=(3,3),strides=(1,1),padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(self.weight_decay),use_bias=False)(img_input) 108 | 109 | x, nchannels = dense_block(x,nblocks,nchannels) 110 | x, nchannels = transition(x,nchannels) 111 | x, nchannels = dense_block(x,nblocks,nchannels) 112 | x, nchannels = transition(x,nchannels) 113 | x, nchannels = dense_block(x,nblocks,nchannels) 114 | x, nchannels = transition(x,nchannels) 115 | x = bn_relu(x) 116 | x = GlobalAveragePooling2D()(x) 117 | x = dense_layer(x) 118 | return x 119 | 120 | def train(self): 121 | # load data 122 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 123 | y_train = keras.utils.to_categorical(y_train, self.num_classes) 124 | y_test = keras.utils.to_categorical(y_test, self.num_classes) 125 | x_train = x_train.astype('float32') 126 | x_test = x_test.astype('float32') 127 | 128 | # color preprocessing 129 | x_train, x_test = self.color_preprocessing(x_train, x_test) 130 | 131 | # build network 132 | img_input = Input(shape=(self.img_rows,self.img_cols,self.img_channels)) 133 | output = self.densenet(img_input,self.num_classes) 134 | model = Model(img_input, output) 135 | model.summary() 136 | 137 | # plot_model(model, show_shapes=True, to_file='model.png') 138 | 139 | # set optimizer 140 | sgd = optimizers.SGD(lr=.1, momentum=0.9, nesterov=True) 141 | model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) 142 | 143 | # set callback 144 | tb_cb = TensorBoard(log_dir=self.log_filepath, histogram_freq=0) 145 | change_lr = LearningRateScheduler(self.scheduler) 146 | ckpt = ModelCheckpoint(self.model_filename, monitor='val_loss', verbose=0, save_best_only= True, mode='auto') 147 | plot_callback = PlotLearning() 148 | cbks = [change_lr,tb_cb,ckpt, plot_callback] 149 | 150 | # set data augmentation 151 | print('Using real-time data augmentation.') 152 | datagen = ImageDataGenerator(horizontal_flip=True,width_shift_range=0.125,height_shift_range=0.125,fill_mode='constant',cval=0.) 153 | 154 | datagen.fit(x_train) 155 | 156 | # start training 157 | model.fit_generator(datagen.flow(x_train, y_train,batch_size=self.batch_size), steps_per_epoch=self.iterations, epochs=self.epochs, callbacks=cbks,validation_data=(x_test, y_test)) 158 | model.save(self.model_filename) 159 | 160 | self._model = model 161 | self.param_count = self._model.count_params() 162 | 163 | def color_process(self, imgs): 164 | if imgs.ndim < 4: 165 | imgs = np.array([imgs]) 166 | imgs = imgs.astype('float32') 167 | mean = [125.307, 122.95, 113.865] 168 | std = [62.9932, 62.0887, 66.7048] 169 | for img in imgs: 170 | for i in range(3): 171 | img[:,:,i] = (img[:,:,i] - mean[i]) / std[i] 172 | return imgs 173 | 174 | def predict(self, img): 175 | processed = self.color_process(img) 176 | return self._model.predict(processed, batch_size=self.batch_size) 177 | 178 | def predict_one(self, img): 179 | return self.predict(img)[0] 180 | 181 | def accuracy(self): 182 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 183 | y_train = keras.utils.to_categorical(y_train, self.num_classes) 184 | y_test = keras.utils.to_categorical(y_test, self.num_classes) 185 | 186 | # color preprocessing 187 | x_train, x_test = self.color_preprocessing(x_train, x_test) 188 | 189 | return self._model.evaluate(x_test, y_test, verbose=0)[1] -------------------------------------------------------------------------------- /networks/capsulenet/capsule_layers.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Added minor tweak to allow user define clip value in Mask layer, other are same as XifengGuo repo code 4 | 5 | Some key layers used for constructing a Capsule Network. These layers can used to construct CapsNet on other dataset, 6 | not just on MNIST. 7 | *NOTE*: some functions can be implemented in multiple ways, I keep all of them. You can try them for yourself just by 8 | uncommenting them and commenting their counterparts. 9 | Author: Xifeng Guo, E-mail: `guoxifeng1990@163.com`, Github: `https://github.com/XifengGuo/CapsNet-Keras` 10 | """ 11 | 12 | import keras.backend as K 13 | import tensorflow as tf 14 | from keras import initializers, layers 15 | 16 | 17 | class Length(layers.Layer): 18 | """ 19 | Compute the length of vectors. This is used to compute a Tensor that has the same shape with y_true in margin_loss 20 | inputs: shape=[dim_1, ..., dim_{n-1}, dim_n] 21 | output: shape=[dim_1, ..., dim_{n-1}] 22 | """ 23 | def call(self, inputs, **kwargs): 24 | return K.sqrt(K.sum(K.square(inputs), -1)) 25 | 26 | def compute_output_shape(self, input_shape): 27 | return input_shape[:-1] 28 | 29 | 30 | class Mask(layers.Layer): 31 | """ 32 | Mask Tensor layer by the max value in first axis 33 | Input shape: [None,d1,d2] 34 | Output shape: [None,d2] 35 | """ 36 | clip_value = (0,1) 37 | 38 | def Mask(self,clip_value=(0,1),**kwargs): 39 | self.clip_value = clip_value # what if clip value is not 0 and 1? 40 | 41 | def call(self,inputs,**kwargs): 42 | if type(inputs) is list: 43 | assert len(inputs) == 2 44 | inputs,mask = inputs 45 | else: 46 | x = inputs 47 | # enlarge range of values in x by mapping max(new_x) = 1, others 48 | x = (x - K.max(x,1,True)) / K.epsilon() + 1 49 | mask = K.clip(x,self.clip_value[0],self.clip_value[1]) # clip value beween 0 and 1 50 | masked_input = K.batch_dot(inputs, mask, [1,1]) 51 | return masked_input 52 | 53 | def compute_output_shape(self, input_shape): 54 | if type(input_shape[0]) is tuple: 55 | return tuple([None,input_shape[0][-1]]) 56 | 57 | else: 58 | return tuple([None, input_shape[-1]]) 59 | 60 | def squash(vector, axis=-1): 61 | """ 62 | The non-linear activation used in Capsule. It drives the length of a large vector to near 1 and small vector to 0 63 | :param vectors: some vectors to be squashed, N-dim tensor 64 | :param axis: the axis to squash 65 | :return: a Tensor with same shape as input vectors 66 | """ 67 | s_squared_norm = K.sum(K.square(vector), axis, keepdims=True) 68 | scale = s_squared_norm/(1+s_squared_norm)/K.sqrt(s_squared_norm) 69 | return scale*vector 70 | 71 | class CapsuleLayer(layers.Layer): 72 | """ 73 | The capsule layer. It is similar to Dense layer. Dense layer has `in_num` inputs, each is a scalar, the output of the 74 | neuron from the former layer, and it has `out_num` output neurons. CapsuleLayer just expand the output of the neuron 75 | from scalar to vector. So its input shape = [None, input_num_capsule, input_dim_vector] and output shape = \ 76 | [None, num_capsule, dim_vector]. For Dense Layer, input_dim_vector = dim_vector = 1. 77 | 78 | :param num_capsule: number of capsules in this layer 79 | :param dim_vector: dimension of the output vectors of the capsules in this layer 80 | :param num_routings: number of iterations for the routing algorithm 81 | """ 82 | def __init__(self, num_capsule, dim_vector, num_routing=3, 83 | kernel_initializer='glorot_uniform', 84 | bias_initializer='zeros', 85 | **kwargs): 86 | super(CapsuleLayer, self).__init__(**kwargs) 87 | self.num_capsule = num_capsule 88 | self.dim_vector = dim_vector 89 | self.num_routing = num_routing 90 | self.kernel_initializer = initializers.get(kernel_initializer) 91 | self.bias_initializer = initializers.get(bias_initializer) 92 | 93 | def build(self,input_shape): 94 | assert len(input_shape) >= 3, "Input tensor must have shape=[None, input_num_capsule,input_dim_vector]" 95 | self.input_num_capsule = input_shape[1] 96 | self.input_dim_vector = input_shape[2] 97 | 98 | self.W = self.add_weight(shape=[self.input_num_capsule,self.num_capsule,self.input_dim_vector,self.dim_vector], 99 | initializer=self.kernel_initializer, 100 | name='W') 101 | self.bias = self.add_weight(shape=[1,self.input_num_capsule,self.num_capsule,1,1], 102 | initializer=self.bias_initializer, 103 | name='bias',trainable=False) 104 | self.built = True 105 | 106 | def call(self,inputs,training=None): 107 | # inputs.shape=[None, input_num_capsule, input_dim_vector] 108 | # Expand dims to [None, input_num_capsule, 1, 1, input_dim_vector] 109 | inputs_expand = K.expand_dims(K.expand_dims(inputs, 2), 2) 110 | 111 | # Replicate num_capsule dimension to prepare being multiplied by W 112 | # Now it has shape = [None, input_num_capsule, num_capsule, 1, input_dim_vector] 113 | inputs_tiled = K.tile(inputs_expand, [1, 1, self.num_capsule, 1, 1]) 114 | 115 | """ 116 | # Compute `inputs * W` by expanding the first dim of W. More time-consuming and need batch_size. 117 | # Now W has shape = [batch_size, input_num_capsule, num_capsule, input_dim_vector, dim_vector] 118 | w_tiled = K.tile(K.expand_dims(self.W, 0), [self.batch_size, 1, 1, 1, 1]) 119 | 120 | # Transformed vectors, inputs_hat.shape = [None, input_num_capsule, num_capsule, 1, dim_vector] 121 | inputs_hat = K.batch_dot(inputs_tiled, w_tiled, [4, 3]) 122 | """ 123 | # Compute `inputs * W` by scanning inputs_tiled on dimension 0. This is faster but requires Tensorflow. 124 | # inputs_hat.shape = [None, input_num_capsule, num_capsule, 1, dim_vector] 125 | inputs_hat = tf.scan(lambda ac, x: K.batch_dot(x, self.W, [3, 2]), 126 | elems=inputs_tiled, 127 | initializer=K.zeros([self.input_num_capsule, self.num_capsule, 1, self.dim_vector])) 128 | """ 129 | # Routing algorithm V1. Use tf.while_loop in a dynamic way. 130 | def body(i, b, outputs): 131 | c = tf.nn.softmax(self.bias, dim=2) # dim=2 is the num_capsule dimension 132 | outputs = squash(K.sum(c * inputs_hat, 1, keepdims=True)) 133 | b = b + K.sum(inputs_hat * outputs, -1, keepdims=True) 134 | return [i-1, b, outputs] 135 | cond = lambda i, b, inputs_hat: i > 0 136 | loop_vars = [K.constant(self.num_routing), self.bias, K.sum(inputs_hat, 1, keepdims=True)] 137 | _, _, outputs = tf.while_loop(cond, body, loop_vars) 138 | """ 139 | # Routing algorithm V2. Use iteration. V2 and V1 both work without much difference on performance 140 | 141 | assert self.num_routing > 0, 'The num_routing should be > 0.' 142 | for i in range(self.num_routing): 143 | c = tf.nn.softmax(self.bias, dim=2) # dim=2 is the num_capsule dimension 144 | # outputs.shape=[None, 1, num_capsule, 1, dim_vector] 145 | outputs = squash(K.sum(c * inputs_hat, 1, keepdims=True)) 146 | 147 | # last iteration needs not compute bias which will not be passed to the graph any more anyway. 148 | if i != self.num_routing - 1: 149 | # self.bias = K.update_add(self.bias, K.sum(inputs_hat * outputs, [0, -1], keepdims=True)) 150 | self.bias += K.sum(inputs_hat * outputs, -1, keepdims=True) 151 | # tf.summary.histogram('BigBee', self.bias) # for debugging 152 | 153 | return K.reshape(outputs, [-1, self.num_capsule, self.dim_vector]) 154 | 155 | def compute_output_shape(self, input_shape): 156 | return tuple([None, self.num_capsule, self.dim_vector]) 157 | 158 | def PrimaryCapsule(inputs, dim_vector, n_channels, kernel_size, strides, padding): 159 | """ 160 | Apply Conv2D `n_channels` times and concatenate all capsules 161 | :param inputs: 4D tensor, shape=[None, width, height, channels] 162 | :param dim_vector: the dim of the output vector of capsule 163 | :param n_channels: the number of types of capsules 164 | :return: output tensor, shape=[None, num_capsule, dim_vector] 165 | """ 166 | output = layers.Conv2D(filters=dim_vector*n_channels, kernel_size=kernel_size, strides=strides, padding=padding)(inputs) 167 | outputs = layers.Reshape(target_shape=[-1, dim_vector])(output) 168 | return layers.Lambda(squash)(outputs) -------------------------------------------------------------------------------- /attack.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import argparse 4 | 5 | import numpy as np 6 | import pandas as pd 7 | from keras.datasets import cifar10 8 | import pickle 9 | 10 | # Custom Networks 11 | from networks.lenet import LeNet 12 | from networks.pure_cnn import PureCnn 13 | from networks.network_in_network import NetworkInNetwork 14 | from networks.resnet import ResNet 15 | from networks.densenet import DenseNet 16 | from networks.wide_resnet import WideResNet 17 | from networks.capsnet import CapsNet 18 | 19 | # Helper functions 20 | from differential_evolution import differential_evolution 21 | import helper 22 | 23 | 24 | class PixelAttacker: 25 | def __init__(self, models, data, class_names, dimensions=(32, 32)): 26 | # Load data and model 27 | self.models = models 28 | self.x_test, self.y_test = data 29 | self.class_names = class_names 30 | self.dimensions = dimensions 31 | 32 | network_stats, correct_imgs = helper.evaluate_models(self.models, self.x_test, self.y_test) 33 | self.correct_imgs = pd.DataFrame(correct_imgs, columns=['name', 'img', 'label', 'confidence', 'pred']) 34 | self.network_stats = pd.DataFrame(network_stats, columns=['name', 'accuracy', 'param_count']) 35 | 36 | def predict_classes(self, xs, img, target_class, model, minimize=True): 37 | # Perturb the image with the given pixel(s) x and get the prediction of the model 38 | imgs_perturbed = helper.perturb_image(xs, img) 39 | predictions = model.predict(imgs_perturbed)[:, target_class] 40 | # This function should always be minimized, so return its complement if needed 41 | return predictions if minimize else 1 - predictions 42 | 43 | def attack_success(self, x, img, target_class, model, targeted_attack=False, verbose=False): 44 | # Perturb the image with the given pixel(s) and get the prediction of the model 45 | attack_image = helper.perturb_image(x, img) 46 | 47 | confidence = model.predict(attack_image)[0] 48 | predicted_class = np.argmax(confidence) 49 | 50 | # If the prediction is what we want (misclassification or 51 | # targeted classification), return True 52 | if verbose: 53 | print('Confidence:', confidence[target_class]) 54 | if ((targeted_attack and predicted_class == target_class) or 55 | (not targeted_attack and predicted_class != target_class)): 56 | return True 57 | 58 | def attack(self, img_id, model, target=None, pixel_count=1, 59 | maxiter=75, popsize=400, verbose=False, plot=False): 60 | # Change the target class based on whether this is a targeted attack or not 61 | targeted_attack = target is not None 62 | target_class = target if targeted_attack else self.y_test[img_id, 0] 63 | 64 | # Define bounds for a flat vector of x,y,r,g,b values 65 | # For more pixels, repeat this layout 66 | dim_x, dim_y = self.dimensions 67 | bounds = [(0, dim_x), (0, dim_y), (0, 256), (0, 256), (0, 256)] * pixel_count 68 | 69 | # Population multiplier, in terms of the size of the perturbation vector x 70 | popmul = max(1, popsize // len(bounds)) 71 | 72 | # Format the predict/callback functions for the differential evolution algorithm 73 | def predict_fn(xs): 74 | return self.predict_classes(xs, self.x_test[img_id], target_class, model, target is None) 75 | 76 | def callback_fn(x, convergence): 77 | return self.attack_success(x, self.x_test[img_id], target_class, model, targeted_attack, verbose) 78 | 79 | # Call Scipy's Implementation of Differential Evolution 80 | attack_result = differential_evolution( 81 | predict_fn, bounds, maxiter=maxiter, popsize=popmul, 82 | recombination=1, atol=-1, callback=callback_fn, polish=False) 83 | 84 | # Calculate some useful statistics to return from this function 85 | attack_image = helper.perturb_image(attack_result.x, self.x_test[img_id])[0] 86 | prior_probs = model.predict(np.array([self.x_test[img_id]]))[0] 87 | predicted_probs = model.predict(np.array([attack_image]))[0] 88 | predicted_class = np.argmax(predicted_probs) 89 | actual_class = self.y_test[img_id, 0] 90 | success = predicted_class != actual_class 91 | cdiff = prior_probs[actual_class] - predicted_probs[actual_class] 92 | 93 | # Show the best attempt at a solution (successful or not) 94 | if plot: 95 | helper.plot_image(attack_image, actual_class, self.class_names, predicted_class) 96 | 97 | return [model.name, pixel_count, img_id, actual_class, predicted_class, success, cdiff, prior_probs, 98 | predicted_probs, attack_result.x] 99 | 100 | def attack_all(self, models, samples=500, pixels=(1, 3, 5), targeted=False, 101 | maxiter=75, popsize=400, verbose=False): 102 | results = [] 103 | for model in models: 104 | model_results = [] 105 | valid_imgs = self.correct_imgs[self.correct_imgs.name == model.name].img 106 | img_samples = np.random.choice(valid_imgs, samples) 107 | 108 | for pixel_count in pixels: 109 | for i, img in enumerate(img_samples): 110 | print(model.name, '- image', img, '-', i + 1, '/', len(img_samples)) 111 | targets = [None] if not targeted else range(10) 112 | 113 | for target in targets: 114 | if targeted: 115 | print('Attacking with target', self.class_names[target]) 116 | if target == self.y_test[img, 0]: 117 | continue 118 | result = self.attack(img, model, target, pixel_count, 119 | maxiter=maxiter, popsize=popsize, 120 | verbose=verbose) 121 | model_results.append(result) 122 | 123 | results += model_results 124 | helper.checkpoint(results, targeted) 125 | return results 126 | 127 | 128 | if __name__ == '__main__': 129 | model_defs = { 130 | 'lenet': LeNet, 131 | 'pure_cnn': PureCnn, 132 | 'net_in_net': NetworkInNetwork, 133 | 'resnet': ResNet, 134 | 'densenet': DenseNet, 135 | 'wide_resnet': WideResNet, 136 | 'capsnet': CapsNet 137 | } 138 | 139 | parser = argparse.ArgumentParser(description='Attack models on Cifar10') 140 | parser.add_argument('--model', nargs='+', choices=model_defs.keys(), default=model_defs.keys(), 141 | help='Specify one or more models by name to evaluate.') 142 | parser.add_argument('--pixels', nargs='+', default=(1, 3, 5), type=int, 143 | help='The number of pixels that can be perturbed.') 144 | parser.add_argument('--maxiter', default=75, type=int, 145 | help='The maximum number of iterations in the differential evolution algorithm before giving up and failing the attack.') 146 | parser.add_argument('--popsize', default=400, type=int, 147 | help='The number of adversarial images generated each iteration in the differential evolution algorithm. Increasing this number requires more computation.') 148 | parser.add_argument('--samples', default=500, type=int, 149 | help='The number of image samples to attack. Images are sampled randomly from the dataset.') 150 | parser.add_argument('--targeted', action='store_true', help='Set this switch to test for targeted attacks.') 151 | parser.add_argument('--save', default='networks/results/results.pkl', help='Save location for the results (pickle)') 152 | parser.add_argument('--verbose', action='store_true', help='Print out additional information every iteration.') 153 | 154 | args = parser.parse_args() 155 | 156 | # Load data and model 157 | _, test = cifar10.load_data() 158 | class_names = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'] 159 | models = [model_defs[m](load_weights=True) for m in args.model] 160 | 161 | attacker = PixelAttacker(models, test, class_names) 162 | 163 | print('Starting attack') 164 | 165 | results = attacker.attack_all(models, samples=args.samples, pixels=args.pixels, targeted=args.targeted, 166 | maxiter=args.maxiter, popsize=args.popsize, verbose=args.verbose) 167 | 168 | columns = ['model', 'pixels', 'image', 'true', 'predicted', 'success', 'cdiff', 'prior_probs', 'predicted_probs', 169 | 'perturbation'] 170 | results_table = pd.DataFrame(results, columns=columns) 171 | 172 | print(results_table[['model', 'pixels', 'image', 'true', 'predicted', 'success']]) 173 | 174 | print('Saving to', args.save) 175 | with open(args.save, 'wb') as file: 176 | pickle.dump(results, file) 177 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # One Pixel Attack 2 | 3 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/hyperparticle/one-pixel-attack-keras/blob/master/1_one-pixel-attack-cifar10.ipynb) [![Contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg)](CONTRIBUTING.md) [![MIT License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) 4 | 5 | [![Who would win?](images/who-would-win.jpg "one thicc boi that's who")](https://www.reddit.com/r/ProgrammerHumor/comments/79g0m6/one_pixel_attack_for_fooling_deep_neural_networks/?ref=share&ref_source=link) 6 | 7 | How simple is it to cause a deep neural network to misclassify an image if an attacker is only allowed to modify the color of one pixel and only see the prediction probability? Turns out it is very simple. In many cases, an attacker can even cause the network to return any answer they want. 8 | 9 | The following project is a Keras reimplementation and tutorial of ["One pixel attack for fooling deep neural networks"](https://arxiv.org/abs/1710.08864). The official code for the paper can be found [here](https://github.com/Carina02/One-Pixel-Attack). 10 | 11 | ## How It Works 12 | 13 | For this attack, we will use the [Cifar10 dataset](https://www.cs.toronto.edu/~kriz/cifar.html). The task of the dataset is to correctly classify a 32x32 pixel image in 1 of 10 categories (e.g., bird, deer, truck). The black-box attack requires only the probability labels (the probability value for each category) that get outputted by the neural network. We generate adversarial images by selecting a pixel and modifying it to a certain color. 14 | 15 | By using an Evolutionary Algorithm called [Differential Evolution](https://en.wikipedia.org/wiki/Differential_evolution) (DE), we can iteratively generate adversarial images to try to minimize the confidence (probability) of the neural network's classification. 16 | 17 | [![Ackley GIF](images/Ackley.gif)](https://en.wikipedia.org/wiki/Differential_evolution) 18 | 19 | Credit: [Pablo R. Mier's Blog](https://pablormier.github.io/2017/09/05/a-tutorial-on-differential-evolution-with-python/) 20 | 21 | First, generate several adversarial samples that modify a random pixel and run the images through the neural network. Next, combine the previous pixels' positions and colors together, generate several more adversarial samples from them, and run the new images through the neural network. If there were pixels that lowered the confidence of the network from the last step, replace them as the current best known solutions. Repeat these steps for a few iterations; then on the last step return the adversarial image that reduced the network's confidence the most. If successful, the confidence would be reduced so much that a new (incorrect) category now has the highest classification confidence. 22 | 23 | See below for some examples of successful attacks: 24 | 25 | [![Examples](images/pred.png "thicc indeed")](1_one-pixel-attack-cifar10.ipynb) 26 | 27 | ## Getting Started 28 | 29 | Need a GPU or just want to read? [View the first tutorial notebook with Google Colab](https://colab.research.google.com/github/hyperparticle/one-pixel-attack-keras/blob/master/1_one-pixel-attack-cifar10.ipynb). 30 | 31 | To run the code in the tutorial locally, a dedicated GPU suitable for running with Keras (`tensorflow-gpu`) is recommended. Python 3.5+ required. 32 | 33 | 1. Clone the repository. 34 | 35 | ```bash 36 | git clone https://github.com/Hyperparticle/one-pixel-attack-keras 37 | cd ./one-pixel-attack-keras 38 | ``` 39 | 40 | 2. Install the python packages in requirements.txt if you don't have them already. 41 | 42 | ```bash 43 | pip install -r ./requirements.txt 44 | ``` 45 | 46 | 3. Run the iPython tutorial notebook with Jupyter. 47 | 48 | ```bash 49 | jupyter notebook ./one-pixel-attack.ipynb 50 | ``` 51 | 52 | ## Training and Testing 53 | 54 | To train a model, run `train.py`. The model will be checkpointed (saved) after each epoch to the `networks/models` directory. 55 | 56 | For example, to train a ResNet with 200 epochs and a batch size of 128: 57 | 58 | ```bash 59 | python train.py --model resnet --epochs 200 --batch_size 128 60 | ``` 61 | 62 | To perform attack, run `attack.py`. By default this will run all models with default parameters. To specify the types of models to test, use `--model`. 63 | 64 | ```bash 65 | python attack.py --model densenet capsnet 66 | ``` 67 | 68 | The available models currently are: 69 | - `lenet` - [LeNet, first CNN model](http://yann.lecun.com/exdb/lenet/) 70 | - `pure_cnn` - [A NN with just convolutional layers](https://en.wikipedia.org/wiki/Convolutional_neural_network) 71 | - `net_in_net` - [Network in Network](https://arxiv.org/abs/1312.4400) 72 | - `resnet` - [Deep Residual Learning for Image Recognition](https://arxiv.org/abs/1512.03385) 73 | - `densenet` - [Densely Connected Convolutional Networks](https://arxiv.org/abs/1608.06993) 74 | - `wide_resnet` - [Wide Residual Networks](https://arxiv.org/abs/1605.07146) 75 | - `capsnet` - [Dynamic Routing Between Capsules](https://arxiv.org/abs/1710.09829) 76 | 77 | ## Results 78 | 79 | Preliminary results after running several experiments on various models. Each experiment generates 100 adversarial images and calculates the attack success rate, i.e., the ratio of images that successfully caused the model to misclassify an image over the total number of images. For a given model, multiple experiments are run based on the number of pixels that may be modified in an image (1,3, or 5). The differential algorithm was run with a population size of 400 and a max iteration count of 75. 80 | 81 | **Attack on 1,3,5 pixel perturbations (100 samples)** 82 | 83 | | model | parameters | test accuracy | pixels | attack success (untargeted) | attack success (targeted) | 84 | | ------------------ | ---------- | ------------- | ------ | ----------------------------- | ------------------------- | 85 | | LeNet | 62K | 74.9% | 1 | 63.0% | 34.4% | 86 | | | | | 3 | 92.0% | 64.4% | 87 | | | | | 5 | 93.0% | 64.4% | 88 | | | | | | | | 89 | | Pure CNN | 1.4M | 88.8% | 1 | 13.0% | 6.67% | 90 | | | | | 3 | 58.0% | 13.3% | 91 | | | | | 5 | 63.0% | 18.9% | 92 | | | | | | | | 93 | | Network in Network | 970K | 90.8% | 1 | 34.0% | 10.0% | 94 | | | | | 3 | 73.0% | 24.4% | 95 | | | | | 5 | 73.0% | 31.1% | 96 | | | | | | | | 97 | | ResNet | 470K | 92.3% | 1 | 34.0% | 14.4% | 98 | | | | | 3 | 79.0% | 21.1% | 99 | | | | | 5 | 79.0% | 22.2% | 100 | | | | | | | | 101 | | DenseNet | 850K | 94.7% | 1 | 31.0% | 4.44% | 102 | | | | | 3 | 71.0% | 23.3% | 103 | | | | | 5 | 69.0% | 28.9% | 104 | | | | | | | | 105 | | Wide ResNet | 11M | 95.3% | 1 | 19.0% | 1.11% | 106 | | | | | 3 | 58.0% | 18.9% | 107 | | | | | 5 | 65.0% | 22.2% | 108 | | | | | | | | 109 | | CapsNet | 12M | 79.8% | 1 | 19.0% | 0.00% | 110 | | | | | 3 | 39.0% | 4.44% | 111 | | | | | 5 | 36.0% | 4.44% | 112 | 113 | It appears that the capsule network CapsNet, while more resilient to the one pixel attack than all other CNNs, is still vulnerable. 114 | 115 | ## Milestones 116 | 117 | - [x] Cifar10 dataset 118 | - [x] Tutorial notebook 119 | - [x] LeNet, Network in Network, Residual Network, DenseNet models 120 | - [x] CapsNet (capsule network) model 121 | - [x] Configurable command-line interface 122 | - [x] Efficient differential evolution implementation 123 | - [x] ImageNet dataset 124 | -------------------------------------------------------------------------------- /networks/capsulenet/capsulenet.py: -------------------------------------------------------------------------------- 1 | """ 2 | Keras implementation of CapsNet in Hinton's paper Dynamic Routing Between Capsules. 3 | The current version maybe only works for TensorFlow backend. Actually it will be straightforward to re-write to TF code. 4 | Adopting to other backends should be easy, but I have not tested this. 5 | 6 | Usage: 7 | python CapsNet.py 8 | python CapsNet.py --epochs 100 9 | python CapsNet.py --epochs 100 --num_routing 3 10 | ... ... 11 | 12 | Result: 13 | Validation accuracy > 99.5% after 20 epochs. Still under-fitting. 14 | About 110 seconds per epoch on a single GTX1070 GPU card 15 | 16 | Author: Xifeng Guo, E-mail: `guoxifeng1990@163.com`, Github: `https://github.com/XifengGuo/CapsNet-Keras` 17 | """ 18 | 19 | from keras import layers, models, optimizers 20 | from keras import backend as K 21 | from keras.utils import to_categorical 22 | import numpy as np 23 | 24 | from networks.capsulenet.capsulelayers import CapsuleLayer, PrimaryCap, Length, Mask 25 | 26 | def CapsNet(input_shape, n_class, n_route): 27 | """ 28 | A Capsule Network on MNIST. 29 | :param input_shape: data shape, 3d, [width, height, channels] 30 | :param n_class: number of classes 31 | :param num_routing: number of routing iterations 32 | :return: A Keras Model with 2 inputs and 2 outputs 33 | """ 34 | x = layers.Input(shape=input_shape) 35 | 36 | # Layer 1: Just a conventional Conv2D layer 37 | conv1 = layers.Conv2D(filters=256, kernel_size=8, strides=1, padding='valid', activation='relu', name='conv1')(x) 38 | 39 | # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector] 40 | primarycaps = PrimaryCap(conv1, dim_vector=8, n_channels=32, kernel_size=9, strides=2, padding='valid') 41 | 42 | # Layer 3: Capsule layer. Routing algorithm works here. 43 | digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=16, num_routing=n_route, name='digitcaps')(primarycaps) 44 | 45 | # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. 46 | # If using tensorflow, this will not be necessary. :) 47 | out_caps = Length(name='output')(digitcaps) 48 | 49 | # Decoder network. 50 | y = layers.Input(shape=(n_class,)) 51 | masked = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. 52 | x_recon = layers.Dense(512, activation='relu')(masked) 53 | x_recon = layers.Dense(1024, activation='relu')(x_recon) 54 | x_recon = layers.Dense(np.prod(input_shape), activation='sigmoid')(x_recon) 55 | x_recon = layers.Reshape(target_shape=input_shape, name='out_recon')(x_recon) 56 | 57 | # two-input-two-output keras Model 58 | return models.Model([x, y], [out_caps, x_recon]) 59 | 60 | 61 | def margin_loss(y_true, y_pred): 62 | """ 63 | Margin loss for Eq.(4). When y_true[i, :] contains not just one `1`, this loss should work too. Not test it. 64 | :param y_true: [None, n_classes] 65 | :param y_pred: [None, num_capsule] 66 | :return: a scalar loss value. 67 | """ 68 | L = y_true * K.square(K.maximum(0., 0.9 - y_pred)) + \ 69 | 0.5 * (1 - y_true) * K.square(K.maximum(0., y_pred - 0.1)) 70 | 71 | return K.mean(K.sum(L, 1)) 72 | 73 | 74 | def train(model, data, args): 75 | """ 76 | Training a CapsuleNet 77 | :param model: the CapsuleNet model 78 | :param data: a tuple containing training and testing data, like `((x_train, y_train), (x_test, y_test))` 79 | :param args: arguments 80 | :return: The trained model 81 | """ 82 | # unpacking the data 83 | (x_train, y_train), (x_test, y_test) = data 84 | 85 | # callbacks 86 | log = callbacks.CSVLogger(args.save_dir + '/log.csv') 87 | tb = callbacks.TensorBoard(log_dir=args.save_dir + '/tensorboard-logs', 88 | batch_size=args.batch_size, histogram_freq=args.debug) 89 | checkpoint = callbacks.ModelCheckpoint(args.save_dir + '/weights-{epoch:02d}.h5', 90 | save_best_only=True, save_weights_only=True, verbose=1) 91 | lr_decay = callbacks.LearningRateScheduler(schedule=lambda epoch: args.lr * (0.9 ** epoch)) 92 | 93 | # compile the model 94 | model.compile(optimizer=optimizers.Adam(lr=args.lr), 95 | loss=[margin_loss, 'mse'], 96 | loss_weights=[1., args.lam_recon], 97 | metrics={'out_caps': 'accuracy'}) 98 | 99 | """ 100 | # Training without data augmentation: 101 | model.fit([x_train, y_train], [y_train, x_train], batch_size=args.batch_size, epochs=args.epochs, 102 | validation_data=[[x_test, y_test], [y_test, x_test]], callbacks=[log, tb, checkpoint, lr_decay]) 103 | """ 104 | 105 | # Begin: Training with data augmentation ---------------------------------------------------------------------# 106 | def train_generator(x, y, batch_size, shift_fraction=0.): 107 | train_datagen = ImageDataGenerator(width_shift_range=shift_fraction, 108 | height_shift_range=shift_fraction) # shift up to 2 pixel for MNIST 109 | generator = train_datagen.flow(x, y, batch_size=batch_size) 110 | while 1: 111 | x_batch, y_batch = generator.next() 112 | yield ([x_batch, y_batch], [y_batch, x_batch]) 113 | 114 | # Training with data augmentation. If shift_fraction=0., also no augmentation. 115 | model.fit_generator(generator=train_generator(x_train, y_train, args.batch_size, args.shift_fraction), 116 | steps_per_epoch=int(y_train.shape[0] / args.batch_size), 117 | epochs=args.epochs, 118 | validation_data=[[x_test, y_test], [y_test, x_test]], 119 | callbacks=[log, tb, checkpoint, lr_decay]) 120 | # End: Training with data augmentation -----------------------------------------------------------------------# 121 | 122 | model.save_weights(args.save_dir + '/trained_model.h5') 123 | print('Trained model saved to \'%s/trained_model.h5\'' % args.save_dir) 124 | 125 | from networks.capsulenet.helper_function import plot_log 126 | plot_log(args.save_dir + '/log.csv', show=True) 127 | 128 | return model 129 | 130 | 131 | def test(model, data): 132 | x_test, y_test = data 133 | y_pred, x_recon = model.predict([x_test, y_test], batch_size=100) 134 | print('-'*50) 135 | print('Test acc:', np.sum(np.argmax(y_pred, 1) == np.argmax(y_test, 1))/y_test.shape[0]) 136 | 137 | import matplotlib.pyplot as plt 138 | from networks.capsulenet.helper_function import combine_images 139 | from PIL import Image 140 | 141 | img = combine_images(np.concatenate([x_test[:50],x_recon[:50]])) 142 | image = img * 255 143 | Image.fromarray(image.astype(np.uint8)).save("real_and_recon.png") 144 | print() 145 | print('Reconstructed images are saved to ./real_and_recon.png') 146 | print('-'*50) 147 | plt.imshow(plt.imread("real_and_recon.png", )) 148 | plt.show() 149 | 150 | 151 | def load_mnist(): 152 | # the data, shuffled and split between train and test sets 153 | from keras.datasets import mnist 154 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 155 | 156 | x_train = x_train.reshape(-1, 28, 28, 1).astype('float32') / 255. 157 | x_test = x_test.reshape(-1, 28, 28, 1).astype('float32') / 255. 158 | y_train = to_categorical(y_train.astype('float32')) 159 | y_test = to_categorical(y_test.astype('float32')) 160 | return (x_train, y_train), (x_test, y_test) 161 | 162 | 163 | if __name__ == "__main__": 164 | import numpy as np 165 | import os 166 | from keras.preprocessing.image import ImageDataGenerator 167 | from keras import callbacks 168 | from keras.utils.vis_utils import plot_model 169 | 170 | # setting the hyper parameters 171 | import argparse 172 | parser = argparse.ArgumentParser() 173 | parser.add_argument('--batch_size', default=100, type=int) 174 | parser.add_argument('--epochs', default=30, type=int) 175 | parser.add_argument('--lam_recon', default=0.392, type=float) # 784 * 0.0005, paper uses sum of SE, here uses MSE 176 | parser.add_argument('--num_routing', default=3, type=int) # num_routing should > 0 177 | parser.add_argument('--shift_fraction', default=0.1, type=float) 178 | parser.add_argument('--debug', default=0, type=int) # debug>0 will save weights by TensorBoard 179 | parser.add_argument('--save_dir', default='./result') 180 | parser.add_argument('--is_training', default=1, type=int) 181 | parser.add_argument('--weights', default=None) 182 | parser.add_argument('--lr', default=0.001, type=float) 183 | args = parser.parse_args() 184 | print(args) 185 | if not os.path.exists(args.save_dir): 186 | os.makedirs(args.save_dir) 187 | 188 | # load data 189 | (x_train, y_train), (x_test, y_test) = load_mnist() 190 | 191 | # define model 192 | model = CapsNet(input_shape=[28, 28, 1], 193 | n_class=len(np.unique(np.argmax(y_train, 1))), 194 | num_routing=args.num_routing) 195 | model.summary() 196 | # plot_model(model, to_file=args.save_dir+'/model.png', show_shapes=True) 197 | 198 | # train or test 199 | if args.weights is not None: # init the model weights with provided one 200 | model.load_weights(args.weights) 201 | if args.is_training: 202 | train(model=model, data=((x_train, y_train), (x_test, y_test)), args=args) 203 | else: # as long as weights are given, will run testing 204 | if args.weights is None: 205 | print('No weights are provided. Will test using random initialized weights.') 206 | test(model=model, data=(x_test, y_test)) -------------------------------------------------------------------------------- /networks/capsulenet/capsulelayers.py: -------------------------------------------------------------------------------- 1 | """ 2 | Some key layers used for constructing a Capsule Network. These layers can used to construct CapsNet on other dataset, 3 | not just on MNIST. 4 | *NOTE*: some functions can be implemented in multiple ways, I keep all of them. You can try them for yourself just by 5 | uncommenting them and commenting their counterparts. 6 | 7 | Author: Xifeng Guo, E-mail: `guoxifeng1990@163.com`, Github: `https://github.com/XifengGuo/CapsNet-Keras` 8 | """ 9 | 10 | import keras.backend as K 11 | import tensorflow as tf 12 | from keras import initializers, layers 13 | 14 | 15 | class Length(layers.Layer): 16 | """ 17 | Compute the length of vectors. This is used to compute a Tensor that has the same shape with y_true in margin_loss 18 | inputs: shape=[dim_1, ..., dim_{n-1}, dim_n] 19 | output: shape=[dim_1, ..., dim_{n-1}] 20 | """ 21 | def call(self, inputs, **kwargs): 22 | return K.sqrt(K.sum(K.square(inputs), -1)) 23 | 24 | def compute_output_shape(self, input_shape): 25 | return input_shape[:-1] 26 | 27 | 28 | class Mask(layers.Layer): 29 | """ 30 | Mask a Tensor with shape=[None, d1, d2] by the max value in axis=1. 31 | Output shape: [None, d2] 32 | """ 33 | def call(self, inputs, **kwargs): 34 | # use true label to select target capsule, shape=[batch_size, num_capsule] 35 | if type(inputs) is list: # true label is provided with shape = [batch_size, n_classes], i.e. one-hot code. 36 | assert len(inputs) == 2 37 | inputs, mask = inputs 38 | else: # if no true label, mask by the max length of vectors of capsules 39 | x = inputs 40 | # Enlarge the range of values in x to make max(new_x)=1 and others < 0 41 | x = (x - K.max(x, 1, True)) / K.epsilon() + 1 42 | mask = K.clip(x, 0, 1) # the max value in x clipped to 1 and other to 0 43 | 44 | # masked inputs, shape = [batch_size, dim_vector] 45 | inputs_masked = K.batch_dot(inputs, mask, [1, 1]) 46 | return inputs_masked 47 | 48 | def compute_output_shape(self, input_shape): 49 | if type(input_shape[0]) is tuple: # true label provided 50 | return tuple([None, input_shape[0][-1]]) 51 | else: 52 | return tuple([None, input_shape[-1]]) 53 | 54 | 55 | def squash(vectors, axis=-1): 56 | """ 57 | The non-linear activation used in Capsule. It drives the length of a large vector to near 1 and small vector to 0 58 | :param vectors: some vectors to be squashed, N-dim tensor 59 | :param axis: the axis to squash 60 | :return: a Tensor with same shape as input vectors 61 | """ 62 | s_squared_norm = K.sum(K.square(vectors), axis, keepdims=True) 63 | scale = s_squared_norm / (1 + s_squared_norm) / K.sqrt(s_squared_norm + K.epsilon()) 64 | return scale * vectors 65 | 66 | 67 | class CapsuleLayer(layers.Layer): 68 | """ 69 | The capsule layer. It is similar to Dense layer. Dense layer has `in_num` inputs, each is a scalar, the output of the 70 | neuron from the former layer, and it has `out_num` output neurons. CapsuleLayer just expand the output of the neuron 71 | from scalar to vector. So its input shape = [None, input_num_capsule, input_dim_vector] and output shape = \ 72 | [None, num_capsule, dim_vector]. For Dense Layer, input_dim_vector = dim_vector = 1. 73 | 74 | :param num_capsule: number of capsules in this layer 75 | :param dim_vector: dimension of the output vectors of the capsules in this layer 76 | :param num_routings: number of iterations for the routing algorithm 77 | """ 78 | def __init__(self, num_capsule, dim_vector, num_routing=3, 79 | kernel_initializer='glorot_uniform', 80 | bias_initializer='zeros', 81 | **kwargs): 82 | super(CapsuleLayer, self).__init__(**kwargs) 83 | self.num_capsule = num_capsule 84 | self.dim_vector = dim_vector 85 | self.num_routing = num_routing 86 | self.kernel_initializer = initializers.get(kernel_initializer) 87 | self.bias_initializer = initializers.get(bias_initializer) 88 | 89 | def build(self, input_shape): 90 | assert len(input_shape) >= 3, "The input Tensor should have shape=[None, input_num_capsule, input_dim_vector]" 91 | self.input_num_capsule = input_shape[1] 92 | self.input_dim_vector = input_shape[2] 93 | 94 | # Transform matrix 95 | self.W = self.add_weight(shape=[self.input_num_capsule, self.num_capsule, self.input_dim_vector, self.dim_vector], 96 | initializer=self.kernel_initializer, 97 | name='W') 98 | 99 | # Coupling coefficient. The redundant dimensions are just to facilitate subsequent matrix calculation. 100 | self.bias = self.add_weight(shape=[1, self.input_num_capsule, self.num_capsule, 1, 1], 101 | initializer=self.bias_initializer, 102 | name='bias', 103 | trainable=False) 104 | self.built = True 105 | 106 | def call(self, inputs, training=None): 107 | # inputs.shape=[None, input_num_capsule, input_dim_vector] 108 | # Expand dims to [None, input_num_capsule, 1, 1, input_dim_vector] 109 | inputs_expand = K.expand_dims(K.expand_dims(inputs, 2), 2) 110 | 111 | # Replicate num_capsule dimension to prepare being multiplied by W 112 | # Now it has shape = [None, input_num_capsule, num_capsule, 1, input_dim_vector] 113 | inputs_tiled = K.tile(inputs_expand, [1, 1, self.num_capsule, 1, 1]) 114 | 115 | """ 116 | # Begin: inputs_hat computation V1 ---------------------------------------------------------------------# 117 | # Compute `inputs * W` by expanding the first dim of W. More time-consuming and need batch_size. 118 | # w_tiled.shape = [batch_size, input_num_capsule, num_capsule, input_dim_vector, dim_vector] 119 | w_tiled = K.tile(K.expand_dims(self.W, 0), [self.batch_size, 1, 1, 1, 1]) 120 | 121 | # Transformed vectors, inputs_hat.shape = [None, input_num_capsule, num_capsule, 1, dim_vector] 122 | inputs_hat = K.batch_dot(inputs_tiled, w_tiled, [4, 3]) 123 | # End: inputs_hat computation V1 ---------------------------------------------------------------------# 124 | """ 125 | 126 | # Begin: inputs_hat computation V2 ---------------------------------------------------------------------# 127 | # Compute `inputs * W` by scanning inputs_tiled on dimension 0. This is faster but requires Tensorflow. 128 | # inputs_hat.shape = [None, input_num_capsule, num_capsule, 1, dim_vector] 129 | inputs_hat = tf.scan(lambda ac, x: K.batch_dot(x, self.W, [3, 2]), 130 | elems=inputs_tiled, 131 | initializer=K.zeros([self.input_num_capsule, self.num_capsule, 1, self.dim_vector])) 132 | # End: inputs_hat computation V2 ---------------------------------------------------------------------# 133 | """ 134 | # Begin: routing algorithm V1, dynamic ------------------------------------------------------------# 135 | def body(i, b, outputs): 136 | c = tf.nn.softmax(b, dim=2) # dim=2 is the num_capsule dimension 137 | outputs = squash(K.sum(c * inputs_hat, 1, keepdims=True)) 138 | if i != 1: 139 | b = b + K.sum(inputs_hat * outputs, -1, keepdims=True) 140 | return [i-1, b, outputs] 141 | 142 | cond = lambda i, b, inputs_hat: i > 0 143 | loop_vars = [K.constant(self.num_routing), self.bias, K.sum(inputs_hat, 1, keepdims=True)] 144 | shape_invariants = [tf.TensorShape([]), 145 | tf.TensorShape([None, self.input_num_capsule, self.num_capsule, 1, 1]), 146 | tf.TensorShape([None, 1, self.num_capsule, 1, self.dim_vector])] 147 | _, _, outputs = tf.while_loop(cond, body, loop_vars, shape_invariants) 148 | # End: routing algorithm V1, dynamic ------------------------------------------------------------# 149 | """ 150 | 151 | # Begin: routing algorithm V2, static -----------------------------------------------------------# 152 | # Routing algorithm V2. Use iteration. V2 and V1 both work without much difference on performance 153 | assert self.num_routing > 0, 'The num_routing should be > 0.' 154 | for i in range(self.num_routing): 155 | c = tf.nn.softmax(self.bias, axis=2) # dim=2 is the num_capsule dimension 156 | # outputs.shape=[None, 1, num_capsule, 1, dim_vector] 157 | outputs = squash(K.sum(c * inputs_hat, 1, keepdims=True)) 158 | 159 | # last iteration needs not compute bias which will not be passed to the graph any more anyway. 160 | if i != self.num_routing - 1: 161 | # self.bias = K.update_add(self.bias, K.sum(inputs_hat * outputs, [0, -1], keepdims=True)) 162 | self.bias += K.sum(inputs_hat * outputs, -1, keepdims=True) 163 | # tf.summary.histogram('BigBee', self.bias) # for debugging 164 | # End: routing algorithm V2, static ------------------------------------------------------------# 165 | 166 | return K.reshape(outputs, [-1, self.num_capsule, self.dim_vector]) 167 | 168 | def compute_output_shape(self, input_shape): 169 | return tuple([None, self.num_capsule, self.dim_vector]) 170 | 171 | 172 | def PrimaryCap(inputs, dim_vector, n_channels, kernel_size, strides, padding): 173 | """ 174 | Apply Conv2D `n_channels` times and concatenate all capsules 175 | :param inputs: 4D tensor, shape=[None, width, height, channels] 176 | :param dim_vector: the dim of the output vector of capsule 177 | :param n_channels: the number of types of capsules 178 | :return: output tensor, shape=[None, num_capsule, dim_vector] 179 | """ 180 | output = layers.Conv2D(filters=dim_vector*n_channels, kernel_size=kernel_size, strides=strides, padding=padding, 181 | name='primarycap_conv2d')(inputs) 182 | outputs = layers.Reshape(target_shape=[-1, dim_vector], name='primarycap_reshape')(output) 183 | return layers.Lambda(squash, name='primarycap_squash')(outputs) 184 | 185 | 186 | """ 187 | # The following is another way to implement primary capsule layer. This is much slower. 188 | # Apply Conv2D `n_channels` times and concatenate all capsules 189 | def PrimaryCap(inputs, dim_vector, n_channels, kernel_size, strides, padding): 190 | outputs = [] 191 | for _ in range(n_channels): 192 | output = layers.Conv2D(filters=dim_vector, kernel_size=kernel_size, strides=strides, padding=padding)(inputs) 193 | outputs.append(layers.Reshape([output.get_shape().as_list()[1] ** 2, dim_vector])(output)) 194 | outputs = layers.Concatenate(axis=1)(outputs) 195 | return layers.Lambda(squash)(outputs) 196 | """ -------------------------------------------------------------------------------- /data/imagenet_class_index.json: -------------------------------------------------------------------------------- 1 | {"0": ["n01440764", "tench"], "1": ["n01443537", "goldfish"], "2": ["n01484850", "great_white_shark"], "3": ["n01491361", "tiger_shark"], "4": ["n01494475", "hammerhead"], "5": ["n01496331", "electric_ray"], "6": ["n01498041", "stingray"], "7": ["n01514668", "cock"], "8": ["n01514859", "hen"], "9": ["n01518878", "ostrich"], "10": ["n01530575", "brambling"], "11": ["n01531178", "goldfinch"], "12": ["n01532829", "house_finch"], "13": ["n01534433", "junco"], "14": ["n01537544", "indigo_bunting"], "15": ["n01558993", "robin"], "16": ["n01560419", "bulbul"], "17": ["n01580077", "jay"], "18": ["n01582220", "magpie"], "19": ["n01592084", "chickadee"], "20": ["n01601694", "water_ouzel"], "21": ["n01608432", "kite"], "22": ["n01614925", "bald_eagle"], "23": ["n01616318", "vulture"], "24": ["n01622779", "great_grey_owl"], "25": ["n01629819", "European_fire_salamander"], "26": ["n01630670", "common_newt"], "27": ["n01631663", "eft"], "28": ["n01632458", "spotted_salamander"], "29": ["n01632777", "axolotl"], "30": ["n01641577", "bullfrog"], "31": ["n01644373", "tree_frog"], "32": ["n01644900", "tailed_frog"], "33": ["n01664065", "loggerhead"], "34": ["n01665541", "leatherback_turtle"], "35": ["n01667114", "mud_turtle"], "36": ["n01667778", "terrapin"], "37": ["n01669191", "box_turtle"], "38": ["n01675722", "banded_gecko"], "39": ["n01677366", "common_iguana"], "40": ["n01682714", "American_chameleon"], "41": ["n01685808", "whiptail"], "42": ["n01687978", "agama"], "43": ["n01688243", "frilled_lizard"], "44": ["n01689811", "alligator_lizard"], "45": ["n01692333", "Gila_monster"], "46": ["n01693334", "green_lizard"], "47": ["n01694178", "African_chameleon"], "48": ["n01695060", "Komodo_dragon"], "49": ["n01697457", "African_crocodile"], "50": ["n01698640", "American_alligator"], "51": ["n01704323", "triceratops"], "52": ["n01728572", "thunder_snake"], "53": ["n01728920", "ringneck_snake"], "54": ["n01729322", "hognose_snake"], "55": ["n01729977", "green_snake"], "56": ["n01734418", "king_snake"], "57": ["n01735189", "garter_snake"], "58": ["n01737021", "water_snake"], "59": ["n01739381", "vine_snake"], "60": ["n01740131", "night_snake"], "61": ["n01742172", "boa_constrictor"], "62": ["n01744401", "rock_python"], "63": ["n01748264", "Indian_cobra"], "64": ["n01749939", "green_mamba"], "65": ["n01751748", "sea_snake"], "66": ["n01753488", "horned_viper"], "67": ["n01755581", "diamondback"], "68": ["n01756291", "sidewinder"], "69": ["n01768244", "trilobite"], "70": ["n01770081", "harvestman"], "71": ["n01770393", "scorpion"], "72": ["n01773157", "black_and_gold_garden_spider"], "73": ["n01773549", "barn_spider"], "74": ["n01773797", "garden_spider"], "75": ["n01774384", "black_widow"], "76": ["n01774750", "tarantula"], "77": ["n01775062", "wolf_spider"], "78": ["n01776313", "tick"], "79": ["n01784675", "centipede"], "80": ["n01795545", "black_grouse"], "81": ["n01796340", "ptarmigan"], "82": ["n01797886", "ruffed_grouse"], "83": ["n01798484", "prairie_chicken"], "84": ["n01806143", "peacock"], "85": ["n01806567", "quail"], "86": ["n01807496", "partridge"], "87": ["n01817953", "African_grey"], "88": ["n01818515", "macaw"], "89": ["n01819313", "sulphur-crested_cockatoo"], "90": ["n01820546", "lorikeet"], "91": ["n01824575", "coucal"], "92": ["n01828970", "bee_eater"], "93": ["n01829413", "hornbill"], "94": ["n01833805", "hummingbird"], "95": ["n01843065", "jacamar"], "96": ["n01843383", "toucan"], "97": ["n01847000", "drake"], "98": ["n01855032", "red-breasted_merganser"], "99": ["n01855672", "goose"], "100": ["n01860187", "black_swan"], "101": ["n01871265", "tusker"], "102": ["n01872401", "echidna"], "103": ["n01873310", "platypus"], "104": ["n01877812", "wallaby"], "105": ["n01882714", "koala"], "106": ["n01883070", "wombat"], "107": ["n01910747", "jellyfish"], "108": ["n01914609", "sea_anemone"], "109": ["n01917289", "brain_coral"], "110": ["n01924916", "flatworm"], "111": ["n01930112", "nematode"], "112": ["n01943899", "conch"], "113": ["n01944390", "snail"], "114": ["n01945685", "slug"], "115": ["n01950731", "sea_slug"], "116": ["n01955084", "chiton"], "117": ["n01968897", "chambered_nautilus"], "118": ["n01978287", "Dungeness_crab"], "119": ["n01978455", "rock_crab"], "120": ["n01980166", "fiddler_crab"], "121": ["n01981276", "king_crab"], "122": ["n01983481", "American_lobster"], "123": ["n01984695", "spiny_lobster"], "124": ["n01985128", "crayfish"], "125": ["n01986214", "hermit_crab"], "126": ["n01990800", "isopod"], "127": ["n02002556", "white_stork"], "128": ["n02002724", "black_stork"], "129": ["n02006656", "spoonbill"], "130": ["n02007558", "flamingo"], "131": ["n02009229", "little_blue_heron"], "132": ["n02009912", "American_egret"], "133": ["n02011460", "bittern"], "134": ["n02012849", "crane"], "135": ["n02013706", "limpkin"], "136": ["n02017213", "European_gallinule"], "137": ["n02018207", "American_coot"], "138": ["n02018795", "bustard"], "139": ["n02025239", "ruddy_turnstone"], "140": ["n02027492", "red-backed_sandpiper"], "141": ["n02028035", "redshank"], "142": ["n02033041", "dowitcher"], "143": ["n02037110", "oystercatcher"], "144": ["n02051845", "pelican"], "145": ["n02056570", "king_penguin"], "146": ["n02058221", "albatross"], "147": ["n02066245", "grey_whale"], "148": ["n02071294", "killer_whale"], "149": ["n02074367", "dugong"], "150": ["n02077923", "sea_lion"], "151": ["n02085620", "Chihuahua"], "152": ["n02085782", "Japanese_spaniel"], "153": ["n02085936", "Maltese_dog"], "154": ["n02086079", "Pekinese"], "155": ["n02086240", "Shih-Tzu"], "156": ["n02086646", "Blenheim_spaniel"], "157": ["n02086910", "papillon"], "158": ["n02087046", "toy_terrier"], "159": ["n02087394", "Rhodesian_ridgeback"], "160": ["n02088094", "Afghan_hound"], "161": ["n02088238", "basset"], "162": ["n02088364", "beagle"], "163": ["n02088466", "bloodhound"], "164": ["n02088632", "bluetick"], "165": ["n02089078", "black-and-tan_coonhound"], "166": ["n02089867", "Walker_hound"], "167": ["n02089973", "English_foxhound"], "168": ["n02090379", "redbone"], "169": ["n02090622", "borzoi"], "170": ["n02090721", "Irish_wolfhound"], "171": ["n02091032", "Italian_greyhound"], "172": ["n02091134", "whippet"], "173": ["n02091244", "Ibizan_hound"], "174": ["n02091467", "Norwegian_elkhound"], "175": ["n02091635", "otterhound"], "176": ["n02091831", "Saluki"], "177": ["n02092002", "Scottish_deerhound"], "178": ["n02092339", "Weimaraner"], "179": ["n02093256", "Staffordshire_bullterrier"], "180": ["n02093428", "American_Staffordshire_terrier"], "181": ["n02093647", "Bedlington_terrier"], "182": ["n02093754", "Border_terrier"], "183": ["n02093859", "Kerry_blue_terrier"], "184": ["n02093991", "Irish_terrier"], "185": ["n02094114", "Norfolk_terrier"], "186": ["n02094258", "Norwich_terrier"], "187": ["n02094433", "Yorkshire_terrier"], "188": ["n02095314", "wire-haired_fox_terrier"], "189": ["n02095570", "Lakeland_terrier"], "190": ["n02095889", "Sealyham_terrier"], "191": ["n02096051", "Airedale"], "192": ["n02096177", "cairn"], "193": ["n02096294", "Australian_terrier"], "194": ["n02096437", "Dandie_Dinmont"], "195": ["n02096585", "Boston_bull"], "196": ["n02097047", "miniature_schnauzer"], "197": ["n02097130", "giant_schnauzer"], "198": ["n02097209", "standard_schnauzer"], "199": ["n02097298", "Scotch_terrier"], "200": ["n02097474", "Tibetan_terrier"], "201": ["n02097658", "silky_terrier"], "202": ["n02098105", "soft-coated_wheaten_terrier"], "203": ["n02098286", "West_Highland_white_terrier"], "204": ["n02098413", "Lhasa"], "205": ["n02099267", "flat-coated_retriever"], "206": ["n02099429", "curly-coated_retriever"], "207": ["n02099601", "golden_retriever"], "208": ["n02099712", "Labrador_retriever"], "209": ["n02099849", "Chesapeake_Bay_retriever"], "210": ["n02100236", "German_short-haired_pointer"], "211": ["n02100583", "vizsla"], "212": ["n02100735", "English_setter"], "213": ["n02100877", "Irish_setter"], "214": ["n02101006", "Gordon_setter"], "215": ["n02101388", "Brittany_spaniel"], "216": ["n02101556", "clumber"], "217": ["n02102040", "English_springer"], "218": ["n02102177", "Welsh_springer_spaniel"], "219": ["n02102318", "cocker_spaniel"], "220": ["n02102480", "Sussex_spaniel"], "221": ["n02102973", "Irish_water_spaniel"], "222": ["n02104029", "kuvasz"], "223": ["n02104365", "schipperke"], "224": ["n02105056", "groenendael"], "225": ["n02105162", "malinois"], "226": ["n02105251", "briard"], "227": ["n02105412", "kelpie"], "228": ["n02105505", "komondor"], "229": ["n02105641", "Old_English_sheepdog"], "230": ["n02105855", "Shetland_sheepdog"], "231": ["n02106030", "collie"], "232": ["n02106166", "Border_collie"], "233": ["n02106382", "Bouvier_des_Flandres"], "234": ["n02106550", "Rottweiler"], "235": ["n02106662", "German_shepherd"], "236": ["n02107142", "Doberman"], "237": ["n02107312", "miniature_pinscher"], "238": ["n02107574", "Greater_Swiss_Mountain_dog"], "239": ["n02107683", "Bernese_mountain_dog"], "240": ["n02107908", "Appenzeller"], "241": ["n02108000", "EntleBucher"], "242": ["n02108089", "boxer"], "243": ["n02108422", "bull_mastiff"], "244": ["n02108551", "Tibetan_mastiff"], "245": ["n02108915", "French_bulldog"], "246": ["n02109047", "Great_Dane"], "247": ["n02109525", "Saint_Bernard"], "248": ["n02109961", "Eskimo_dog"], "249": ["n02110063", "malamute"], "250": ["n02110185", "Siberian_husky"], "251": ["n02110341", "dalmatian"], "252": ["n02110627", "affenpinscher"], "253": ["n02110806", "basenji"], "254": ["n02110958", "pug"], "255": ["n02111129", "Leonberg"], "256": ["n02111277", "Newfoundland"], "257": ["n02111500", "Great_Pyrenees"], "258": ["n02111889", "Samoyed"], "259": ["n02112018", "Pomeranian"], "260": ["n02112137", "chow"], "261": ["n02112350", "keeshond"], "262": ["n02112706", "Brabancon_griffon"], "263": ["n02113023", "Pembroke"], "264": ["n02113186", "Cardigan"], "265": ["n02113624", "toy_poodle"], "266": ["n02113712", "miniature_poodle"], "267": ["n02113799", "standard_poodle"], "268": ["n02113978", "Mexican_hairless"], "269": ["n02114367", "timber_wolf"], "270": ["n02114548", "white_wolf"], "271": ["n02114712", "red_wolf"], "272": ["n02114855", "coyote"], "273": ["n02115641", "dingo"], "274": ["n02115913", "dhole"], "275": ["n02116738", "African_hunting_dog"], "276": ["n02117135", "hyena"], "277": ["n02119022", "red_fox"], "278": ["n02119789", "kit_fox"], "279": ["n02120079", "Arctic_fox"], "280": ["n02120505", "grey_fox"], "281": ["n02123045", "tabby"], "282": ["n02123159", "tiger_cat"], "283": ["n02123394", "Persian_cat"], "284": ["n02123597", "Siamese_cat"], "285": ["n02124075", "Egyptian_cat"], "286": ["n02125311", "cougar"], "287": ["n02127052", "lynx"], "288": ["n02128385", "leopard"], "289": ["n02128757", "snow_leopard"], "290": ["n02128925", "jaguar"], "291": ["n02129165", "lion"], "292": ["n02129604", "tiger"], "293": ["n02130308", "cheetah"], "294": ["n02132136", "brown_bear"], "295": ["n02133161", "American_black_bear"], "296": ["n02134084", "ice_bear"], "297": ["n02134418", "sloth_bear"], "298": ["n02137549", "mongoose"], "299": ["n02138441", "meerkat"], "300": ["n02165105", "tiger_beetle"], "301": ["n02165456", "ladybug"], "302": ["n02167151", "ground_beetle"], "303": ["n02168699", "long-horned_beetle"], "304": ["n02169497", "leaf_beetle"], "305": ["n02172182", "dung_beetle"], "306": ["n02174001", "rhinoceros_beetle"], "307": ["n02177972", "weevil"], "308": ["n02190166", "fly"], "309": ["n02206856", "bee"], "310": ["n02219486", "ant"], "311": ["n02226429", "grasshopper"], "312": ["n02229544", "cricket"], "313": ["n02231487", "walking_stick"], "314": ["n02233338", "cockroach"], "315": ["n02236044", "mantis"], "316": ["n02256656", "cicada"], "317": ["n02259212", "leafhopper"], "318": ["n02264363", "lacewing"], "319": ["n02268443", "dragonfly"], "320": ["n02268853", "damselfly"], "321": ["n02276258", "admiral"], "322": ["n02277742", "ringlet"], "323": ["n02279972", "monarch"], "324": ["n02280649", "cabbage_butterfly"], "325": ["n02281406", "sulphur_butterfly"], "326": ["n02281787", "lycaenid"], "327": ["n02317335", "starfish"], "328": ["n02319095", "sea_urchin"], "329": ["n02321529", "sea_cucumber"], "330": ["n02325366", "wood_rabbit"], "331": ["n02326432", "hare"], "332": ["n02328150", "Angora"], "333": ["n02342885", "hamster"], "334": ["n02346627", "porcupine"], "335": ["n02356798", "fox_squirrel"], "336": ["n02361337", "marmot"], "337": ["n02363005", "beaver"], "338": ["n02364673", "guinea_pig"], "339": ["n02389026", "sorrel"], "340": ["n02391049", "zebra"], "341": ["n02395406", "hog"], "342": ["n02396427", "wild_boar"], "343": ["n02397096", "warthog"], "344": ["n02398521", "hippopotamus"], "345": ["n02403003", "ox"], "346": ["n02408429", "water_buffalo"], "347": ["n02410509", "bison"], "348": ["n02412080", "ram"], "349": ["n02415577", "bighorn"], "350": ["n02417914", "ibex"], "351": ["n02422106", "hartebeest"], "352": ["n02422699", "impala"], "353": ["n02423022", "gazelle"], "354": ["n02437312", "Arabian_camel"], "355": ["n02437616", "llama"], "356": ["n02441942", "weasel"], "357": ["n02442845", "mink"], "358": ["n02443114", "polecat"], "359": ["n02443484", "black-footed_ferret"], "360": ["n02444819", "otter"], "361": ["n02445715", "skunk"], "362": ["n02447366", "badger"], "363": ["n02454379", "armadillo"], "364": ["n02457408", "three-toed_sloth"], "365": ["n02480495", "orangutan"], "366": ["n02480855", "gorilla"], "367": ["n02481823", "chimpanzee"], "368": ["n02483362", "gibbon"], "369": ["n02483708", "siamang"], "370": ["n02484975", "guenon"], "371": ["n02486261", "patas"], "372": ["n02486410", "baboon"], "373": ["n02487347", "macaque"], "374": ["n02488291", "langur"], "375": ["n02488702", "colobus"], "376": ["n02489166", "proboscis_monkey"], "377": ["n02490219", "marmoset"], "378": ["n02492035", "capuchin"], "379": ["n02492660", "howler_monkey"], "380": ["n02493509", "titi"], "381": ["n02493793", "spider_monkey"], "382": ["n02494079", "squirrel_monkey"], "383": ["n02497673", "Madagascar_cat"], "384": ["n02500267", "indri"], "385": ["n02504013", "Indian_elephant"], "386": ["n02504458", "African_elephant"], "387": ["n02509815", "lesser_panda"], "388": ["n02510455", "giant_panda"], "389": ["n02514041", "barracouta"], "390": ["n02526121", "eel"], "391": ["n02536864", "coho"], "392": ["n02606052", "rock_beauty"], "393": ["n02607072", "anemone_fish"], "394": ["n02640242", "sturgeon"], "395": ["n02641379", "gar"], "396": ["n02643566", "lionfish"], "397": ["n02655020", "puffer"], "398": ["n02666196", "abacus"], "399": ["n02667093", "abaya"], "400": ["n02669723", "academic_gown"], "401": ["n02672831", "accordion"], "402": ["n02676566", "acoustic_guitar"], "403": ["n02687172", "aircraft_carrier"], "404": ["n02690373", "airliner"], "405": ["n02692877", "airship"], "406": ["n02699494", "altar"], "407": ["n02701002", "ambulance"], "408": ["n02704792", "amphibian"], "409": ["n02708093", "analog_clock"], "410": ["n02727426", "apiary"], "411": ["n02730930", "apron"], "412": ["n02747177", "ashcan"], "413": ["n02749479", "assault_rifle"], "414": ["n02769748", "backpack"], "415": ["n02776631", "bakery"], "416": ["n02777292", "balance_beam"], "417": ["n02782093", "balloon"], "418": ["n02783161", "ballpoint"], "419": ["n02786058", "Band_Aid"], "420": ["n02787622", "banjo"], "421": ["n02788148", "bannister"], "422": ["n02790996", "barbell"], "423": ["n02791124", "barber_chair"], "424": ["n02791270", "barbershop"], "425": ["n02793495", "barn"], "426": ["n02794156", "barometer"], "427": ["n02795169", "barrel"], "428": ["n02797295", "barrow"], "429": ["n02799071", "baseball"], "430": ["n02802426", "basketball"], "431": ["n02804414", "bassinet"], "432": ["n02804610", "bassoon"], "433": ["n02807133", "bathing_cap"], "434": ["n02808304", "bath_towel"], "435": ["n02808440", "bathtub"], "436": ["n02814533", "beach_wagon"], "437": ["n02814860", "beacon"], "438": ["n02815834", "beaker"], "439": ["n02817516", "bearskin"], "440": ["n02823428", "beer_bottle"], "441": ["n02823750", "beer_glass"], "442": ["n02825657", "bell_cote"], "443": ["n02834397", "bib"], "444": ["n02835271", "bicycle-built-for-two"], "445": ["n02837789", "bikini"], "446": ["n02840245", "binder"], "447": ["n02841315", "binoculars"], "448": ["n02843684", "birdhouse"], "449": ["n02859443", "boathouse"], "450": ["n02860847", "bobsled"], "451": ["n02865351", "bolo_tie"], "452": ["n02869837", "bonnet"], "453": ["n02870880", "bookcase"], "454": ["n02871525", "bookshop"], "455": ["n02877765", "bottlecap"], "456": ["n02879718", "bow"], "457": ["n02883205", "bow_tie"], "458": ["n02892201", "brass"], "459": ["n02892767", "brassiere"], "460": ["n02894605", "breakwater"], "461": ["n02895154", "breastplate"], "462": ["n02906734", "broom"], "463": ["n02909870", "bucket"], "464": ["n02910353", "buckle"], "465": ["n02916936", "bulletproof_vest"], "466": ["n02917067", "bullet_train"], "467": ["n02927161", "butcher_shop"], "468": ["n02930766", "cab"], "469": ["n02939185", "caldron"], "470": ["n02948072", "candle"], "471": ["n02950826", "cannon"], "472": ["n02951358", "canoe"], "473": ["n02951585", "can_opener"], "474": ["n02963159", "cardigan"], "475": ["n02965783", "car_mirror"], "476": ["n02966193", "carousel"], "477": ["n02966687", "carpenter's_kit"], "478": ["n02971356", "carton"], "479": ["n02974003", "car_wheel"], "480": ["n02977058", "cash_machine"], "481": ["n02978881", "cassette"], "482": ["n02979186", "cassette_player"], "483": ["n02980441", "castle"], "484": ["n02981792", "catamaran"], "485": ["n02988304", "CD_player"], "486": ["n02992211", "cello"], "487": ["n02992529", "cellular_telephone"], "488": ["n02999410", "chain"], "489": ["n03000134", "chainlink_fence"], "490": ["n03000247", "chain_mail"], "491": ["n03000684", "chain_saw"], "492": ["n03014705", "chest"], "493": ["n03016953", "chiffonier"], "494": ["n03017168", "chime"], "495": ["n03018349", "china_cabinet"], "496": ["n03026506", "Christmas_stocking"], "497": ["n03028079", "church"], "498": ["n03032252", "cinema"], "499": ["n03041632", "cleaver"], "500": ["n03042490", "cliff_dwelling"], "501": ["n03045698", "cloak"], "502": ["n03047690", "clog"], "503": ["n03062245", "cocktail_shaker"], "504": ["n03063599", "coffee_mug"], "505": ["n03063689", "coffeepot"], "506": ["n03065424", "coil"], "507": ["n03075370", "combination_lock"], "508": ["n03085013", "computer_keyboard"], "509": ["n03089624", "confectionery"], "510": ["n03095699", "container_ship"], "511": ["n03100240", "convertible"], "512": ["n03109150", "corkscrew"], "513": ["n03110669", "cornet"], "514": ["n03124043", "cowboy_boot"], "515": ["n03124170", "cowboy_hat"], "516": ["n03125729", "cradle"], "517": ["n03126707", "crane"], "518": ["n03127747", "crash_helmet"], "519": ["n03127925", "crate"], "520": ["n03131574", "crib"], "521": ["n03133878", "Crock_Pot"], "522": ["n03134739", "croquet_ball"], "523": ["n03141823", "crutch"], "524": ["n03146219", "cuirass"], "525": ["n03160309", "dam"], "526": ["n03179701", "desk"], "527": ["n03180011", "desktop_computer"], "528": ["n03187595", "dial_telephone"], "529": ["n03188531", "diaper"], "530": ["n03196217", "digital_clock"], "531": ["n03197337", "digital_watch"], "532": ["n03201208", "dining_table"], "533": ["n03207743", "dishrag"], "534": ["n03207941", "dishwasher"], "535": ["n03208938", "disk_brake"], "536": ["n03216828", "dock"], "537": ["n03218198", "dogsled"], "538": ["n03220513", "dome"], "539": ["n03223299", "doormat"], "540": ["n03240683", "drilling_platform"], "541": ["n03249569", "drum"], "542": ["n03250847", "drumstick"], "543": ["n03255030", "dumbbell"], "544": ["n03259280", "Dutch_oven"], "545": ["n03271574", "electric_fan"], "546": ["n03272010", "electric_guitar"], "547": ["n03272562", "electric_locomotive"], "548": ["n03290653", "entertainment_center"], "549": ["n03291819", "envelope"], "550": ["n03297495", "espresso_maker"], "551": ["n03314780", "face_powder"], "552": ["n03325584", "feather_boa"], "553": ["n03337140", "file"], "554": ["n03344393", "fireboat"], "555": ["n03345487", "fire_engine"], "556": ["n03347037", "fire_screen"], "557": ["n03355925", "flagpole"], "558": ["n03372029", "flute"], "559": ["n03376595", "folding_chair"], "560": ["n03379051", "football_helmet"], "561": ["n03384352", "forklift"], "562": ["n03388043", "fountain"], "563": ["n03388183", "fountain_pen"], "564": ["n03388549", "four-poster"], "565": ["n03393912", "freight_car"], "566": ["n03394916", "French_horn"], "567": ["n03400231", "frying_pan"], "568": ["n03404251", "fur_coat"], "569": ["n03417042", "garbage_truck"], "570": ["n03424325", "gasmask"], "571": ["n03425413", "gas_pump"], "572": ["n03443371", "goblet"], "573": ["n03444034", "go-kart"], "574": ["n03445777", "golf_ball"], "575": ["n03445924", "golfcart"], "576": ["n03447447", "gondola"], "577": ["n03447721", "gong"], "578": ["n03450230", "gown"], "579": ["n03452741", "grand_piano"], "580": ["n03457902", "greenhouse"], "581": ["n03459775", "grille"], "582": ["n03461385", "grocery_store"], "583": ["n03467068", "guillotine"], "584": ["n03476684", "hair_slide"], "585": ["n03476991", "hair_spray"], "586": ["n03478589", "half_track"], "587": ["n03481172", "hammer"], "588": ["n03482405", "hamper"], "589": ["n03483316", "hand_blower"], "590": ["n03485407", "hand-held_computer"], "591": ["n03485794", "handkerchief"], "592": ["n03492542", "hard_disc"], "593": ["n03494278", "harmonica"], "594": ["n03495258", "harp"], "595": ["n03496892", "harvester"], "596": ["n03498962", "hatchet"], "597": ["n03527444", "holster"], "598": ["n03529860", "home_theater"], "599": ["n03530642", "honeycomb"], "600": ["n03532672", "hook"], "601": ["n03534580", "hoopskirt"], "602": ["n03535780", "horizontal_bar"], "603": ["n03538406", "horse_cart"], "604": ["n03544143", "hourglass"], "605": ["n03584254", "iPod"], "606": ["n03584829", "iron"], "607": ["n03590841", "jack-o'-lantern"], "608": ["n03594734", "jean"], "609": ["n03594945", "jeep"], "610": ["n03595614", "jersey"], "611": ["n03598930", "jigsaw_puzzle"], "612": ["n03599486", "jinrikisha"], "613": ["n03602883", "joystick"], "614": ["n03617480", "kimono"], "615": ["n03623198", "knee_pad"], "616": ["n03627232", "knot"], "617": ["n03630383", "lab_coat"], "618": ["n03633091", "ladle"], "619": ["n03637318", "lampshade"], "620": ["n03642806", "laptop"], "621": ["n03649909", "lawn_mower"], "622": ["n03657121", "lens_cap"], "623": ["n03658185", "letter_opener"], "624": ["n03661043", "library"], "625": ["n03662601", "lifeboat"], "626": ["n03666591", "lighter"], "627": ["n03670208", "limousine"], "628": ["n03673027", "liner"], "629": ["n03676483", "lipstick"], "630": ["n03680355", "Loafer"], "631": ["n03690938", "lotion"], "632": ["n03691459", "loudspeaker"], "633": ["n03692522", "loupe"], "634": ["n03697007", "lumbermill"], "635": ["n03706229", "magnetic_compass"], "636": ["n03709823", "mailbag"], "637": ["n03710193", "mailbox"], "638": ["n03710637", "maillot"], "639": ["n03710721", "maillot"], "640": ["n03717622", "manhole_cover"], "641": ["n03720891", "maraca"], "642": ["n03721384", "marimba"], "643": ["n03724870", "mask"], "644": ["n03729826", "matchstick"], "645": ["n03733131", "maypole"], "646": ["n03733281", "maze"], "647": ["n03733805", "measuring_cup"], "648": ["n03742115", "medicine_chest"], "649": ["n03743016", "megalith"], "650": ["n03759954", "microphone"], "651": ["n03761084", "microwave"], "652": ["n03763968", "military_uniform"], "653": ["n03764736", "milk_can"], "654": ["n03769881", "minibus"], "655": ["n03770439", "miniskirt"], "656": ["n03770679", "minivan"], "657": ["n03773504", "missile"], "658": ["n03775071", "mitten"], "659": ["n03775546", "mixing_bowl"], "660": ["n03776460", "mobile_home"], "661": ["n03777568", "Model_T"], "662": ["n03777754", "modem"], "663": ["n03781244", "monastery"], "664": ["n03782006", "monitor"], "665": ["n03785016", "moped"], "666": ["n03786901", "mortar"], "667": ["n03787032", "mortarboard"], "668": ["n03788195", "mosque"], "669": ["n03788365", "mosquito_net"], "670": ["n03791053", "motor_scooter"], "671": ["n03792782", "mountain_bike"], "672": ["n03792972", "mountain_tent"], "673": ["n03793489", "mouse"], "674": ["n03794056", "mousetrap"], "675": ["n03796401", "moving_van"], "676": ["n03803284", "muzzle"], "677": ["n03804744", "nail"], "678": ["n03814639", "neck_brace"], "679": ["n03814906", "necklace"], "680": ["n03825788", "nipple"], "681": ["n03832673", "notebook"], "682": ["n03837869", "obelisk"], "683": ["n03838899", "oboe"], "684": ["n03840681", "ocarina"], "685": ["n03841143", "odometer"], "686": ["n03843555", "oil_filter"], "687": ["n03854065", "organ"], "688": ["n03857828", "oscilloscope"], "689": ["n03866082", "overskirt"], "690": ["n03868242", "oxcart"], "691": ["n03868863", "oxygen_mask"], "692": ["n03871628", "packet"], "693": ["n03873416", "paddle"], "694": ["n03874293", "paddlewheel"], "695": ["n03874599", "padlock"], "696": ["n03876231", "paintbrush"], "697": ["n03877472", "pajama"], "698": ["n03877845", "palace"], "699": ["n03884397", "panpipe"], "700": ["n03887697", "paper_towel"], "701": ["n03888257", "parachute"], "702": ["n03888605", "parallel_bars"], "703": ["n03891251", "park_bench"], "704": ["n03891332", "parking_meter"], "705": ["n03895866", "passenger_car"], "706": ["n03899768", "patio"], "707": ["n03902125", "pay-phone"], "708": ["n03903868", "pedestal"], "709": ["n03908618", "pencil_box"], "710": ["n03908714", "pencil_sharpener"], "711": ["n03916031", "perfume"], "712": ["n03920288", "Petri_dish"], "713": ["n03924679", "photocopier"], "714": ["n03929660", "pick"], "715": ["n03929855", "pickelhaube"], "716": ["n03930313", "picket_fence"], "717": ["n03930630", "pickup"], "718": ["n03933933", "pier"], "719": ["n03935335", "piggy_bank"], "720": ["n03937543", "pill_bottle"], "721": ["n03938244", "pillow"], "722": ["n03942813", "ping-pong_ball"], "723": ["n03944341", "pinwheel"], "724": ["n03947888", "pirate"], "725": ["n03950228", "pitcher"], "726": ["n03954731", "plane"], "727": ["n03956157", "planetarium"], "728": ["n03958227", "plastic_bag"], "729": ["n03961711", "plate_rack"], "730": ["n03967562", "plow"], "731": ["n03970156", "plunger"], "732": ["n03976467", "Polaroid_camera"], "733": ["n03976657", "pole"], "734": ["n03977966", "police_van"], "735": ["n03980874", "poncho"], "736": ["n03982430", "pool_table"], "737": ["n03983396", "pop_bottle"], "738": ["n03991062", "pot"], "739": ["n03992509", "potter's_wheel"], "740": ["n03995372", "power_drill"], "741": ["n03998194", "prayer_rug"], "742": ["n04004767", "printer"], "743": ["n04005630", "prison"], "744": ["n04008634", "projectile"], "745": ["n04009552", "projector"], "746": ["n04019541", "puck"], "747": ["n04023962", "punching_bag"], "748": ["n04026417", "purse"], "749": ["n04033901", "quill"], "750": ["n04033995", "quilt"], "751": ["n04037443", "racer"], "752": ["n04039381", "racket"], "753": ["n04040759", "radiator"], "754": ["n04041544", "radio"], "755": ["n04044716", "radio_telescope"], "756": ["n04049303", "rain_barrel"], "757": ["n04065272", "recreational_vehicle"], "758": ["n04067472", "reel"], "759": ["n04069434", "reflex_camera"], "760": ["n04070727", "refrigerator"], "761": ["n04074963", "remote_control"], "762": ["n04081281", "restaurant"], "763": ["n04086273", "revolver"], "764": ["n04090263", "rifle"], "765": ["n04099969", "rocking_chair"], "766": ["n04111531", "rotisserie"], "767": ["n04116512", "rubber_eraser"], "768": ["n04118538", "rugby_ball"], "769": ["n04118776", "rule"], "770": ["n04120489", "running_shoe"], "771": ["n04125021", "safe"], "772": ["n04127249", "safety_pin"], "773": ["n04131690", "saltshaker"], "774": ["n04133789", "sandal"], "775": ["n04136333", "sarong"], "776": ["n04141076", "sax"], "777": ["n04141327", "scabbard"], "778": ["n04141975", "scale"], "779": ["n04146614", "school_bus"], "780": ["n04147183", "schooner"], "781": ["n04149813", "scoreboard"], "782": ["n04152593", "screen"], "783": ["n04153751", "screw"], "784": ["n04154565", "screwdriver"], "785": ["n04162706", "seat_belt"], "786": ["n04179913", "sewing_machine"], "787": ["n04192698", "shield"], "788": ["n04200800", "shoe_shop"], "789": ["n04201297", "shoji"], "790": ["n04204238", "shopping_basket"], "791": ["n04204347", "shopping_cart"], "792": ["n04208210", "shovel"], "793": ["n04209133", "shower_cap"], "794": ["n04209239", "shower_curtain"], "795": ["n04228054", "ski"], "796": ["n04229816", "ski_mask"], "797": ["n04235860", "sleeping_bag"], "798": ["n04238763", "slide_rule"], "799": ["n04239074", "sliding_door"], "800": ["n04243546", "slot"], "801": ["n04251144", "snorkel"], "802": ["n04252077", "snowmobile"], "803": ["n04252225", "snowplow"], "804": ["n04254120", "soap_dispenser"], "805": ["n04254680", "soccer_ball"], "806": ["n04254777", "sock"], "807": ["n04258138", "solar_dish"], "808": ["n04259630", "sombrero"], "809": ["n04263257", "soup_bowl"], "810": ["n04264628", "space_bar"], "811": ["n04265275", "space_heater"], "812": ["n04266014", "space_shuttle"], "813": ["n04270147", "spatula"], "814": ["n04273569", "speedboat"], "815": ["n04275548", "spider_web"], "816": ["n04277352", "spindle"], "817": ["n04285008", "sports_car"], "818": ["n04286575", "spotlight"], "819": ["n04296562", "stage"], "820": ["n04310018", "steam_locomotive"], "821": ["n04311004", "steel_arch_bridge"], "822": ["n04311174", "steel_drum"], "823": ["n04317175", "stethoscope"], "824": ["n04325704", "stole"], "825": ["n04326547", "stone_wall"], "826": ["n04328186", "stopwatch"], "827": ["n04330267", "stove"], "828": ["n04332243", "strainer"], "829": ["n04335435", "streetcar"], "830": ["n04336792", "stretcher"], "831": ["n04344873", "studio_couch"], "832": ["n04346328", "stupa"], "833": ["n04347754", "submarine"], "834": ["n04350905", "suit"], "835": ["n04355338", "sundial"], "836": ["n04355933", "sunglass"], "837": ["n04356056", "sunglasses"], "838": ["n04357314", "sunscreen"], "839": ["n04366367", "suspension_bridge"], "840": ["n04367480", "swab"], "841": ["n04370456", "sweatshirt"], "842": ["n04371430", "swimming_trunks"], "843": ["n04371774", "swing"], "844": ["n04372370", "switch"], "845": ["n04376876", "syringe"], "846": ["n04380533", "table_lamp"], "847": ["n04389033", "tank"], "848": ["n04392985", "tape_player"], "849": ["n04398044", "teapot"], "850": ["n04399382", "teddy"], "851": ["n04404412", "television"], "852": ["n04409515", "tennis_ball"], "853": ["n04417672", "thatch"], "854": ["n04418357", "theater_curtain"], "855": ["n04423845", "thimble"], "856": ["n04428191", "thresher"], "857": ["n04429376", "throne"], "858": ["n04435653", "tile_roof"], "859": ["n04442312", "toaster"], "860": ["n04443257", "tobacco_shop"], "861": ["n04447861", "toilet_seat"], "862": ["n04456115", "torch"], "863": ["n04458633", "totem_pole"], "864": ["n04461696", "tow_truck"], "865": ["n04462240", "toyshop"], "866": ["n04465501", "tractor"], "867": ["n04467665", "trailer_truck"], "868": ["n04476259", "tray"], "869": ["n04479046", "trench_coat"], "870": ["n04482393", "tricycle"], "871": ["n04483307", "trimaran"], "872": ["n04485082", "tripod"], "873": ["n04486054", "triumphal_arch"], "874": ["n04487081", "trolleybus"], "875": ["n04487394", "trombone"], "876": ["n04493381", "tub"], "877": ["n04501370", "turnstile"], "878": ["n04505470", "typewriter_keyboard"], "879": ["n04507155", "umbrella"], "880": ["n04509417", "unicycle"], "881": ["n04515003", "upright"], "882": ["n04517823", "vacuum"], "883": ["n04522168", "vase"], "884": ["n04523525", "vault"], "885": ["n04525038", "velvet"], "886": ["n04525305", "vending_machine"], "887": ["n04532106", "vestment"], "888": ["n04532670", "viaduct"], "889": ["n04536866", "violin"], "890": ["n04540053", "volleyball"], "891": ["n04542943", "waffle_iron"], "892": ["n04548280", "wall_clock"], "893": ["n04548362", "wallet"], "894": ["n04550184", "wardrobe"], "895": ["n04552348", "warplane"], "896": ["n04553703", "washbasin"], "897": ["n04554684", "washer"], "898": ["n04557648", "water_bottle"], "899": ["n04560804", "water_jug"], "900": ["n04562935", "water_tower"], "901": ["n04579145", "whiskey_jug"], "902": ["n04579432", "whistle"], "903": ["n04584207", "wig"], "904": ["n04589890", "window_screen"], "905": ["n04590129", "window_shade"], "906": ["n04591157", "Windsor_tie"], "907": ["n04591713", "wine_bottle"], "908": ["n04592741", "wing"], "909": ["n04596742", "wok"], "910": ["n04597913", "wooden_spoon"], "911": ["n04599235", "wool"], "912": ["n04604644", "worm_fence"], "913": ["n04606251", "wreck"], "914": ["n04612504", "yawl"], "915": ["n04613696", "yurt"], "916": ["n06359193", "web_site"], "917": ["n06596364", "comic_book"], "918": ["n06785654", "crossword_puzzle"], "919": ["n06794110", "street_sign"], "920": ["n06874185", "traffic_light"], "921": ["n07248320", "book_jacket"], "922": ["n07565083", "menu"], "923": ["n07579787", "plate"], "924": ["n07583066", "guacamole"], "925": ["n07584110", "consomme"], "926": ["n07590611", "hot_pot"], "927": ["n07613480", "trifle"], "928": ["n07614500", "ice_cream"], "929": ["n07615774", "ice_lolly"], "930": ["n07684084", "French_loaf"], "931": ["n07693725", "bagel"], "932": ["n07695742", "pretzel"], "933": ["n07697313", "cheeseburger"], "934": ["n07697537", "hotdog"], "935": ["n07711569", "mashed_potato"], "936": ["n07714571", "head_cabbage"], "937": ["n07714990", "broccoli"], "938": ["n07715103", "cauliflower"], "939": ["n07716358", "zucchini"], "940": ["n07716906", "spaghetti_squash"], "941": ["n07717410", "acorn_squash"], "942": ["n07717556", "butternut_squash"], "943": ["n07718472", "cucumber"], "944": ["n07718747", "artichoke"], "945": ["n07720875", "bell_pepper"], "946": ["n07730033", "cardoon"], "947": ["n07734744", "mushroom"], "948": ["n07742313", "Granny_Smith"], "949": ["n07745940", "strawberry"], "950": ["n07747607", "orange"], "951": ["n07749582", "lemon"], "952": ["n07753113", "fig"], "953": ["n07753275", "pineapple"], "954": ["n07753592", "banana"], "955": ["n07754684", "jackfruit"], "956": ["n07760859", "custard_apple"], "957": ["n07768694", "pomegranate"], "958": ["n07802026", "hay"], "959": ["n07831146", "carbonara"], "960": ["n07836838", "chocolate_sauce"], "961": ["n07860988", "dough"], "962": ["n07871810", "meat_loaf"], "963": ["n07873807", "pizza"], "964": ["n07875152", "potpie"], "965": ["n07880968", "burrito"], "966": ["n07892512", "red_wine"], "967": ["n07920052", "espresso"], "968": ["n07930864", "cup"], "969": ["n07932039", "eggnog"], "970": ["n09193705", "alp"], "971": ["n09229709", "bubble"], "972": ["n09246464", "cliff"], "973": ["n09256479", "coral_reef"], "974": ["n09288635", "geyser"], "975": ["n09332890", "lakeside"], "976": ["n09399592", "promontory"], "977": ["n09421951", "sandbar"], "978": ["n09428293", "seashore"], "979": ["n09468604", "valley"], "980": ["n09472597", "volcano"], "981": ["n09835506", "ballplayer"], "982": ["n10148035", "groom"], "983": ["n10565667", "scuba_diver"], "984": ["n11879895", "rapeseed"], "985": ["n11939491", "daisy"], "986": ["n12057211", "yellow_lady's_slipper"], "987": ["n12144580", "corn"], "988": ["n12267677", "acorn"], "989": ["n12620546", "hip"], "990": ["n12768682", "buckeye"], "991": ["n12985857", "coral_fungus"], "992": ["n12998815", "agaric"], "993": ["n13037406", "gyromitra"], "994": ["n13040303", "stinkhorn"], "995": ["n13044778", "earthstar"], "996": ["n13052670", "hen-of-the-woods"], "997": ["n13054560", "bolete"], "998": ["n13133613", "ear"], "999": ["n15075141", "toilet_tissue"]} -------------------------------------------------------------------------------- /differential_evolution.py: -------------------------------------------------------------------------------- 1 | """ 2 | A slight modification to Scipy's implementation of differential evolution. To speed up predictions, the entire parameters array is passed to `self.func`, where a neural network model can batch its computations and execute in parallel. Search for `CHANGES` to find all code changes. 3 | 4 | Dan Kondratyuk 2018 5 | 6 | Original code adapted from 7 | https://github.com/scipy/scipy/blob/70e61dee181de23fdd8d893eaa9491100e2218d7/scipy/optimize/_differentialevolution.py 8 | ---------- 9 | 10 | differential_evolution: The differential evolution global optimization algorithm 11 | Added by Andrew Nelson 2014 12 | """ 13 | from __future__ import division, print_function, absolute_import 14 | import numpy as np 15 | from scipy.optimize import OptimizeResult, minimize 16 | from scipy.optimize.optimize import _status_message 17 | from scipy._lib._util import check_random_state 18 | from scipy._lib.six import xrange, string_types 19 | import warnings 20 | 21 | 22 | __all__ = ['differential_evolution'] 23 | 24 | _MACHEPS = np.finfo(np.float64).eps 25 | 26 | 27 | def differential_evolution(func, bounds, args=(), strategy='best1bin', 28 | maxiter=1000, popsize=15, tol=0.01, 29 | mutation=(0.5, 1), recombination=0.7, seed=None, 30 | callback=None, disp=False, polish=True, 31 | init='latinhypercube', atol=0): 32 | """Finds the global minimum of a multivariate function. 33 | Differential Evolution is stochastic in nature (does not use gradient 34 | methods) to find the minimium, and can search large areas of candidate 35 | space, but often requires larger numbers of function evaluations than 36 | conventional gradient based techniques. 37 | The algorithm is due to Storn and Price [1]_. 38 | Parameters 39 | ---------- 40 | func : callable 41 | The objective function to be minimized. Must be in the form 42 | ``f(x, *args)``, where ``x`` is the argument in the form of a 1-D array 43 | and ``args`` is a tuple of any additional fixed parameters needed to 44 | completely specify the function. 45 | bounds : sequence 46 | Bounds for variables. ``(min, max)`` pairs for each element in ``x``, 47 | defining the lower and upper bounds for the optimizing argument of 48 | `func`. It is required to have ``len(bounds) == len(x)``. 49 | ``len(bounds)`` is used to determine the number of parameters in ``x``. 50 | args : tuple, optional 51 | Any additional fixed parameters needed to 52 | completely specify the objective function. 53 | strategy : str, optional 54 | The differential evolution strategy to use. Should be one of: 55 | - 'best1bin' 56 | - 'best1exp' 57 | - 'rand1exp' 58 | - 'randtobest1exp' 59 | - 'currenttobest1exp' 60 | - 'best2exp' 61 | - 'rand2exp' 62 | - 'randtobest1bin' 63 | - 'currenttobest1bin' 64 | - 'best2bin' 65 | - 'rand2bin' 66 | - 'rand1bin' 67 | The default is 'best1bin'. 68 | maxiter : int, optional 69 | The maximum number of generations over which the entire population is 70 | evolved. The maximum number of function evaluations (with no polishing) 71 | is: ``(maxiter + 1) * popsize * len(x)`` 72 | popsize : int, optional 73 | A multiplier for setting the total population size. The population has 74 | ``popsize * len(x)`` individuals (unless the initial population is 75 | supplied via the `init` keyword). 76 | tol : float, optional 77 | Relative tolerance for convergence, the solving stops when 78 | ``np.std(pop) <= atol + tol * np.abs(np.mean(population_energies))``, 79 | where and `atol` and `tol` are the absolute and relative tolerance 80 | respectively. 81 | mutation : float or tuple(float, float), optional 82 | The mutation constant. In the literature this is also known as 83 | differential weight, being denoted by F. 84 | If specified as a float it should be in the range [0, 2]. 85 | If specified as a tuple ``(min, max)`` dithering is employed. Dithering 86 | randomly changes the mutation constant on a generation by generation 87 | basis. The mutation constant for that generation is taken from 88 | ``U[min, max)``. Dithering can help speed convergence significantly. 89 | Increasing the mutation constant increases the search radius, but will 90 | slow down convergence. 91 | recombination : float, optional 92 | The recombination constant, should be in the range [0, 1]. In the 93 | literature this is also known as the crossover probability, being 94 | denoted by CR. Increasing this value allows a larger number of mutants 95 | to progress into the next generation, but at the risk of population 96 | stability. 97 | seed : int or `np.random.RandomState`, optional 98 | If `seed` is not specified the `np.RandomState` singleton is used. 99 | If `seed` is an int, a new `np.random.RandomState` instance is used, 100 | seeded with seed. 101 | If `seed` is already a `np.random.RandomState instance`, then that 102 | `np.random.RandomState` instance is used. 103 | Specify `seed` for repeatable minimizations. 104 | disp : bool, optional 105 | Display status messages 106 | callback : callable, `callback(xk, convergence=val)`, optional 107 | A function to follow the progress of the minimization. ``xk`` is 108 | the current value of ``x0``. ``val`` represents the fractional 109 | value of the population convergence. When ``val`` is greater than one 110 | the function halts. If callback returns `True`, then the minimization 111 | is halted (any polishing is still carried out). 112 | polish : bool, optional 113 | If True (default), then `scipy.optimize.minimize` with the `L-BFGS-B` 114 | method is used to polish the best population member at the end, which 115 | can improve the minimization slightly. 116 | init : str or array-like, optional 117 | Specify which type of population initialization is performed. Should be 118 | one of: 119 | - 'latinhypercube' 120 | - 'random' 121 | - array specifying the initial population. The array should have 122 | shape ``(M, len(x))``, where len(x) is the number of parameters. 123 | `init` is clipped to `bounds` before use. 124 | The default is 'latinhypercube'. Latin Hypercube sampling tries to 125 | maximize coverage of the available parameter space. 'random' 126 | initializes the population randomly - this has the drawback that 127 | clustering can occur, preventing the whole of parameter space being 128 | covered. Use of an array to specify a population subset could be used, 129 | for example, to create a tight bunch of initial guesses in an location 130 | where the solution is known to exist, thereby reducing time for 131 | convergence. 132 | atol : float, optional 133 | Absolute tolerance for convergence, the solving stops when 134 | ``np.std(pop) <= atol + tol * np.abs(np.mean(population_energies))``, 135 | where and `atol` and `tol` are the absolute and relative tolerance 136 | respectively. 137 | Returns 138 | ------- 139 | res : OptimizeResult 140 | The optimization result represented as a `OptimizeResult` object. 141 | Important attributes are: ``x`` the solution array, ``success`` a 142 | Boolean flag indicating if the optimizer exited successfully and 143 | ``message`` which describes the cause of the termination. See 144 | `OptimizeResult` for a description of other attributes. If `polish` 145 | was employed, and a lower minimum was obtained by the polishing, then 146 | OptimizeResult also contains the ``jac`` attribute. 147 | Notes 148 | ----- 149 | Differential evolution is a stochastic population based method that is 150 | useful for global optimization problems. At each pass through the population 151 | the algorithm mutates each candidate solution by mixing with other candidate 152 | solutions to create a trial candidate. There are several strategies [2]_ for 153 | creating trial candidates, which suit some problems more than others. The 154 | 'best1bin' strategy is a good starting point for many systems. In this 155 | strategy two members of the population are randomly chosen. Their difference 156 | is used to mutate the best member (the `best` in `best1bin`), :math:`b_0`, 157 | so far: 158 | .. math:: 159 | b' = b_0 + mutation * (population[rand0] - population[rand1]) 160 | A trial vector is then constructed. Starting with a randomly chosen 'i'th 161 | parameter the trial is sequentially filled (in modulo) with parameters from 162 | `b'` or the original candidate. The choice of whether to use `b'` or the 163 | original candidate is made with a binomial distribution (the 'bin' in 164 | 'best1bin') - a random number in [0, 1) is generated. If this number is 165 | less than the `recombination` constant then the parameter is loaded from 166 | `b'`, otherwise it is loaded from the original candidate. The final 167 | parameter is always loaded from `b'`. Once the trial candidate is built 168 | its fitness is assessed. If the trial is better than the original candidate 169 | then it takes its place. If it is also better than the best overall 170 | candidate it also replaces that. 171 | To improve your chances of finding a global minimum use higher `popsize` 172 | values, with higher `mutation` and (dithering), but lower `recombination` 173 | values. This has the effect of widening the search radius, but slowing 174 | convergence. 175 | .. versionadded:: 0.15.0 176 | Examples 177 | -------- 178 | Let us consider the problem of minimizing the Rosenbrock function. This 179 | function is implemented in `rosen` in `scipy.optimize`. 180 | >>> from scipy.optimize import rosen, differential_evolution 181 | >>> bounds = [(0,2), (0, 2), (0, 2), (0, 2), (0, 2)] 182 | >>> result = differential_evolution(rosen, bounds) 183 | >>> result.x, result.fun 184 | (array([1., 1., 1., 1., 1.]), 1.9216496320061384e-19) 185 | Next find the minimum of the Ackley function 186 | (http://en.wikipedia.org/wiki/Test_functions_for_optimization). 187 | >>> from scipy.optimize import differential_evolution 188 | >>> import numpy as np 189 | >>> def ackley(x): 190 | ... arg1 = -0.2 * np.sqrt(0.5 * (x[0] ** 2 + x[1] ** 2)) 191 | ... arg2 = 0.5 * (np.cos(2. * np.pi * x[0]) + np.cos(2. * np.pi * x[1])) 192 | ... return -20. * np.exp(arg1) - np.exp(arg2) + 20. + np.e 193 | >>> bounds = [(-5, 5), (-5, 5)] 194 | >>> result = differential_evolution(ackley, bounds) 195 | >>> result.x, result.fun 196 | (array([ 0., 0.]), 4.4408920985006262e-16) 197 | References 198 | ---------- 199 | .. [1] Storn, R and Price, K, Differential Evolution - a Simple and 200 | Efficient Heuristic for Global Optimization over Continuous Spaces, 201 | Journal of Global Optimization, 1997, 11, 341 - 359. 202 | .. [2] http://www1.icsi.berkeley.edu/~storn/code.html 203 | .. [3] http://en.wikipedia.org/wiki/Differential_evolution 204 | """ 205 | 206 | solver = DifferentialEvolutionSolver(func, bounds, args=args, 207 | strategy=strategy, maxiter=maxiter, 208 | popsize=popsize, tol=tol, 209 | mutation=mutation, 210 | recombination=recombination, 211 | seed=seed, polish=polish, 212 | callback=callback, 213 | disp=disp, init=init, atol=atol) 214 | return solver.solve() 215 | 216 | 217 | class DifferentialEvolutionSolver(object): 218 | 219 | """This class implements the differential evolution solver 220 | Parameters 221 | ---------- 222 | func : callable 223 | The objective function to be minimized. Must be in the form 224 | ``f(x, *args)``, where ``x`` is the argument in the form of a 1-D array 225 | and ``args`` is a tuple of any additional fixed parameters needed to 226 | completely specify the function. 227 | bounds : sequence 228 | Bounds for variables. ``(min, max)`` pairs for each element in ``x``, 229 | defining the lower and upper bounds for the optimizing argument of 230 | `func`. It is required to have ``len(bounds) == len(x)``. 231 | ``len(bounds)`` is used to determine the number of parameters in ``x``. 232 | args : tuple, optional 233 | Any additional fixed parameters needed to 234 | completely specify the objective function. 235 | strategy : str, optional 236 | The differential evolution strategy to use. Should be one of: 237 | - 'best1bin' 238 | - 'best1exp' 239 | - 'rand1exp' 240 | - 'randtobest1exp' 241 | - 'currenttobest1exp' 242 | - 'best2exp' 243 | - 'rand2exp' 244 | - 'randtobest1bin' 245 | - 'currenttobest1bin' 246 | - 'best2bin' 247 | - 'rand2bin' 248 | - 'rand1bin' 249 | The default is 'best1bin' 250 | maxiter : int, optional 251 | The maximum number of generations over which the entire population is 252 | evolved. The maximum number of function evaluations (with no polishing) 253 | is: ``(maxiter + 1) * popsize * len(x)`` 254 | popsize : int, optional 255 | A multiplier for setting the total population size. The population has 256 | ``popsize * len(x)`` individuals (unless the initial population is 257 | supplied via the `init` keyword). 258 | tol : float, optional 259 | Relative tolerance for convergence, the solving stops when 260 | ``np.std(pop) <= atol + tol * np.abs(np.mean(population_energies))``, 261 | where and `atol` and `tol` are the absolute and relative tolerance 262 | respectively. 263 | mutation : float or tuple(float, float), optional 264 | The mutation constant. In the literature this is also known as 265 | differential weight, being denoted by F. 266 | If specified as a float it should be in the range [0, 2]. 267 | If specified as a tuple ``(min, max)`` dithering is employed. Dithering 268 | randomly changes the mutation constant on a generation by generation 269 | basis. The mutation constant for that generation is taken from 270 | U[min, max). Dithering can help speed convergence significantly. 271 | Increasing the mutation constant increases the search radius, but will 272 | slow down convergence. 273 | recombination : float, optional 274 | The recombination constant, should be in the range [0, 1]. In the 275 | literature this is also known as the crossover probability, being 276 | denoted by CR. Increasing this value allows a larger number of mutants 277 | to progress into the next generation, but at the risk of population 278 | stability. 279 | seed : int or `np.random.RandomState`, optional 280 | If `seed` is not specified the `np.random.RandomState` singleton is 281 | used. 282 | If `seed` is an int, a new `np.random.RandomState` instance is used, 283 | seeded with `seed`. 284 | If `seed` is already a `np.random.RandomState` instance, then that 285 | `np.random.RandomState` instance is used. 286 | Specify `seed` for repeatable minimizations. 287 | disp : bool, optional 288 | Display status messages 289 | callback : callable, `callback(xk, convergence=val)`, optional 290 | A function to follow the progress of the minimization. ``xk`` is 291 | the current value of ``x0``. ``val`` represents the fractional 292 | value of the population convergence. When ``val`` is greater than one 293 | the function halts. If callback returns `True`, then the minimization 294 | is halted (any polishing is still carried out). 295 | polish : bool, optional 296 | If True, then `scipy.optimize.minimize` with the `L-BFGS-B` method 297 | is used to polish the best population member at the end. This requires 298 | a few more function evaluations. 299 | maxfun : int, optional 300 | Set the maximum number of function evaluations. However, it probably 301 | makes more sense to set `maxiter` instead. 302 | init : str or array-like, optional 303 | Specify which type of population initialization is performed. Should be 304 | one of: 305 | - 'latinhypercube' 306 | - 'random' 307 | - array specifying the initial population. The array should have 308 | shape ``(M, len(x))``, where len(x) is the number of parameters. 309 | `init` is clipped to `bounds` before use. 310 | The default is 'latinhypercube'. Latin Hypercube sampling tries to 311 | maximize coverage of the available parameter space. 'random' 312 | initializes the population randomly - this has the drawback that 313 | clustering can occur, preventing the whole of parameter space being 314 | covered. Use of an array to specify a population could be used, for 315 | example, to create a tight bunch of initial guesses in an location 316 | where the solution is known to exist, thereby reducing time for 317 | convergence. 318 | atol : float, optional 319 | Absolute tolerance for convergence, the solving stops when 320 | ``np.std(pop) <= atol + tol * np.abs(np.mean(population_energies))``, 321 | where and `atol` and `tol` are the absolute and relative tolerance 322 | respectively. 323 | """ 324 | 325 | # Dispatch of mutation strategy method (binomial or exponential). 326 | _binomial = {'best1bin': '_best1', 327 | 'randtobest1bin': '_randtobest1', 328 | 'currenttobest1bin': '_currenttobest1', 329 | 'best2bin': '_best2', 330 | 'rand2bin': '_rand2', 331 | 'rand1bin': '_rand1'} 332 | _exponential = {'best1exp': '_best1', 333 | 'rand1exp': '_rand1', 334 | 'randtobest1exp': '_randtobest1', 335 | 'currenttobest1exp': '_currenttobest1', 336 | 'best2exp': '_best2', 337 | 'rand2exp': '_rand2'} 338 | 339 | __init_error_msg = ("The population initialization method must be one of " 340 | "'latinhypercube' or 'random', or an array of shape " 341 | "(M, N) where N is the number of parameters and M>5") 342 | 343 | def __init__(self, func, bounds, args=(), 344 | strategy='best1bin', maxiter=1000, popsize=15, 345 | tol=0.01, mutation=(0.5, 1), recombination=0.7, seed=None, 346 | maxfun=np.inf, callback=None, disp=False, polish=True, 347 | init='latinhypercube', atol=0): 348 | 349 | if strategy in self._binomial: 350 | self.mutation_func = getattr(self, self._binomial[strategy]) 351 | elif strategy in self._exponential: 352 | self.mutation_func = getattr(self, self._exponential[strategy]) 353 | else: 354 | raise ValueError("Please select a valid mutation strategy") 355 | self.strategy = strategy 356 | 357 | self.callback = callback 358 | self.polish = polish 359 | 360 | # relative and absolute tolerances for convergence 361 | self.tol, self.atol = tol, atol 362 | 363 | # Mutation constant should be in [0, 2). If specified as a sequence 364 | # then dithering is performed. 365 | self.scale = mutation 366 | if (not np.all(np.isfinite(mutation)) or 367 | np.any(np.array(mutation) >= 2) or 368 | np.any(np.array(mutation) < 0)): 369 | raise ValueError('The mutation constant must be a float in ' 370 | 'U[0, 2), or specified as a tuple(min, max)' 371 | ' where min < max and min, max are in U[0, 2).') 372 | 373 | self.dither = None 374 | if hasattr(mutation, '__iter__') and len(mutation) > 1: 375 | self.dither = [mutation[0], mutation[1]] 376 | self.dither.sort() 377 | 378 | self.cross_over_probability = recombination 379 | 380 | self.func = func 381 | self.args = args 382 | 383 | # convert tuple of lower and upper bounds to limits 384 | # [(low_0, high_0), ..., (low_n, high_n] 385 | # -> [[low_0, ..., low_n], [high_0, ..., high_n]] 386 | self.limits = np.array(bounds, dtype='float').T 387 | if (np.size(self.limits, 0) != 2 or not 388 | np.all(np.isfinite(self.limits))): 389 | raise ValueError('bounds should be a sequence containing ' 390 | 'real valued (min, max) pairs for each value' 391 | ' in x') 392 | 393 | if maxiter is None: # the default used to be None 394 | maxiter = 1000 395 | self.maxiter = maxiter 396 | if maxfun is None: # the default used to be None 397 | maxfun = np.inf 398 | self.maxfun = maxfun 399 | 400 | # population is scaled to between [0, 1]. 401 | # We have to scale between parameter <-> population 402 | # save these arguments for _scale_parameter and 403 | # _unscale_parameter. This is an optimization 404 | self.__scale_arg1 = 0.5 * (self.limits[0] + self.limits[1]) 405 | self.__scale_arg2 = np.fabs(self.limits[0] - self.limits[1]) 406 | 407 | self.parameter_count = np.size(self.limits, 1) 408 | 409 | self.random_number_generator = check_random_state(seed) 410 | 411 | # default population initialization is a latin hypercube design, but 412 | # there are other population initializations possible. 413 | # the minimum is 5 because 'best2bin' requires a population that's at 414 | # least 5 long 415 | self.num_population_members = max(5, popsize * self.parameter_count) 416 | 417 | self.population_shape = (self.num_population_members, 418 | self.parameter_count) 419 | 420 | self._nfev = 0 421 | if isinstance(init, string_types): 422 | if init == 'latinhypercube': 423 | self.init_population_lhs() 424 | elif init == 'random': 425 | self.init_population_random() 426 | else: 427 | raise ValueError(self.__init_error_msg) 428 | else: 429 | self.init_population_array(init) 430 | 431 | self.disp = disp 432 | 433 | def init_population_lhs(self): 434 | """ 435 | Initializes the population with Latin Hypercube Sampling. 436 | Latin Hypercube Sampling ensures that each parameter is uniformly 437 | sampled over its range. 438 | """ 439 | rng = self.random_number_generator 440 | 441 | # Each parameter range needs to be sampled uniformly. The scaled 442 | # parameter range ([0, 1)) needs to be split into 443 | # `self.num_population_members` segments, each of which has the following 444 | # size: 445 | segsize = 1.0 / self.num_population_members 446 | 447 | # Within each segment we sample from a uniform random distribution. 448 | # We need to do this sampling for each parameter. 449 | samples = (segsize * rng.random_sample(self.population_shape) 450 | 451 | # Offset each segment to cover the entire parameter range [0, 1) 452 | + np.linspace(0., 1., self.num_population_members, 453 | endpoint=False)[:, np.newaxis]) 454 | 455 | # Create an array for population of candidate solutions. 456 | self.population = np.zeros_like(samples) 457 | 458 | # Initialize population of candidate solutions by permutation of the 459 | # random samples. 460 | for j in range(self.parameter_count): 461 | order = rng.permutation(range(self.num_population_members)) 462 | self.population[:, j] = samples[order, j] 463 | 464 | # reset population energies 465 | self.population_energies = (np.ones(self.num_population_members) * 466 | np.inf) 467 | 468 | # reset number of function evaluations counter 469 | self._nfev = 0 470 | 471 | def init_population_random(self): 472 | """ 473 | Initialises the population at random. This type of initialization 474 | can possess clustering, Latin Hypercube sampling is generally better. 475 | """ 476 | rng = self.random_number_generator 477 | self.population = rng.random_sample(self.population_shape) 478 | 479 | # reset population energies 480 | self.population_energies = (np.ones(self.num_population_members) * 481 | np.inf) 482 | 483 | # reset number of function evaluations counter 484 | self._nfev = 0 485 | 486 | def init_population_array(self, init): 487 | """ 488 | Initialises the population with a user specified population. 489 | Parameters 490 | ---------- 491 | init : np.ndarray 492 | Array specifying subset of the initial population. The array should 493 | have shape (M, len(x)), where len(x) is the number of parameters. 494 | The population is clipped to the lower and upper `bounds`. 495 | """ 496 | # make sure you're using a float array 497 | popn = np.asfarray(init) 498 | 499 | if (np.size(popn, 0) < 5 or 500 | popn.shape[1] != self.parameter_count or 501 | len(popn.shape) != 2): 502 | raise ValueError("The population supplied needs to have shape" 503 | " (M, len(x)), where M > 4.") 504 | 505 | # scale values and clip to bounds, assigning to population 506 | self.population = np.clip(self._unscale_parameters(popn), 0, 1) 507 | 508 | self.num_population_members = np.size(self.population, 0) 509 | 510 | self.population_shape = (self.num_population_members, 511 | self.parameter_count) 512 | 513 | # reset population energies 514 | self.population_energies = (np.ones(self.num_population_members) * 515 | np.inf) 516 | 517 | # reset number of function evaluations counter 518 | self._nfev = 0 519 | 520 | @property 521 | def x(self): 522 | """ 523 | The best solution from the solver 524 | Returns 525 | ------- 526 | x : ndarray 527 | The best solution from the solver. 528 | """ 529 | return self._scale_parameters(self.population[0]) 530 | 531 | @property 532 | def convergence(self): 533 | """ 534 | The standard deviation of the population energies divided by their 535 | mean. 536 | """ 537 | return (np.std(self.population_energies) / 538 | np.abs(np.mean(self.population_energies) + _MACHEPS)) 539 | 540 | def solve(self): 541 | """ 542 | Runs the DifferentialEvolutionSolver. 543 | Returns 544 | ------- 545 | res : OptimizeResult 546 | The optimization result represented as a ``OptimizeResult`` object. 547 | Important attributes are: ``x`` the solution array, ``success`` a 548 | Boolean flag indicating if the optimizer exited successfully and 549 | ``message`` which describes the cause of the termination. See 550 | `OptimizeResult` for a description of other attributes. If `polish` 551 | was employed, and a lower minimum was obtained by the polishing, 552 | then OptimizeResult also contains the ``jac`` attribute. 553 | """ 554 | nit, warning_flag = 0, False 555 | status_message = _status_message['success'] 556 | 557 | # The population may have just been initialized (all entries are 558 | # np.inf). If it has you have to calculate the initial energies. 559 | # Although this is also done in the evolve generator it's possible 560 | # that someone can set maxiter=0, at which point we still want the 561 | # initial energies to be calculated (the following loop isn't run). 562 | if np.all(np.isinf(self.population_energies)): 563 | self._calculate_population_energies() 564 | 565 | # do the optimisation. 566 | for nit in xrange(1, self.maxiter + 1): 567 | # evolve the population by a generation 568 | try: 569 | next(self) 570 | except StopIteration: 571 | warning_flag = True 572 | status_message = _status_message['maxfev'] 573 | break 574 | 575 | if self.disp: 576 | print("differential_evolution step %d: f(x)= %g" 577 | % (nit, 578 | self.population_energies[0])) 579 | 580 | # should the solver terminate? 581 | convergence = self.convergence 582 | 583 | if (self.callback and 584 | self.callback(self._scale_parameters(self.population[0]), 585 | convergence=self.tol / convergence) is True): 586 | 587 | warning_flag = True 588 | status_message = ('callback function requested stop early ' 589 | 'by returning True') 590 | break 591 | 592 | intol = (np.std(self.population_energies) <= 593 | self.atol + 594 | self.tol * np.abs(np.mean(self.population_energies))) 595 | if warning_flag or intol: 596 | break 597 | 598 | else: 599 | status_message = _status_message['maxiter'] 600 | warning_flag = True 601 | 602 | DE_result = OptimizeResult( 603 | x=self.x, 604 | fun=self.population_energies[0], 605 | nfev=self._nfev, 606 | nit=nit, 607 | message=status_message, 608 | success=(warning_flag is not True)) 609 | 610 | if self.polish: 611 | result = minimize(self.func, 612 | np.copy(DE_result.x), 613 | method='L-BFGS-B', 614 | bounds=self.limits.T, 615 | args=self.args) 616 | 617 | self._nfev += result.nfev 618 | DE_result.nfev = self._nfev 619 | 620 | if result.fun < DE_result.fun: 621 | DE_result.fun = result.fun 622 | DE_result.x = result.x 623 | DE_result.jac = result.jac 624 | # to keep internal state consistent 625 | self.population_energies[0] = result.fun 626 | self.population[0] = self._unscale_parameters(result.x) 627 | 628 | return DE_result 629 | 630 | def _calculate_population_energies(self): 631 | """ 632 | Calculate the energies of all the population members at the same time. 633 | Puts the best member in first place. Useful if the population has just 634 | been initialised. 635 | """ 636 | 637 | ############## 638 | ## CHANGES: self.func operates on the entire parameters array 639 | ############## 640 | itersize = max(0, min(len(self.population), self.maxfun - self._nfev + 1)) 641 | candidates = self.population[:itersize] 642 | parameters = np.array([self._scale_parameters(c) for c in candidates]) # TODO: can be vectorized 643 | energies = self.func(parameters, *self.args) 644 | self.population_energies = energies 645 | self._nfev += itersize 646 | 647 | # for index, candidate in enumerate(self.population): 648 | # if self._nfev > self.maxfun: 649 | # break 650 | 651 | # parameters = self._scale_parameters(candidate) 652 | # self.population_energies[index] = self.func(parameters, 653 | # *self.args) 654 | # self._nfev += 1 655 | 656 | ############## 657 | ############## 658 | 659 | 660 | 661 | minval = np.argmin(self.population_energies) 662 | 663 | # put the lowest energy into the best solution position. 664 | lowest_energy = self.population_energies[minval] 665 | self.population_energies[minval] = self.population_energies[0] 666 | self.population_energies[0] = lowest_energy 667 | 668 | self.population[[0, minval], :] = self.population[[minval, 0], :] 669 | 670 | def __iter__(self): 671 | return self 672 | 673 | def __next__(self): 674 | """ 675 | Evolve the population by a single generation 676 | Returns 677 | ------- 678 | x : ndarray 679 | The best solution from the solver. 680 | fun : float 681 | Value of objective function obtained from the best solution. 682 | """ 683 | # the population may have just been initialized (all entries are 684 | # np.inf). If it has you have to calculate the initial energies 685 | if np.all(np.isinf(self.population_energies)): 686 | self._calculate_population_energies() 687 | 688 | if self.dither is not None: 689 | self.scale = (self.random_number_generator.rand() 690 | * (self.dither[1] - self.dither[0]) + self.dither[0]) 691 | 692 | ############## 693 | ## CHANGES: self.func operates on the entire parameters array 694 | ############## 695 | 696 | itersize = max(0, min(self.num_population_members, self.maxfun - self._nfev + 1)) 697 | trials = np.array([self._mutate(c) for c in range(itersize)]) # TODO: can be vectorized 698 | for trial in trials: self._ensure_constraint(trial) 699 | parameters = np.array([self._scale_parameters(trial) for trial in trials]) 700 | energies = self.func(parameters, *self.args) 701 | self._nfev += itersize 702 | 703 | for candidate,(energy,trial) in enumerate(zip(energies, trials)): 704 | # if the energy of the trial candidate is lower than the 705 | # original population member then replace it 706 | if energy < self.population_energies[candidate]: 707 | self.population[candidate] = trial 708 | self.population_energies[candidate] = energy 709 | 710 | # if the trial candidate also has a lower energy than the 711 | # best solution then replace that as well 712 | if energy < self.population_energies[0]: 713 | self.population_energies[0] = energy 714 | self.population[0] = trial 715 | 716 | # for candidate in range(self.num_population_members): 717 | # if self._nfev > self.maxfun: 718 | # raise StopIteration 719 | 720 | # # create a trial solution 721 | # trial = self._mutate(candidate) 722 | 723 | # # ensuring that it's in the range [0, 1) 724 | # self._ensure_constraint(trial) 725 | 726 | # # scale from [0, 1) to the actual parameter value 727 | # parameters = self._scale_parameters(trial) 728 | 729 | # # determine the energy of the objective function 730 | # energy = self.func(parameters, *self.args) 731 | # self._nfev += 1 732 | 733 | # # if the energy of the trial candidate is lower than the 734 | # # original population member then replace it 735 | # if energy < self.population_energies[candidate]: 736 | # self.population[candidate] = trial 737 | # self.population_energies[candidate] = energy 738 | 739 | # # if the trial candidate also has a lower energy than the 740 | # # best solution then replace that as well 741 | # if energy < self.population_energies[0]: 742 | # self.population_energies[0] = energy 743 | # self.population[0] = trial 744 | 745 | ############## 746 | ############## 747 | 748 | return self.x, self.population_energies[0] 749 | 750 | def next(self): 751 | """ 752 | Evolve the population by a single generation 753 | Returns 754 | ------- 755 | x : ndarray 756 | The best solution from the solver. 757 | fun : float 758 | Value of objective function obtained from the best solution. 759 | """ 760 | # next() is required for compatibility with Python2.7. 761 | return self.__next__() 762 | 763 | def _scale_parameters(self, trial): 764 | """ 765 | scale from a number between 0 and 1 to parameters. 766 | """ 767 | return self.__scale_arg1 + (trial - 0.5) * self.__scale_arg2 768 | 769 | def _unscale_parameters(self, parameters): 770 | """ 771 | scale from parameters to a number between 0 and 1. 772 | """ 773 | return (parameters - self.__scale_arg1) / self.__scale_arg2 + 0.5 774 | 775 | def _ensure_constraint(self, trial): 776 | """ 777 | make sure the parameters lie between the limits 778 | """ 779 | for index in np.where((trial < 0) | (trial > 1))[0]: 780 | trial[index] = self.random_number_generator.rand() 781 | 782 | def _mutate(self, candidate): 783 | """ 784 | create a trial vector based on a mutation strategy 785 | """ 786 | trial = np.copy(self.population[candidate]) 787 | 788 | rng = self.random_number_generator 789 | 790 | fill_point = rng.randint(0, self.parameter_count) 791 | 792 | if self.strategy in ['currenttobest1exp', 'currenttobest1bin']: 793 | bprime = self.mutation_func(candidate, 794 | self._select_samples(candidate, 5)) 795 | else: 796 | bprime = self.mutation_func(self._select_samples(candidate, 5)) 797 | 798 | if self.strategy in self._binomial: 799 | crossovers = rng.rand(self.parameter_count) 800 | crossovers = crossovers < self.cross_over_probability 801 | # the last one is always from the bprime vector for binomial 802 | # If you fill in modulo with a loop you have to set the last one to 803 | # true. If you don't use a loop then you can have any random entry 804 | # be True. 805 | crossovers[fill_point] = True 806 | trial = np.where(crossovers, bprime, trial) 807 | return trial 808 | 809 | elif self.strategy in self._exponential: 810 | i = 0 811 | while (i < self.parameter_count and 812 | rng.rand() < self.cross_over_probability): 813 | 814 | trial[fill_point] = bprime[fill_point] 815 | fill_point = (fill_point + 1) % self.parameter_count 816 | i += 1 817 | 818 | return trial 819 | 820 | def _best1(self, samples): 821 | """ 822 | best1bin, best1exp 823 | """ 824 | r0, r1 = samples[:2] 825 | return (self.population[0] + self.scale * 826 | (self.population[r0] - self.population[r1])) 827 | 828 | def _rand1(self, samples): 829 | """ 830 | rand1bin, rand1exp 831 | """ 832 | r0, r1, r2 = samples[:3] 833 | return (self.population[r0] + self.scale * 834 | (self.population[r1] - self.population[r2])) 835 | 836 | def _randtobest1(self, samples): 837 | """ 838 | randtobest1bin, randtobest1exp 839 | """ 840 | r0, r1, r2 = samples[:3] 841 | bprime = np.copy(self.population[r0]) 842 | bprime += self.scale * (self.population[0] - bprime) 843 | bprime += self.scale * (self.population[r1] - 844 | self.population[r2]) 845 | return bprime 846 | 847 | def _currenttobest1(self, candidate, samples): 848 | """ 849 | currenttobest1bin, currenttobest1exp 850 | """ 851 | r0, r1 = samples[:2] 852 | bprime = (self.population[candidate] + self.scale * 853 | (self.population[0] - self.population[candidate] + 854 | self.population[r0] - self.population[r1])) 855 | return bprime 856 | 857 | def _best2(self, samples): 858 | """ 859 | best2bin, best2exp 860 | """ 861 | r0, r1, r2, r3 = samples[:4] 862 | bprime = (self.population[0] + self.scale * 863 | (self.population[r0] + self.population[r1] - 864 | self.population[r2] - self.population[r3])) 865 | 866 | return bprime 867 | 868 | def _rand2(self, samples): 869 | """ 870 | rand2bin, rand2exp 871 | """ 872 | r0, r1, r2, r3, r4 = samples 873 | bprime = (self.population[r0] + self.scale * 874 | (self.population[r1] + self.population[r2] - 875 | self.population[r3] - self.population[r4])) 876 | 877 | return bprime 878 | 879 | def _select_samples(self, candidate, number_samples): 880 | """ 881 | obtain random integers from range(self.num_population_members), 882 | without replacement. You can't have the original candidate either. 883 | """ 884 | idxs = list(range(self.num_population_members)) 885 | idxs.remove(candidate) 886 | self.random_number_generator.shuffle(idxs) 887 | idxs = idxs[:number_samples] 888 | return idxs 889 | --------------------------------------------------------------------------------