├── README.md ├── allconv.py ├── allconv_fashionmnist.py ├── baseline_cifar.py ├── fashionmnsit_baseline.py ├── multiloss.py └── multiloss_fashionmnsit.py /README.md: -------------------------------------------------------------------------------- 1 | # Label-embeddings-in-image-classification 2 | Convolutional Neural Networks (CNNs) are being widely used for various tasks in Computer Vision. We focus on the task of image classification particularly using CNNs with more focus on the relation or similarity between class labels. The similarity between labels is judged using label word embeddings and incorporated into the loss layer. We propose that shallower networks be learnt with more complex and structured losses, in order to gain from shorter training time and equivalent complexity. We train two variants of CNNs with multiple architectures , all limited to a maximum of ten convolution layers to obtain an accuracy of 93.27% on the Fashion-MNIST dataset and 86.40% on the CIFAR 10 dataset. We further probe the adversarial robustness of the model as well the classspecific behavior by visualizing the class confusion matrix.We also show some preliminary results towards extending a trained variant to zero-shot learning. 3 | -------------------------------------------------------------------------------- /allconv.py: -------------------------------------------------------------------------------- 1 | '''Train a simple deep CNN on the CIFAR10 small images dataset. 2 | 3 | It gets to 75% validation accuracy in 25 epochs, and 79% after 50 epochs. 4 | (it's still underfitting at that point, though). 5 | ''' 6 | 7 | from __future__ import print_function 8 | import keras 9 | import h5py 10 | from keras.datasets import cifar10 11 | from keras.preprocessing.image import ImageDataGenerator 12 | from keras.models import Sequential 13 | from keras.models import model_from_json 14 | from keras.layers import Dense, Dropout, Activation, Flatten 15 | from keras.layers import Input, Dense, ZeroPadding2D, GlobalAveragePooling2D,Convolution2D, MaxPooling2D,BatchNormalization 16 | from keras.models import Model 17 | import numpy as np 18 | import tensorflow as tf 19 | import matplotlib.pyplot as plt 20 | import os 21 | 22 | batch_size = 250 23 | num_classes = 10 24 | epochs = 250 25 | data_augmentation = False 26 | num_predictions = 20 27 | save_dir = os.path.join(os.getcwd(), 'saved_models') 28 | model_name = 'keras_cifar10_allconv1_model.h5' 29 | 30 | def main_loss(y_true,y_pred): 31 | ind=0 32 | if(ind==0): 33 | arr = np.load('labels.npy') 34 | arr = tf.convert_to_tensor(arr,dtype='float32') 35 | sh = tf.shape(y_pred) 36 | vec1 = tf.tensordot(y_pred,arr,axes=[1,0]) 37 | vec2 = tf.tensordot(y_true,arr,axes=[1,0]) 38 | ans = tf.square(vec1-vec2) 39 | 40 | return ans 41 | 42 | # The data, shuffled and split between train and test sets: 43 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 44 | 45 | # Convert class vectors to binary class matrices. 46 | y_train = keras.utils.to_categorical(y_train, 10) 47 | y_test = keras.utils.to_categorical(y_test, 10) 48 | 49 | model = Sequential() 50 | 51 | model.add(Convolution2D(96, 3, 3, border_mode = 'same', input_shape=(32,32,3))) 52 | model.add(Activation('relu')) 53 | model.add(Convolution2D(96, 3, 3,border_mode='same')) 54 | model.add(Activation('relu')) 55 | model.add(Convolution2D(96, 3, 3, border_mode='same', subsample = (2,2))) 56 | model.add(Dropout(0.5)) 57 | 58 | model.add(Convolution2D(192, 3, 3, border_mode = 'same')) 59 | model.add(Activation('relu')) 60 | model.add(Convolution2D(192, 3, 3,border_mode='same')) 61 | model.add(Activation('relu')) 62 | model.add(Convolution2D(192, 3, 3,border_mode='same', subsample = (2,2))) 63 | model.add(Dropout(0.5)) 64 | 65 | model.add(Convolution2D(192, 3, 3, border_mode = 'same')) 66 | model.add(Activation('relu')) 67 | model.add(Convolution2D(192, 1, 1,border_mode='valid')) 68 | model.add(Activation('relu')) 69 | model.add(Convolution2D(10, 1, 1, border_mode='valid')) 70 | 71 | 72 | 73 | model.add(GlobalAveragePooling2D()) 74 | model.add(Activation('softmax')) 75 | 76 | 77 | 78 | 79 | 80 | # initiate RMSprop optimizer 81 | #opt = keras.optimizers.Adagrad(lr=0.01, epsilon=1e-08, decay=0.0) 82 | #opt = keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0) 83 | opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6) 84 | 85 | 86 | # Let's train the model using RMSprop 87 | model.compile(loss='categorical_crossentropy', 88 | optimizer=opt, 89 | metrics=['accuracy']) 90 | 91 | 92 | if not data_augmentation: 93 | print('Not using data augmentation.') 94 | history = model.fit(x_train, y_train, 95 | batch_size=batch_size, 96 | epochs=epochs, 97 | validation_split=0.1, 98 | shuffle=True,verbose=2) 99 | if not os.path.isdir(save_dir): 100 | os.makedirs(save_dir) 101 | model_path = os.path.join(save_dir, model_name) 102 | model.save(model_path) 103 | print('Saved trained model at %s ' % model_path) 104 | # list all data in history 105 | print(history.history.keys()) 106 | plt.plot(history.history['acc']) 107 | plt.plot(history.history['val_acc']) 108 | plt.title('Model classification accuracy') 109 | plt.ylabel('Accuracy') 110 | plt.xlabel('Epoch') 111 | plt.legend(['train', 'validation'], loc='upper left') 112 | plt.savefig('allconv.png') 113 | else: 114 | print('Using real-time data augmentation.') 115 | # This will do preprocessing and realtime data augmentation: 116 | datagen = ImageDataGenerator( 117 | featurewise_center=False, # set input mean to 0 over the dataset 118 | samplewise_center=False, # set each sample mean to 0 119 | featurewise_std_normalization=False, # divide inputs by std of the dataset 120 | samplewise_std_normalization=False, # divide each input by its std 121 | zca_whitening=False, # apply ZCA whitening 122 | rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180) 123 | width_shift_range=0.1, # randomly shift images horizontally (fraction of total width) 124 | height_shift_range=0.1, # randomly shift images vertically (fraction of total height) 125 | horizontal_flip=True, # randomly flip images 126 | vertical_flip=False) # randomly flip images 127 | 128 | # Compute quantities required for feature-wise normalization 129 | # (std, mean, and principal components if ZCA whitening is applied). 130 | datagen.fit(x_train) 131 | 132 | # Fit the model on the batches generated by datagen.flow(). 133 | model.fit_generator(datagen.flow(x_train, y_train, 134 | batch_size=batch_size),epochs=epochs, 135 | validation_data=(x_test, y_test), 136 | steps_per_epoch=x_train.shape[0] // batch_size) 137 | 138 | # Save model and weights 139 | if not os.path.isdir(save_dir): 140 | os.makedirs(save_dir) 141 | model_path = os.path.join(save_dir, model_name) 142 | model.save(model_path) 143 | print('Saved trained model at %s ' % model_path) 144 | 145 | # Score trained model. 146 | scores = model.evaluate(x_test, y_test, verbose=1) 147 | print('Scores:', scores) 148 | -------------------------------------------------------------------------------- /allconv_fashionmnist.py: -------------------------------------------------------------------------------- 1 | '''Train a simple deep CNN on the CIFAR10 small images dataset. 2 | 3 | It gets to 75% validation accuracy in 25 epochs, and 79% after 50 epochs. 4 | (it's still underfitting at that point, though). 5 | ''' 6 | 7 | from __future__ import print_function 8 | import keras 9 | import h5py 10 | import matplotlib 11 | matplotlib.use('Agg') 12 | from keras.preprocessing.image import ImageDataGenerator 13 | from keras.models import Sequential 14 | from keras.models import model_from_json 15 | from keras.layers import Dense, Dropout, Activation, Flatten 16 | from keras.layers import Input, Dense, ZeroPadding2D, GlobalAveragePooling2D,Convolution2D, MaxPooling2D,BatchNormalization 17 | from keras.models import Model 18 | from keras.datasets import fashion_mnist 19 | import numpy as np 20 | import tensorflow as tf 21 | import os 22 | from keras import backend as K 23 | import matplotlib.pyplot as plt 24 | 25 | batch_size = 250 26 | num_classes = 10 27 | epochs = 500 28 | data_augmentation = False 29 | num_predictions = 20 30 | save_dir = os.path.join(os.getcwd(), 'saved_models') 31 | model_name = 'keras_fashionmnist_allconv_model.h5' 32 | 33 | def main_loss(y_true,y_pred): 34 | ind=0 35 | if(ind==0): 36 | arr = np.load('mnsitlabels.npy') 37 | arr = tf.convert_to_tensor(arr,dtype='float32') 38 | sh = tf.shape(y_pred) 39 | vec1 = tf.tensordot(y_pred,arr,axes=[1,0]) 40 | vec2 = tf.tensordot(y_true,arr,axes=[1,0]) 41 | ans = tf.square(vec1-vec2) 42 | 43 | return ans 44 | # input image dimensions 45 | img_rows, img_cols = 28, 28 46 | 47 | # the data, shuffled and split between train and test sets 48 | (x_train, y_train), (x_test, y_test) = fashion_mnist.load_data() 49 | 50 | if K.image_data_format() == 'channels_first': 51 | x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols) 52 | x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols) 53 | input_shape = (1, img_rows, img_cols) 54 | else: 55 | x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) 56 | x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) 57 | input_shape = (img_rows, img_cols, 1) 58 | 59 | x_train = x_train.astype('float32') 60 | x_test = x_test.astype('float32') 61 | x_train /= 255 62 | x_test /= 255 63 | 64 | print('x_train shape:', x_train.shape) 65 | print(x_train.shape[0], 'train samples') 66 | print(x_test.shape[0], 'test samples') 67 | 68 | # The data, shuffled and split between train and test sets: 69 | #(x_train, y_train), (x_test, y_test) = cifar10.load_data() 70 | 71 | # Convert class vectors to binary class matrices. 72 | y_train = keras.utils.to_categorical(y_train, 10) 73 | y_test = keras.utils.to_categorical(y_test, 10) 74 | 75 | model = Sequential() 76 | 77 | model.add(Convolution2D(96, 3, 3, border_mode = 'same', input_shape=(28,28,1))) 78 | model.add(Activation('relu')) 79 | model.add(Convolution2D(96, 3, 3,border_mode='same')) 80 | model.add(Activation('relu')) 81 | model.add(Convolution2D(96, 3, 3, border_mode='same', subsample = (2,2))) 82 | model.add(Dropout(0.5)) 83 | 84 | model.add(Convolution2D(192, 3, 3, border_mode = 'same')) 85 | model.add(Activation('relu')) 86 | model.add(Convolution2D(192, 3, 3,border_mode='same')) 87 | model.add(Activation('relu')) 88 | model.add(Convolution2D(192, 3, 3,border_mode='same', subsample = (2,2))) 89 | model.add(Dropout(0.5)) 90 | 91 | model.add(Convolution2D(192, 3, 3, border_mode = 'same')) 92 | model.add(Activation('relu')) 93 | model.add(Convolution2D(192, 1, 1,border_mode='valid')) 94 | model.add(Activation('relu')) 95 | model.add(Convolution2D(10, 1, 1, border_mode='valid')) 96 | 97 | model.add(GlobalAveragePooling2D()) 98 | model.add(Activation('softmax')) 99 | 100 | 101 | # initiate RMSprop optimizer 102 | #opt = keras.optimizers.Adagrad(lr=0.01, epsilon=1e-08, decay=0.0) 103 | #opt = keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0) 104 | opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6) 105 | 106 | 107 | # Let's train the model using RMSprop 108 | model.compile(loss=main_loss, 109 | optimizer=opt, 110 | metrics=['accuracy']) 111 | 112 | 113 | if not data_augmentation: 114 | 115 | print('Not using data augmentation.') 116 | cb = keras.callbacks.TensorBoard(log_dir='./Graph', histogram_freq=0, 117 | write_graph=True, write_images=True) 118 | history = model.fit(x_train, y_train, 119 | batch_size=batch_size, 120 | epochs=epochs, 121 | validation_split = 0.1, 122 | shuffle=True,verbose=2,callbacks = [cb]) 123 | 124 | if not os.path.isdir(save_dir): 125 | os.makedirs(save_dir) 126 | model_path = os.path.join(save_dir, model_name) 127 | model.save(model_path) 128 | print('Saved trained model at %s ' % model_path) 129 | # list all data in history 130 | print(history.history.keys()) 131 | # summarize history for loss 132 | plt.plot(history.history['acc']) 133 | plt.plot(history.history['val_acc']) 134 | plt.title('Model classification accuracy') 135 | plt.ylabel('Accuracy') 136 | plt.xlabel('Epoch') 137 | plt.legend(['train', 'validation'], loc='lower right') 138 | plt.savefig("allconv_fashionmnsit_classification.png") 139 | plt.show() 140 | #summarize for other loss 141 | plt.plot(history.history['loss']) 142 | plt.plot(history.history['val_loss']) 143 | plt.title('Model regression loss') 144 | plt.ylabel('Loss') 145 | plt.xlabel('Epoch') 146 | plt.legend(['train', 'validation'], loc='lower right') 147 | plt.savefig("allconv_fashionmnsit_regression.png") 148 | plt.show() 149 | else: 150 | print('Using real-time data augmentation.') 151 | # This will do preprocessing and realtime data augmentation: 152 | datagen = ImageDataGenerator( 153 | featurewise_center=False, # set input mean to 0 over the dataset 154 | samplewise_center=False, # set each sample mean to 0 155 | featurewise_std_normalization=False, # divide inputs by std of the dataset 156 | samplewise_std_normalization=False, # divide each input by its std 157 | zca_whitening=False, # apply ZCA whitening 158 | rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180) 159 | width_shift_range=0.1, # randomly shift images horizontally (fraction of total width) 160 | height_shift_range=0.1, # randomly shift images vertically (fraction of total height) 161 | horizontal_flip=True, # randomly flip images 162 | vertical_flip=False) # randomly flip images 163 | 164 | # Compute quantities required for feature-wise normalization 165 | # (std, mean, and principal components if ZCA whitening is applied). 166 | datagen.fit(x_train) 167 | 168 | # Fit the model on the batches generated by datagen.flow(). 169 | model.fit_generator(datagen.flow(x_train, y_train, 170 | batch_size=batch_size),epochs=epochs, 171 | validation_data=(x_test, y_test), 172 | steps_per_epoch=x_train.shape[0] // batch_size) 173 | 174 | # Save model and weights 175 | if not os.path.isdir(save_dir): 176 | os.makedirs(save_dir) 177 | model_path = os.path.join(save_dir, model_name) 178 | model.save(model_path) 179 | print('Saved trained model at %s ' % model_path) 180 | 181 | # Score trained model. 182 | scores = model.evaluate(x_test, y_test, verbose=1) 183 | print('Scores:', scores) 184 | -------------------------------------------------------------------------------- /baseline_cifar.py: -------------------------------------------------------------------------------- 1 | '''Train a simple deep CNN on the CIFAR10 small images dataset. 2 | 3 | It gets to 75% validation accuracy in 25 epochs, and 79% after 50 epochs. 4 | (it's still underfitting at that point, though). 5 | ''' 6 | 7 | from __future__ import print_function 8 | import keras 9 | import h5py 10 | from keras.datasets import cifar10 11 | from keras.preprocessing.image import ImageDataGenerator 12 | from keras.models import Sequential 13 | from keras.models import model_from_json 14 | from keras.layers import Dense, Dropout, Activation, Flatten 15 | from keras.layers import Input, Dense, ZeroPadding2D, GlobalAveragePooling2D,Convolution2D, MaxPooling2D,BatchNormalization 16 | from keras.models import Model 17 | import numpy as np 18 | import tensorflow as tf 19 | import matplotlib.pyplot as plt 20 | import os 21 | 22 | batch_size = 250 23 | num_classes = 10 24 | epochs = 250 25 | data_augmentation = False 26 | num_predictions = 20 27 | save_dir = os.path.join(os.getcwd(), 'saved_models') 28 | model_name = 'keras_cifar10_allconv1_model.h5' 29 | 30 | def main_loss(y_true,y_pred): 31 | ind=0 32 | if(ind==0): 33 | arr = np.load('labels.npy') 34 | arr = tf.convert_to_tensor(arr,dtype='float32') 35 | sh = tf.shape(y_pred) 36 | vec1 = tf.tensordot(y_pred,arr,axes=[1,0]) 37 | vec2 = tf.tensordot(y_true,arr,axes=[1,0]) 38 | ans = tf.square(vec1-vec2) 39 | 40 | return ans 41 | 42 | # The data, shuffled and split between train and test sets: 43 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 44 | 45 | # Convert class vectors to binary class matrices. 46 | y_train = keras.utils.to_categorical(y_train, 10) 47 | y_test = keras.utils.to_categorical(y_test, 10) 48 | 49 | model = Sequential() 50 | 51 | model.add(Convolution2D(96, 3, 3, border_mode = 'same', input_shape=(32,32,3))) 52 | model.add(Activation('relu')) 53 | model.add(Convolution2D(96, 3, 3,border_mode='same')) 54 | model.add(Activation('relu')) 55 | model.add(Convolution2D(96, 3, 3, border_mode='same', subsample = (2,2))) 56 | model.add(Dropout(0.5)) 57 | 58 | model.add(Convolution2D(192, 3, 3, border_mode = 'same')) 59 | model.add(Activation('relu')) 60 | model.add(Convolution2D(192, 3, 3,border_mode='same')) 61 | model.add(Activation('relu')) 62 | model.add(Convolution2D(192, 3, 3,border_mode='same', subsample = (2,2))) 63 | model.add(Dropout(0.5)) 64 | 65 | model.add(Convolution2D(192, 3, 3, border_mode = 'same')) 66 | model.add(Activation('relu')) 67 | model.add(Convolution2D(192, 1, 1,border_mode='valid')) 68 | model.add(Activation('relu')) 69 | model.add(Convolution2D(10, 1, 1, border_mode='valid')) 70 | 71 | 72 | 73 | model.add(GlobalAveragePooling2D()) 74 | model.add(Activation('softmax')) 75 | 76 | 77 | 78 | 79 | 80 | # initiate RMSprop optimizer 81 | #opt = keras.optimizers.Adagrad(lr=0.01, epsilon=1e-08, decay=0.0) 82 | #opt = keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0) 83 | opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6) 84 | 85 | 86 | # Let's train the model using RMSprop 87 | model.compile(loss='categorical_crossentropy', 88 | optimizer=opt, 89 | metrics=['accuracy']) 90 | 91 | 92 | if not data_augmentation: 93 | print('Not using data augmentation.') 94 | history = model.fit(x_train, y_train, 95 | batch_size=batch_size, 96 | epochs=epochs, 97 | validation_split=0.1, 98 | shuffle=True,verbose=2) 99 | if not os.path.isdir(save_dir): 100 | os.makedirs(save_dir) 101 | model_path = os.path.join(save_dir, model_name) 102 | model.save(model_path) 103 | print('Saved trained model at %s ' % model_path) 104 | # list all data in history 105 | print(history.history.keys()) 106 | plt.plot(history.history['acc']) 107 | plt.plot(history.history['val_acc']) 108 | plt.title('Model classification accuracy') 109 | plt.ylabel('Accuracy') 110 | plt.xlabel('Epoch') 111 | plt.legend(['train', 'validation'], loc='upper left') 112 | plt.savefig('allconv.png') 113 | else: 114 | print('Using real-time data augmentation.') 115 | # This will do preprocessing and realtime data augmentation: 116 | datagen = ImageDataGenerator( 117 | featurewise_center=False, # set input mean to 0 over the dataset 118 | samplewise_center=False, # set each sample mean to 0 119 | featurewise_std_normalization=False, # divide inputs by std of the dataset 120 | samplewise_std_normalization=False, # divide each input by its std 121 | zca_whitening=False, # apply ZCA whitening 122 | rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180) 123 | width_shift_range=0.1, # randomly shift images horizontally (fraction of total width) 124 | height_shift_range=0.1, # randomly shift images vertically (fraction of total height) 125 | horizontal_flip=True, # randomly flip images 126 | vertical_flip=False) # randomly flip images 127 | 128 | # Compute quantities required for feature-wise normalization 129 | # (std, mean, and principal components if ZCA whitening is applied). 130 | datagen.fit(x_train) 131 | 132 | # Fit the model on the batches generated by datagen.flow(). 133 | model.fit_generator(datagen.flow(x_train, y_train, 134 | batch_size=batch_size),epochs=epochs, 135 | validation_data=(x_test, y_test), 136 | steps_per_epoch=x_train.shape[0] // batch_size) 137 | 138 | # Save model and weights 139 | if not os.path.isdir(save_dir): 140 | os.makedirs(save_dir) 141 | model_path = os.path.join(save_dir, model_name) 142 | model.save(model_path) 143 | print('Saved trained model at %s ' % model_path) 144 | 145 | # Score trained model. 146 | scores = model.evaluate(x_test, y_test, verbose=1) 147 | print('Scores:', scores) 148 | -------------------------------------------------------------------------------- /fashionmnsit_baseline.py: -------------------------------------------------------------------------------- 1 | import keras 2 | import matplotlib 3 | matplotlib.use('Agg') 4 | from keras.models import Sequential 5 | from keras.datasets import fashion_mnist 6 | from keras import backend as K 7 | from keras.layers import Dense, Dropout, Activation, Flatten 8 | from keras.layers import Input, Dense, ZeroPadding2D, Conv2D, MaxPooling2D 9 | from keras.models import Model 10 | import numpy as np 11 | import tensorflow as tf 12 | from keras import backend as K 13 | import matplotlib.pyplot as plt 14 | import os 15 | 16 | batch_size = 128 17 | num_classes = 10 18 | epochs = 100 19 | save_dir = os.path.join(os.getcwd(), 'saved_models') 20 | model_name = 'keras_fashionmnsit_baseline_model.h5' 21 | 22 | # input image dimensions 23 | img_rows, img_cols = 28, 28 24 | 25 | # the data, shuffled and split between train and test sets 26 | (x_train, y_train), (x_test, y_test) = fashion_mnist.load_data() 27 | 28 | if K.image_data_format() == 'channels_first': 29 | x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols) 30 | x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols) 31 | input_shape = (1, img_rows, img_cols) 32 | else: 33 | x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) 34 | x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) 35 | input_shape = (img_rows, img_cols, 1) 36 | 37 | x_train = x_train.astype('float32') 38 | x_test = x_test.astype('float32') 39 | x_train /= 255 40 | x_test /= 255 41 | 42 | print('x_train shape:', x_train.shape) 43 | print(x_train.shape[0], 'train samples') 44 | print(x_test.shape[0], 'test samples') 45 | 46 | # convert class vectors to binary class matrices 47 | y_train = keras.utils.to_categorical(y_train, num_classes) 48 | y_test = keras.utils.to_categorical(y_test, num_classes) 49 | 50 | model = Sequential() 51 | model.add(Conv2D(32, kernel_size=(3, 3), 52 | activation='relu', 53 | input_shape=input_shape)) 54 | model.add(Conv2D(64, (3, 3), activation='relu')) 55 | model.add(MaxPooling2D(pool_size=(2, 2))) 56 | model.add(Dropout(0.25)) 57 | model.add(Flatten()) 58 | model.add(Dense(128, activation='relu')) 59 | model.add(Dropout(0.5)) 60 | model.add(Dense(num_classes, activation='softmax')) 61 | 62 | opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6) 63 | 64 | '''model.compile(loss=main_loss, 65 | optimizer=opt, 66 | metrics=['accuracy'])''' 67 | model.compile(loss=keras.losses.categorical_crossentropy, 68 | optimizer=opt, 69 | metrics=['accuracy']) 70 | 71 | cb = keras.callbacks.TensorBoard(log_dir='./Graph', histogram_freq=0, 72 | write_graph=True, write_images=True) 73 | history = model.fit(x_train, y_train, 74 | batch_size=batch_size, 75 | epochs=epochs, 76 | verbose=1, 77 | validation_split=0.1,callbacks = [cb]) 78 | if not os.path.isdir(save_dir): 79 | os.makedirs(save_dir) 80 | model_path = os.path.join(save_dir, model_name) 81 | model.save(model_path) 82 | print('Saved trained model at %s ' % model_path) 83 | # list all data in history 84 | print(history.history.keys()) 85 | # summarize history for loss 86 | plt.plot(history.history['acc']) 87 | plt.plot(history.history['val_acc']) 88 | plt.title('Model classification accuracy') 89 | plt.ylabel('Accuracy') 90 | plt.xlabel('Epoch') 91 | plt.legend(['train', 'validation'], loc='lower right') 92 | plt.savefig("fashionmnsit_baseline_classification.png") 93 | plt.show() 94 | #summarize for other loss 95 | plt.plot(history.history['loss']) 96 | plt.plot(history.history['val_loss']) 97 | plt.title('Model regression loss') 98 | plt.ylabel('Loss') 99 | plt.xlabel('Epoch') 100 | plt.legend(['train', 'validation'], loc='lower right') 101 | plt.savefig("fashionmnsit_baseline_regression.png") 102 | plt.show() 103 | 104 | score = model.evaluate(x_test, y_test, verbose=0) 105 | print('Test loss:', score[0]) 106 | print('Test accuracy:', score[1]) 107 | -------------------------------------------------------------------------------- /multiloss.py: -------------------------------------------------------------------------------- 1 | '''Train a simple deep CNN on the CIFAR10 small images dataset. 2 | 3 | It gets to 75% validation accuracy in 25 epochs, and 79% after 50 epochs. 4 | (it's still underfitting at that point, though). 5 | ''' 6 | 7 | from __future__ import print_function 8 | import keras 9 | import h5py 10 | from keras.datasets import cifar10 11 | from keras.preprocessing.image import ImageDataGenerator 12 | from keras.models import Sequential 13 | from keras.models import model_from_json 14 | from keras.layers import Dense, Dropout, Activation, Flatten 15 | from keras.layers import Input, Dense, ZeroPadding2D, Conv2D, MaxPooling2D,BatchNormalization 16 | from keras.models import Model 17 | import numpy as np 18 | import tensorflow as tf 19 | import os 20 | import matplotlib.pyplot as plt 21 | 22 | batch_size = 400 23 | num_classes = 10 24 | epochs = 400 25 | data_augmentation = False 26 | num_predictions = 20 27 | save_dir = os.path.join(os.getcwd(), 'saved_models') 28 | model_name = 'keras_cifar10_multiloss_model.h5' 29 | 30 | def main_loss(y_true,y_pred): 31 | ind=0 32 | if(ind==0): 33 | arr = np.load('labels.npy') 34 | arr = tf.convert_to_tensor(arr,dtype='float32') 35 | sh = tf.shape(y_pred) 36 | vec1 = tf.tensordot(y_pred,arr,axes=[1,0]) 37 | vec2 = tf.tensordot(y_true,arr,axes=[1,0]) 38 | ans = tf.square(vec1-vec2) 39 | 40 | return ans 41 | 42 | # The data, shuffled and split between train and test sets: 43 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 44 | (x_train, y1), (x_test, y_t1) = cifar10.load_data() 45 | 46 | '''l=[] 47 | for i in range(y_train.shape[0]): 48 | if(y_train[i]==3): 49 | l.append(i) 50 | print('here') 51 | 52 | 53 | x_train=np.delete(x_train,l,axis=0) 54 | y_train=np.delete(y_train,l,axis=0) 55 | y1=np.delete(y1,l,axis=0) 56 | print(x_train.shape) 57 | print('here')''' 58 | 59 | 60 | arr = np.load('labels.npy') 61 | 62 | y = np.zeros((y1.shape[0],300)) 63 | 64 | y_t = np.zeros((y_t1.shape[0],300)) 65 | 66 | 67 | for i in range(y.shape[0]): 68 | y[i,:] = arr[int(y1[i])] 69 | 70 | 71 | for i in range(y_t.shape[0]): 72 | y_t[i,:] = arr[int(y_t1[i])] 73 | 74 | # Convert class vectors to binary class matrices. 75 | y_train = keras.utils.to_categorical(y_train, 10) 76 | y_test = keras.utils.to_categorical(y_test, 10) 77 | 78 | inputs = Input(shape=(32,32,3)) 79 | 80 | x = (Conv2D(32, (3, 3), activation='relu'))(inputs) 81 | x = Conv2D(64, (3, 3), activation='relu')(x) 82 | x =(MaxPooling2D((2,2), strides=(2,2)))(x) 83 | x = (Dropout(0.5))(x) 84 | 85 | 86 | 87 | x = (Conv2D(128, (3, 3), activation='relu'))(x) 88 | x = Conv2D(256, (3, 3), activation='relu')(x) 89 | x = BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001, center=True, scale=True)(x) 90 | x =(MaxPooling2D((2,2), strides=(2,2)))(x) 91 | x = (Dropout(0.5))(x) 92 | 93 | x = Conv2D(256, (1,1), activation='relu')(x) 94 | x = Conv2D(256, (1,1), activation='relu')(x) 95 | x = BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001, center=True, scale=True)(x) 96 | x = (Flatten())(x) 97 | x = (Dense(512, activation='relu'))(x) 98 | x = (Dropout(0.5))(x) 99 | fine = (Dense(10, activation='softmax'))(x) 100 | 101 | word2vec = (Dense(300, activation='softmax'))(fine) 102 | 103 | 104 | model = Model(inputs=[inputs], outputs=[fine,word2vec]) 105 | 106 | 107 | # initiate RMSprop optimizer 108 | #opt = keras.optimizers.Adagrad(lr=0.01, epsilon=1e-08, decay=0.0) 109 | #opt = keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0) 110 | opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6) 111 | 112 | 113 | # Let's train the model using RMSprop 114 | model.compile(loss=['categorical_crossentropy','mse'], 115 | optimizer=opt, 116 | metrics=['accuracy'],loss_weights=[1,0.2]) 117 | 118 | 119 | if not data_augmentation: 120 | print('Not using data augmentation.') 121 | cb = keras.callbacks.TensorBoard(log_dir='./Graph', histogram_freq=0, 122 | write_graph=True, write_images=True) 123 | history = model.fit(x_train, [y_train,y], 124 | batch_size=batch_size, 125 | epochs=epochs, 126 | validation_split=0.1, 127 | shuffle=True,verbose=2,callbacks = [cb]) 128 | if not os.path.isdir(save_dir): 129 | os.makedirs(save_dir) 130 | model_path = os.path.join(save_dir, model_name) 131 | model.save(model_path) 132 | print('Saved trained model at %s ' % model_path) 133 | # list all data in history 134 | print(history.history.keys()) 135 | # summarize history for loss 136 | plt.plot(history.history['dense_2_acc']) 137 | plt.plot(history.history['val_dense_2_acc']) 138 | plt.title('Model classification accuracy') 139 | plt.ylabel('Accuracy') 140 | plt.xlabel('Epoch') 141 | plt.legend(['train', 'validation'], loc='upper left') 142 | plt.show() 143 | #summarize for other loss 144 | plt.plot(history.history['dense_2_loss']) 145 | plt.plot(history.history['val_dense_2_loss']) 146 | plt.title('Model regression loss') 147 | plt.ylabel('Loss') 148 | plt.xlabel('Epoch') 149 | plt.legend(['train', 'validation'], loc='upper left') 150 | plt.show() 151 | 152 | pred1,pred2 = model.predict(x_test,batch_size=100) 153 | np.save('pred1.npy',pred1) 154 | np.save('pred2.npy',pred2) 155 | else: 156 | print('Using real-time data augmentation.') 157 | # This will do preprocessing and realtime data augmentation: 158 | datagen = ImageDataGenerator( 159 | featurewise_center=False, # set input mean to 0 over the dataset 160 | samplewise_center=False, # set each sample mean to 0 161 | featurewise_std_normalization=False, # divide inputs by std of the dataset 162 | samplewise_std_normalization=False, # divide each input by its std 163 | zca_whitening=False, # apply ZCA whitening 164 | rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180) 165 | width_shift_range=0.1, # randomly shift images horizontally (fraction of total width) 166 | height_shift_range=0.1, # randomly shift images vertically (fraction of total height) 167 | horizontal_flip=True, # randomly flip images 168 | vertical_flip=False) # randomly flip images 169 | 170 | # Compute quantities required for feature-wise normalization 171 | # (std, mean, and principal components if ZCA whitening is applied). 172 | datagen.fit(x_train) 173 | 174 | # Fit the model on the batches generated by datagen.flow(). 175 | model.fit_generator(datagen.flow(x_train, y_train, 176 | batch_size=batch_size),epochs=epochs, 177 | validation_data=(x_test, y_test), 178 | steps_per_epoch=x_train.shape[0] // batch_size) 179 | 180 | # Save model and weights 181 | if not os.path.isdir(save_dir): 182 | os.makedirs(save_dir) 183 | model_path = os.path.join(save_dir, model_name) 184 | model.save(model_path) 185 | print('Saved trained model at %s ' % model_path) 186 | 187 | # Score trained model. 188 | 189 | scores = model.evaluate(x_test, [y_test,y_t], verbose=1) 190 | print('Scores:', scores) 191 | -------------------------------------------------------------------------------- /multiloss_fashionmnsit.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import keras 3 | import h5py 4 | import matplotlib 5 | matplotlib.use('Agg') 6 | from keras.preprocessing.image import ImageDataGenerator 7 | from keras.models import Sequential 8 | from keras.models import model_from_json 9 | from keras.layers import Dense, Dropout, Activation, Flatten 10 | from keras.layers import Input, Dense, ZeroPadding2D, Conv2D, MaxPooling2D,BatchNormalization 11 | from keras.models import Model 12 | import numpy as np 13 | import tensorflow as tf 14 | import os 15 | from keras.datasets import fashion_mnist 16 | from keras import backend as K 17 | import matplotlib.pyplot as plt 18 | import os 19 | 20 | batch_size = 250 21 | num_classes = 10 22 | epochs = 400 23 | data_augmentation = False 24 | save_dir = os.path.join(os.getcwd(), 'saved_models') 25 | model_name = 'keras_fashionmnsit_multiloss_model.h5' 26 | 27 | # input image dimensions 28 | img_rows, img_cols = 28, 28 29 | 30 | # the data, shuffled and split between train and test sets 31 | (x_train, y_train), (x_test, y_test) = fashion_mnist.load_data() 32 | 33 | if K.image_data_format() == 'channels_first': 34 | x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols) 35 | x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols) 36 | input_shape = (1, img_rows, img_cols) 37 | else: 38 | x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) 39 | x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) 40 | input_shape = (img_rows, img_cols, 1) 41 | 42 | x_train = x_train.astype('float32') 43 | x_test = x_test.astype('float32') 44 | x_train /= 255 45 | x_test /= 255 46 | 47 | print('x_train shape:', x_train.shape) 48 | print(x_train.shape[0], 'train samples') 49 | print(x_test.shape[0], 'test samples') 50 | 51 | 52 | arr = np.load('mnsitlabels.npy') 53 | 54 | y = np.zeros((y_train.shape[0],300)) 55 | 56 | y_t = np.zeros((y_test.shape[0],300)) 57 | 58 | 59 | for i in range(y.shape[0]): 60 | y[i,:] = arr[int(y_train[i])] 61 | 62 | 63 | for i in range(y_t.shape[0]): 64 | y_t[i,:] = arr[int(y_test[i])] 65 | 66 | # Convert class vectors to binary class matrices. 67 | y_train = keras.utils.to_categorical(y_train, 10) 68 | y_test = keras.utils.to_categorical(y_test, 10) 69 | 70 | inputs = Input(shape=(28,28,1)) 71 | 72 | x = (Conv2D(32, (3, 3), activation='relu'))(inputs) 73 | x = Conv2D(64, (3, 3), activation='relu')(x) 74 | x =(MaxPooling2D((2,2), strides=(2,2)))(x) 75 | x = (Dropout(0.5))(x) 76 | 77 | x = (Conv2D(128, (3, 3), activation='relu'))(x) 78 | x = Conv2D(256, (3, 3), activation='relu')(x) 79 | x = BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001, center=True, scale=True)(x) 80 | x =(MaxPooling2D((2,2), strides=(2,2)))(x) 81 | x = (Dropout(0.5))(x) 82 | 83 | x = Conv2D(256, (1,1), activation='relu')(x) 84 | x = Conv2D(256, (1,1), activation='relu')(x) 85 | x = BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001, center=True, scale=True)(x) 86 | x = (Flatten())(x) 87 | x = (Dense(512, activation='relu'))(x) 88 | x = (Dropout(0.5))(x) 89 | fine = (Dense(10, activation='softmax'))(x) 90 | 91 | word2vec = (Dense(300, activation='softmax'))(fine) 92 | 93 | 94 | model = Model(inputs=[inputs], outputs=[fine, word2vec]) 95 | 96 | 97 | # initiate RMSprop optimizer 98 | #opt = keras.optimizers.Adagrad(lr=0.01, epsilon=1e-08, decay=0.0) 99 | #opt = keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0) 100 | opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6) 101 | 102 | # Let's train the model using RMSprop 103 | model.compile(loss=['categorical_crossentropy','mse'], 104 | optimizer=opt, 105 | metrics=['accuracy'],loss_weights=[1,0.2]) 106 | 107 | if not data_augmentation: 108 | print('Not using data augmentation.') 109 | cb = keras.callbacks.TensorBoard(log_dir='./Graph', histogram_freq=0, 110 | write_graph=True, write_images=True) 111 | history = model.fit(x_train, [y_train,y], 112 | batch_size=batch_size, 113 | epochs=epochs, 114 | validation_split=0.1, 115 | shuffle=True,verbose=2,callbacks = [cb]) 116 | 117 | if not os.path.isdir(save_dir): 118 | os.makedirs(save_dir) 119 | model_path = os.path.join(save_dir, model_name) 120 | model.save(model_path) 121 | print('Saved trained model at %s ' % model_path) 122 | # list all data in history 123 | print(history.history.keys()) 124 | # summarize history for loss 125 | plt.plot(history.history['dense_2_acc']) 126 | plt.plot(history.history['val_dense_2_acc']) 127 | plt.title('Model classification accuracy') 128 | plt.ylabel('Accuracy') 129 | plt.xlabel('Epoch') 130 | plt.legend(['train', 'validation'], loc='upper left') 131 | plt.savefig("multiloss_fashionmnsit_classification.png") 132 | plt.show() 133 | #summarize for other loss 134 | plt.plot(history.history['dense_2_loss']) 135 | plt.plot(history.history['val_dense_2_loss']) 136 | plt.title('Model regression loss') 137 | plt.ylabel('Loss') 138 | plt.xlabel('Epoch') 139 | plt.legend(['train', 'validation'], loc='upper left') 140 | plt.savefig("multiloss_fashionmnsit_regression.png") 141 | plt.show() 142 | 143 | else: 144 | print('Using real-time data augmentation.') 145 | # This will do preprocessing and realtime data augmentation: 146 | datagen = ImageDataGenerator( 147 | featurewise_center=False, # set input mean to 0 over the dataset 148 | samplewise_center=False, # set each sample mean to 0 149 | featurewise_std_normalization=False, # divide inputs by std of the dataset 150 | samplewise_std_normalization=False, # divide each input by its std 151 | zca_whitening=False, # apply ZCA whitening 152 | rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180) 153 | width_shift_range=0.1, # randomly shift images horizontally (fraction of total width) 154 | height_shift_range=0.1, # randomly shift images vertically (fraction of total height) 155 | horizontal_flip=True, # randomly flip images 156 | vertical_flip=False) # randomly flip images 157 | 158 | # Compute quantities required for feature-wise normalization 159 | # (std, mean, and principal components if ZCA whitening is applied). 160 | datagen.fit(x_train) 161 | 162 | # Fit the model on the batches generated by datagen.flow(). 163 | model.fit_generator(datagen.flow(x_train, y_train, 164 | batch_size=batch_size),epochs=epochs, 165 | validation_data=(x_test, y_test), 166 | steps_per_epoch=x_train.shape[0] // batch_size) 167 | 168 | # Save model and weights 169 | if not os.path.isdir(save_dir): 170 | os.makedirs(save_dir) 171 | model_path = os.path.join(save_dir, model_name) 172 | model.save(model_path) 173 | print('Saved trained model at %s ' % model_path) 174 | 175 | # Score trained model. 176 | scores = model.evaluate(x_test, [y_test,y_t], verbose=1) 177 | print('Scores:', scores) 178 | --------------------------------------------------------------------------------