├── =1.9.0 ├── README.md ├── Readme.py ├── __pycache__ ├── grad_cam.cpython-36.pyc ├── otsu.cpython-36.pyc ├── se_block.cpython-36.pyc ├── se_inception_v3.cpython-36.pyc └── se_inception_v3_1.cpython-36.pyc ├── cifar10_resnet50.py ├── database ├── VMMRdb_3040_db.py ├── __pycache__ │ └── cub2002011_db.cpython-36.pyc ├── cub2002011_db.py └── fg_project.db ├── elephant.jpg ├── grad_cam.py ├── history_inceptv3_with_OSME_miru_4481211-152929.json ├── mnist_cnn.py ├── mobilenet_without_MAMC.py ├── model.png ├── model_OSME_miru.py ├── model_OSME_miru_beta.py ├── model_OSME_miru_crop_dog_APN.py ├── model_OSME_miru_crop_dog_test.py ├── model_OSME_miru_dog.py ├── model_inceptv3.png ├── model_inceptv3_miru.png ├── model_miru.py ├── model_resnet50_with_OSME.py ├── model_resnet50_with_rescale_no_trainable.py ├── model_seinceptv3_with_OSME.py ├── model_with_OSME.py ├── model_with_OSME_SE.py ├── model_with_OSME_SE_alpha.py ├── model_with_OSME_SE_alpha_p.py ├── model_with_OSME_crop.py ├── model_with_OSME_se.py ├── model_with_OSME_vmmrdb.py ├── model_without_MAMC.py ├── model_without_OSME.py ├── otsu.py ├── res-net-101.py ├── resnet50_alpha.py ├── resnet50_model.png ├── se_block.py ├── se_inception_v3.py ├── se_inception_v3_1.py ├── test0.py ├── train_test_splitter.py ├── vgg16_model_withou_MAMC.py └── vgg19_model_without_MAMC.py /=1.9.0: -------------------------------------------------------------------------------- 1 | Collecting PyHamcrest 2 | Downloading https://files.pythonhosted.org/packages/9a/d5/d37fd731b7d0e91afcc84577edeccf4638b4f9b82f5ffe2f8b62e2ddc609/PyHamcrest-1.9.0-py2.py3-none-any.whl (52kB) 3 | Requirement already satisfied: setuptools in /home/n-kamiya/anaconda3/lib/python3.6/site-packages (from PyHamcrest) (39.1.0) 4 | Requirement already satisfied: six in /home/n-kamiya/anaconda3/lib/python3.6/site-packages (from PyHamcrest) (1.11.0) 5 | Installing collected packages: PyHamcrest 6 | Successfully installed PyHamcrest-1.9.0 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fine_grained_classification 2 | Studying fine grained classification on [CUB2002011 dataset](http://www.vision.caltech.edu/visipedia-data/CUB-200-2011/CUB_200_2011.tgz). 3 | This repo contains: 4 | - Implementation of ["Multi-Attention Multi-Class Constraint for Fine-grained Image Recognition"](https://arxiv.org/abs/1806.05372) without MAMC loss function. (No plans to MAMC module implementation now) 5 | - Implementation of [[MIRU2018] Global Average Poolingの特性を用いたAttention Branch Network](https://www.slideshare.net/greentea1125/miru2018-global-average-poolingattention-branch-network) 6 | 7 | 8 | ## Networks 9 | - ResNet50 10 | - ResNet50 + OSME 11 | - InceptionV3 12 | - InceptionV3 + OSME 13 | - SE-InceptionV3 14 | - SE-InceptionV3 + OSME 15 | - SE-InceptionV3 + Attention Branch Network 16 | -------------------------------------------------------------------------------- /Readme.py: -------------------------------------------------------------------------------- 1 | <<<<<<< HEAD 2 | ======= 3 | 4 | >>>>>>> develop 5 | #Fine grained Classification 6 | # 7 | #1.Models 8 | # model_OSME_miru.py -> Attention branch + OSME 9 | # model_OSME_alpha.py -> OSME alpha 10 | # model_with_OSME_SE.py -> OSME 11 | # model_OSME_miru_dog.py -> Attention branch + OSME with StandfordDogs dataset 12 | # 13 | #2. How to use 14 | # Just run the python scripts. 15 | # To change models or other things rewrite the code. 16 | # 17 | #3.Using ImageDataGenerator.flow_from_directory: 18 | #Example: 19 | 20 | train_datagen = ImageDataGenerator(rescale = 1./255, 21 | zoom_range=[0.6,1], 22 | rotation_range=30, 23 | horizontal_flip=True) 24 | test_datagen = ImageDataGenerator(rescale = 1./255) 25 | 26 | train_generator = train_datagen.flow_from_directory( 27 | train_path, 28 | target_size=(img_size, img_size), 29 | batch_size=BATCH_SIZE, 30 | seed = 13, 31 | multi_outputs=True, # If the model has multi outputs: True, else False.(default is False) 32 | out_n = 2 # write the number of outputs that the model has here. 33 | ) 34 | 35 | validation_generator = test_datagen.flow_from_directory( 36 | test_path, 37 | target_size=(img_size, img_size), 38 | batch_size=BATCH_SIZE, 39 | seed = 13, 40 | multi_outputs=True, # If the model has multi outputs: True, else False.(default is False) 41 | out_n = 2 # write the number of outputs that the model has here. 42 | ) 43 | 44 | #4. Using Crop generator. 45 | # 46 | # Example: 47 | 48 | def random_crop(img, random_crop_size,seed): 49 | # Note: image_data_format is 'channel_last' 50 | np.random.seed(seed=seed) 51 | assert img.shape[2] == 3 52 | height, width = img.shape[0], img.shape[1] 53 | dy, dx = random_crop_size 54 | x = np.random.randint(0, width - dx + 1) 55 | y = np.random.randint(0, height - dy + 1) 56 | return img[y:(y+dy), x:(x+dx), :] 57 | 58 | 59 | def crop_generator(batches, crop_length , seed): 60 | """Take as input a Keras ImageGen (Iterator) and generate random 61 | crops from the image batches generated by the original iterator. 62 | """ 63 | while True: 64 | batch_x, batch_y = next(batches) 65 | batch_crops = np.zeros((batch_x.shape[0], crop_length, crop_length, 3)) 66 | for i in range(batch_x.shape[0]): 67 | batch_crops[i] = random_crop(batch_x[i], (crop_length, crop_length),seed) 68 | yield (batch_crops, batch_y) 69 | 70 | seed = 13 71 | crop_size = 392 72 | train_generator_cropped = crop_generator(train_generator, crop_size, seed) 73 | validation_generator_cropped = crop_generator(validation_generator, crop_size, seed) -------------------------------------------------------------------------------- /__pycache__/grad_cam.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xcnkx/fine_grained_classification/26797fb79d89242330f06ac405eb1bde925769e8/__pycache__/grad_cam.cpython-36.pyc -------------------------------------------------------------------------------- /__pycache__/otsu.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xcnkx/fine_grained_classification/26797fb79d89242330f06ac405eb1bde925769e8/__pycache__/otsu.cpython-36.pyc -------------------------------------------------------------------------------- /__pycache__/se_block.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xcnkx/fine_grained_classification/26797fb79d89242330f06ac405eb1bde925769e8/__pycache__/se_block.cpython-36.pyc -------------------------------------------------------------------------------- /__pycache__/se_inception_v3.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xcnkx/fine_grained_classification/26797fb79d89242330f06ac405eb1bde925769e8/__pycache__/se_inception_v3.cpython-36.pyc -------------------------------------------------------------------------------- /__pycache__/se_inception_v3_1.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xcnkx/fine_grained_classification/26797fb79d89242330f06ac405eb1bde925769e8/__pycache__/se_inception_v3_1.cpython-36.pyc -------------------------------------------------------------------------------- /cifar10_resnet50.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Thu Oct 25 07:46:49 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | #!/usr/bin/env python3 9 | # -*- coding: utf-8 -*- 10 | """ 11 | Created on Tue Oct 23 08:45:20 2018 12 | 13 | @author: n-kamiya 14 | """ 15 | 16 | from keras.applications.resnet50 import ResNet50 17 | from keras.applications.resnet50 import preprocess_input 18 | from keras.models import Sequential, Model 19 | from keras.layers import Input, Dense, Dropout, Activation, Flatten 20 | from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, AveragePooling2D 21 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 22 | from keras.optimizers import SGD 23 | from keras.utils.np_utils import to_categorical 24 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array 25 | from keras.regularizers import l2 26 | import matplotlib.image as mpimg 27 | import numpy as np 28 | import keras.backend as K 29 | import pathlib 30 | from PIL import Image 31 | import cv2 32 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 33 | import matplotlib.pyplot as plt 34 | from keras.utils.vis_utils import plot_model 35 | import pandas as pd 36 | from keras.datasets import mnist 37 | #%% 38 | 39 | K.clear_session() 40 | BATCH_SIZE = 32 41 | test_nb = 5794 42 | train_nb = 5994 43 | num_classes = 200 44 | img_size= 32 45 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 46 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 47 | #%% create data generator 48 | from keras.datasets import cifar10 49 | 50 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 51 | y_train = to_categorical(y_train, num_classes) 52 | y_test = to_categorical(y_test, num_classes) 53 | 54 | datagen = ImageDataGenerator( 55 | featurewise_center=True, 56 | featurewise_std_normalization=True, 57 | rotation_range=20, 58 | width_shift_range=0.2, 59 | height_shift_range=0.2, 60 | horizontal_flip=True) 61 | #%% finetuning resnet50 62 | 63 | input_tensor = Input(shape=(img_size, img_size, 3)) 64 | base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 65 | 66 | # change only the output layer to a FC that it's output is a softmax layer 67 | top_model = Sequential() 68 | top_model.add(Flatten(input_shape=base_model.output_shape[1:])) 69 | top_model.add(Dense(1024,activation="relu")) 70 | top_model.add(Dropout(0.5)) 71 | top_model.add(Dense(num_classes, activation="softmax")) 72 | 73 | model = Model(input=base_model.input, output=top_model(base_model.output)) 74 | 75 | for layer in base_model.layers: 76 | layer.trainable = False 77 | 78 | opt = SGD(lr=0.01, momentum=0.9) 79 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 80 | 81 | #%% 82 | plot_model(model, to_file="model.png", show_shapes=True) 83 | 84 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 85 | checkpointer = ModelCheckpoint(filepath='model.{epoch:02d}-{val_loss:.2f}.hdf5', verbose=1, save_best_only=True) 86 | 87 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, 88 | patience=5, min_lr=0.001) 89 | #%% fit_generator 90 | history = model.fit_generator(datagen.flow(x_train, y_train, batch_size=32), 91 | steps_per_epoch=len(x_train) / 32, epochs=10) 92 | #%%plot history 93 | plt.plot(history.history['acc']) 94 | plt.plot(history.history['val_acc']) 95 | plt.title('model accuracy') 96 | plt.ylabel('accuracy') 97 | plt.xlabel('epoch') 98 | plt.legend(['train', 'test'], loc='upper left') 99 | plt.show() 100 | #loss 101 | plt.plot(history.history['loss']) 102 | plt.plot(history.history['val_loss']) 103 | plt.title('model loss') 104 | plt.ylabel('loss') 105 | plt.xlabel('epoch') 106 | plt.legend(['train', 'test'], loc='upper left') 107 | plt.show() -------------------------------------------------------------------------------- /database/VMMRdb_3040_db.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | -------------------------------------------------------------------------------- /database/__pycache__/cub2002011_db.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xcnkx/fine_grained_classification/26797fb79d89242330f06ac405eb1bde925769e8/database/__pycache__/cub2002011_db.cpython-36.pyc -------------------------------------------------------------------------------- /database/cub2002011_db.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Thu Oct 25 02:19:33 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | import sqlite3 10 | 11 | image_txt = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011" 12 | images_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/images" 13 | root = image_txt 14 | 15 | def create_cub_db(): 16 | 17 | 18 | dbpath = "/home/n-kamiya/fine-grained_project/database/fg_project.db" 19 | 20 | con = sqlite3.connect(dbpath) 21 | 22 | cr = con.cursor() 23 | 24 | 25 | cr.execute("DROP TABLE IF EXISTS CUB2002011") 26 | cr.execute("""CREATE TABLE CUB2002011 27 | (id INTEGER PRIMARY KEY, 28 | class INTEGER, 29 | train INTEGER, 30 | path TEXT);""") 31 | 32 | 33 | with open("%s/images.txt" % root) as f, open("%s/image_class_labels.txt" % root) as g, open("%s/train_test_split.txt" % root) as h: 34 | for line in f.readlines(): 35 | data = line.split() 36 | cr.execute("INSERT INTO CUB2002011 VALUES (?,NULL,NULL,?)",(data[0], data[1])) 37 | for line in g.readlines(): 38 | data = line.split() 39 | cr.execute("UPDATE CUB2002011 SET class=? WHERE id=?", (data[1], data[0])) 40 | for line in h.readlines(): 41 | data = line.split() 42 | cr.execute("UPDATE CUB2002011 SET train=? WHERE id=?", (data[1],data[0])) 43 | 44 | return con , cr -------------------------------------------------------------------------------- /database/fg_project.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xcnkx/fine_grained_classification/26797fb79d89242330f06ac405eb1bde925769e8/database/fg_project.db -------------------------------------------------------------------------------- /elephant.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xcnkx/fine_grained_classification/26797fb79d89242330f06ac405eb1bde925769e8/elephant.jpg -------------------------------------------------------------------------------- /grad_cam.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Nov 6 05:48:40 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | import pandas as pd 10 | import numpy as np 11 | import cv2 12 | from keras import backend as K 13 | from keras.preprocessing.image import array_to_img, img_to_array, load_img 14 | from keras.models import load_model 15 | 16 | def Grad_Cam(input_model, x, layer_name, img_size): 17 | ''' 18 | Args: 19 | input_model: model object 20 | x: image(array) 21 | layer_name: convolution layer's name 22 | 23 | Returns: 24 | jetcam: heat map image(array) 25 | 26 | ''' 27 | 28 | # preprocessing 29 | X = np.expand_dims(x, axis=0) 30 | 31 | X = X.astype('float32') 32 | preprocessed_input = X / 255.0 33 | 34 | model = input_model 35 | 36 | # predict class 37 | 38 | predictions = model.predict(preprocessed_input) 39 | class_idx = np.argmax(predictions[0]) 40 | class_output = model.output[:, class_idx] 41 | 42 | 43 | # get gradients 44 | 45 | conv_output = model.get_layer(layer_name).output # layer_name's output 46 | grads = K.gradients(class_output, conv_output)[0] # gradients(loss, variables) 47 | gradient_function = K.function([model.input], [conv_output, grads]) 48 | 49 | output, grads_val = gradient_function([preprocessed_input]) 50 | output, grads_val = output[0], grads_val[0] 51 | 52 | # get mean of weights, multiply with layer output 53 | weights = np.mean(grads_val, axis=(0, 1)) 54 | cam = np.dot(output, weights) 55 | 56 | 57 | # convert to image and combine with heatmap 58 | 59 | cam = cv2.resize(cam, (img_size, img_size), cv2.INTER_LINEAR) 60 | cam = cam / cam.max() 61 | 62 | jetcam = cv2.applyColorMap(np.uint8(255 * cam), cv2.COLORMAP_JET) 63 | jetcam = cv2.cvtColor(jetcam, cv2.COLOR_BGR2RGB) 64 | jetcam = (np.float32(jetcam) + x / 2) 65 | 66 | return jetcam 67 | 68 | #if __name__ == '__main__': 69 | # 70 | # model = load_model("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME.best_loss.hdf5") 71 | # x = img_to_array(load_img('/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/images/020.Yellow_breasted_Chat/Yellow_Breasted_Chat_0039_21654.jpg', target_size=(448,448))) 72 | # newx= np.expand_dims(x,axis=0) 73 | # preds = model.predict(newx) 74 | # print(classes[preds.argmax()]) -------------------------------------------------------------------------------- /history_inceptv3_with_OSME_miru_4481211-152929.json: -------------------------------------------------------------------------------- 1 | {"val_loss": [10.246591061353683, 8.864582240581512, 7.947382465004921, 7.454795256257057, 7.001306653022766, 6.193873332750679, 4.483866907656193, 4.194421414285898, 3.86499185487628, 2.9456360563635826, 2.5503018721938133, 2.9486624273923363, 2.343650532886386, 2.7020222041755915, 2.4073609802871943, 2.1089947358705103, 1.8060105983167887, 1.8176065397734689, 1.9874356966465712, 1.8957695253193378, 1.4525105318753049, 1.7346048164181411, 1.5062734339496877, 1.5107566453516483, 1.6144204470328987, 1.3637592075392604, 1.6930031310766935, 1.549871151568368, 1.5539613102922345, 1.4979913397692144, 1.4835842242464423, 1.511837782803923, 1.526372313965112, 1.6960994945839047, 1.4994846872764058, 1.6542877475731075, 1.4331006351858377, 1.6508613396435976, 1.4418459828011692, 1.5271794666158092, 1.4853378245607018, 1.615090778679587, 1.62842714227736, 1.426763903349638, 1.5122011499479413, 1.6521863816840814, 1.6433216445147991, 1.640762073919177, 1.2826332640834153, 1.4384573656134307, 1.6117986827157438, 1.5405989887838316, 1.488471280084923, 1.6304481392726302, 1.4826727313920856, 1.5335332246031612, 1.4806270927485854, 1.487013426143676, 1.5956348991021514, 1.4760545687749982], "val_attention_branch_loss": [5.286236964166164, 5.258753441274166, 5.212505131959915, 5.125273860991001, 4.88258708268404, 4.195288289419495, 2.797033667564392, 2.3800107277929783, 2.0134262330830097, 1.556271937675774, 1.3512507257983088, 1.4451950439132086, 1.1921652392484248, 1.385252476669848, 1.2255487218499184, 1.0401160288602114, 0.9023787728510797, 0.938411805889394, 1.0013226834125817, 0.9487628331407905, 0.7306907218880951, 0.8457510588923469, 0.7498111925502815, 0.7524440735578537, 0.7922713083680719, 0.677639075787738, 0.8433975817169994, 0.7607894709799439, 0.7638598216642247, 0.739722570637241, 0.7431366494856775, 0.7384766298346221, 0.749988456023857, 0.8406787076964974, 0.7387054470506045, 0.8191331751877442, 0.7020092036109418, 0.8233731454238296, 0.7125018737278879, 0.7549614671507094, 0.7360009538242593, 0.7873358251526952, 0.8101189737208188, 0.7061366206035018, 0.7528169656870887, 0.8139695078724681, 0.8055679895915091, 0.8082466735504568, 0.6455974793061614, 0.711747647728771, 0.7986318541225046, 0.7514709204435348, 0.7318726697703823, 0.8110811719670892, 0.7394368064124137, 0.7616400178521872, 0.7352424730168711, 0.7346831718459725, 0.7854787954129279, 0.742638168274425], "val_perception_branch_loss": [4.9603540897369385, 3.6058288365602493, 2.7348772902041674, 2.329521404579282, 2.118719559162855, 1.998585031764342, 1.6868332205340266, 1.8144106846302748, 1.851565602235496, 1.3893641191534698, 1.199051140807569, 1.5034673719122857, 1.1514852792024612, 1.316769734956324, 1.1818122593685985, 1.0688786967657506, 0.9036318294238299, 0.8791947369528289, 0.9861130251083523, 0.947006682632491, 0.7218198132468387, 0.8888537688180804, 0.7564622367962752, 0.7583125657401979, 0.8221491402946413, 0.6861201348947361, 0.8496055414434522, 0.7890816772123799, 0.7901014867985603, 0.7582687649410218, 0.7404475753428414, 0.7733611543662846, 0.7763838591054082, 0.8554207687266171, 0.7607792452125266, 0.835154575528577, 0.7310914375120774, 0.8274881953839213, 0.7293441074434668, 0.7722179942681353, 0.7493368778959848, 0.827754947473295, 0.8183081676252186, 0.7206272864714265, 0.759384183504153, 0.8382168787688312, 0.837753655621782, 0.8325154015328735, 0.6370357815176249, 0.7267097111325711, 0.8131668323185295, 0.7891280684550984, 0.7565986111294478, 0.8193669681204483, 0.7432359181111678, 0.7718932039570063, 0.7453846187874822, 0.7523302682675421, 0.8101561014191248, 0.7334164030617103], "val_attention_branch_acc": [0.0234375, 0.0234375, 0.029296875, 0.068359375, 0.0654296875, 0.14554455445544554, 0.3779296875, 0.4287109375, 0.484375, 0.5830078125, 0.6298828125, 0.6158415841584158, 0.681640625, 0.619140625, 0.6611328125, 0.7158203125, 0.7578125, 0.7386138613861386, 0.734375, 0.759765625, 0.8115234375, 0.7880859375, 0.80990099009901, 0.8046875, 0.7919921875, 0.8125, 0.7958984375, 0.7978515625, 0.802970297029703, 0.8056640625, 0.806640625, 0.806640625, 0.8212890625, 0.7900390625, 0.799009900990099, 0.787109375, 0.8173828125, 0.7939453125, 0.81640625, 0.808910891089109, 0.82421875, 0.7939453125, 0.7861328125, 0.80859375, 0.796875, 0.7980198019801981, 0.7939453125, 0.79296875, 0.830078125, 0.818359375, 0.79296875, 0.808910891089109, 0.810546875, 0.787109375, 0.8095703125, 0.8076171875, 0.808910891089109, 0.79296875, 0.8017578125, 0.818359375], "val_perception_branch_acc": [0.0185546875, 0.126953125, 0.3046875, 0.3623046875, 0.4326171875, 0.45643564356435645, 0.537109375, 0.51171875, 0.546875, 0.615234375, 0.6669921875, 0.6188118811881188, 0.7119140625, 0.62890625, 0.6884765625, 0.724609375, 0.763671875, 0.7683168316831683, 0.7509765625, 0.7646484375, 0.806640625, 0.7822265625, 0.808910891089109, 0.8046875, 0.796875, 0.8203125, 0.806640625, 0.7978515625, 0.800990099009901, 0.806640625, 0.810546875, 0.8046875, 0.8212890625, 0.791015625, 0.8, 0.791015625, 0.8154296875, 0.7939453125, 0.82421875, 0.808910891089109, 0.8134765625, 0.791015625, 0.7880859375, 0.8173828125, 0.8115234375, 0.8, 0.7919921875, 0.80078125, 0.8291015625, 0.81640625, 0.7978515625, 0.807920792079208, 0.8193359375, 0.7939453125, 0.8056640625, 0.8115234375, 0.8118811881188119, 0.7958984375, 0.8037109375, 0.8193359375], "loss": [10.497290160364019, 9.545520069681091, 8.227981408914408, 7.3118355645710205, 6.619833082288832, 5.851205764709412, 4.458409670555158, 3.1866839992471006, 2.3887982000937096, 1.8750005471614906, 1.5034595595465765, 1.2218592502611814, 0.9898300489505848, 0.8281772281871385, 0.7058259302033637, 0.5564350618057582, 0.47969450585079226, 0.36689214396962017, 0.2908722714738206, 0.24793292254378568, 0.20569397920284582, 0.15391246053151858, 0.14656172290538844, 0.13210154951493822, 0.12859027731814382, 0.13407531230498204, 0.12896432749700817, 0.13130221413718807, 0.1260027058141646, 0.125055830917002, 0.12118771719483085, 0.12576588326030308, 0.1342875606215394, 0.13310175840262775, 0.1310816124345129, 0.12914791736039552, 0.12787154962271105, 0.12966622604622138, 0.13253972907841186, 0.1281793468796813, 0.13282857985696594, 0.13307960106207206, 0.13161258806198248, 0.12517989135739163, 0.13184807854550737, 0.12004787949947743, 0.1324258305338728, 0.11641457808148992, 0.12552515305218237, 0.13317317272230988, 0.13054548996983267, 0.1248639254548926, 0.12739858885466912, 0.13105690017874574, 0.12853926833387133, 0.12647725699154344, 0.1309509959903446, 0.12486433266380209, 0.1275189352539686, 0.1305483793482448], "attention_branch_loss": [5.2989553924715835, 5.277317991406273, 5.240275974388237, 5.180572132687192, 5.05047611940452, 4.590701527224806, 3.4050439077096657, 2.2513803272673716, 1.5598946701577396, 1.1758178897726563, 0.9138149950637156, 0.7291573266765059, 0.5821574782187596, 0.48395737974811565, 0.4089178176056674, 0.3229138355952006, 0.2744324925166987, 0.21125243351107087, 0.16804298141975582, 0.14278853896223467, 0.12073802395854587, 0.09171985416192391, 0.0877400158016054, 0.07990454971382607, 0.07704457151897756, 0.08017132964518454, 0.07746767519930024, 0.07895498543052941, 0.07535170350804264, 0.0752514287416721, 0.07354852573589997, 0.07533080063444557, 0.08011455316821217, 0.07844976934644433, 0.07767701059241751, 0.0773859202692737, 0.07676041443963666, 0.07757223085498746, 0.07891880590089528, 0.07829104185919783, 0.07827833860933602, 0.07966620001468334, 0.07863963059496792, 0.07575937058971768, 0.07841637466086679, 0.07262998559314249, 0.07905054492158098, 0.07087781716450119, 0.0760765563140999, 0.07920033656978034, 0.07822040368268042, 0.07540110604570911, 0.07654550818749416, 0.07777958998470892, 0.07659451657527679, 0.07599772669904463, 0.07855382948507258, 0.07505772807994285, 0.07745500778034205, 0.07783186233089413], "perception_branch_loss": [5.198334785712175, 4.268202052818046, 2.9877054175019544, 2.131263427826656, 1.5693569550086228, 1.2605042508891873, 1.0533657607771294, 0.9353036786621318, 0.8289035270720829, 0.6991826551613666, 0.5896445645226372, 0.4927019240421019, 0.4076725707467413, 0.34421984821030965, 0.29690811206071743, 0.23352122552441804, 0.20526201382135206, 0.15563971033922067, 0.12282929080981272, 0.10514438406383748, 0.08495595521695383, 0.0621926063944548, 0.05882170708389492, 0.052196999987563095, 0.051545705702211765, 0.05390398282884636, 0.05149665248664491, 0.05234722903854138, 0.05065100241923555, 0.04980440233629924, 0.04763919138310749, 0.05043508255003411, 0.05417300740360696, 0.05465198910093164, 0.05340460192910584, 0.05176199755973525, 0.05111113492452903, 0.05209399518750491, 0.05362092286925098, 0.049888304947146106, 0.05455024145894103, 0.05341340119157746, 0.05297295798410524, 0.04942052087457251, 0.0534317038510794, 0.04741789390633493, 0.053375285806200844, 0.04553676112705682, 0.04944859659265072, 0.05397283613761346, 0.05232508682661706, 0.04946281949495092, 0.05085308073181131, 0.053277309627225884, 0.05194475174243545, 0.05047953076919178, 0.052397166704153036, 0.04980660456024212, 0.050063927358026936, 0.052716517029780724], "attention_branch_acc": [0.005338672005338672, 0.013179846513179846, 0.02485819152485819, 0.05155155155403757, 0.08575241908575242, 0.12912912914901722, 0.33316649984311053, 0.5397063730595945, 0.6604938271604939, 0.7440774107241893, 0.7946279612747399, 0.8353353353552234, 0.8748748748748749, 0.8947280614146161, 0.9094094093895213, 0.9364364364563246, 0.9454454454056692, 0.965965965965966, 0.9758091424758092, 0.9789789789392028, 0.9864864864864865, 0.9923256589923256, 0.9934934934934935, 0.9951618284951619, 0.9961628294961629, 0.9946613279946613, 0.9928261594928262, 0.9951618284951619, 0.9951618284951619, 0.9958291624958292, 0.994994994994995, 0.9953286619953287, 0.993993993993994, 0.9944944944944945, 0.9941608274941608, 0.9951618284951619, 0.9956623289956623, 0.9951618284951619, 0.9934934934934935, 0.9958291624958292, 0.9943276609943277, 0.9948281614550519, 0.9936603269936604, 0.9958291624958292, 0.9956623289956623, 0.9966633299966633, 0.9943276609943277, 0.996996996996997, 0.9958291624958292, 0.9941608274941608, 0.9949949950148831, 0.9943276609943277, 0.994994994994995, 0.9944944944547183, 0.9961628294961629, 0.9954954954954955, 0.9953286619953287, 0.9963296629963296, 0.9956623289956623, 0.9946613279946613], "perception_branch_acc": [0.010677344010677344, 0.07307307309296118, 0.2460794127659675, 0.4202535869600298, 0.5542208875542209, 0.6376376376575258, 0.6921921921723041, 0.7237237237436118, 0.7485819152883582, 0.792459125772571, 0.8264931598463813, 0.8553553553155792, 0.8867200533867201, 0.9017350683619588, 0.9165832499364713, 0.9387721053989959, 0.9447781114447781, 0.9607941274607942, 0.9728061394728061, 0.9766433099965314, 0.9846513179846513, 0.9904904904904905, 0.9926593259528831, 0.9944944944944945, 0.994994994994995, 0.9948281614948281, 0.9924924924924925, 0.9943276609943277, 0.992992992992993, 0.9936603269936604, 0.9946613279946613, 0.9948281614948281, 0.9921588254921588, 0.9931598264931598, 0.9934934934934935, 0.993993993993994, 0.9946613279946613, 0.9933266599933267, 0.9924924924924925, 0.9943276609943277, 0.9928261594928262, 0.9936603269936604, 0.9926593259926594, 0.9954954954954955, 0.9933266599933267, 0.9951618284553856, 0.9948281614948281, 0.9956623289956623, 0.9944944944944945, 0.9936603269936604, 0.9933266599535504, 0.9944944944944945, 0.9948281614948281, 0.9931598264931598, 0.9951618284951619, 0.9943276609943277, 0.9951618284951619, 0.9946613279946613, 0.9948281614948281, 0.993993993993994], "lr": [0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.009999999776482582, 0.0009999999310821295, 0.0009999999310821295, 0.0009999999310821295, 0.0009999999310821295, 9.99999901978299e-05, 9.99999901978299e-05, 9.99999901978299e-05, 9.99999901978299e-05, 9.99999901978299e-05, 9.99999883788405e-06, 9.99999883788405e-06, 9.99999883788405e-06, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07, 9.999999974752427e-07]} -------------------------------------------------------------------------------- /mnist_cnn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Thu Oct 18 08:25:38 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | <<<<<<< HEAD 9 | from keras.losses import categorical_crossentropy 10 | from keras.utils import to_categorical 11 | from keras.datasets import mnist 12 | from keras.models import Sequential 13 | from keras.layers import Dense 14 | from keras.layers import Dropout, Flatten,Conv2D, MaxPooling2D 15 | from keras.optimizers import Adadelta 16 | from keras import backend as K 17 | ======= 18 | from keras.applications.resnet50 import ResNet50 19 | from keras.applications.resnet50 import preprocess_input 20 | from keras.models import Sequential, Model 21 | from keras.layers import Input, Dense, Dropout, Activation, Flatten 22 | from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, AveragePooling2D 23 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 24 | from keras.optimizers import SGD 25 | from keras.utils.np_utils import to_categorical 26 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array 27 | from keras.regularizers import l2 28 | import matplotlib.image as mpimg 29 | import numpy as np 30 | import keras.backend as K 31 | import pathlib 32 | from PIL import Image 33 | import cv2 34 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 35 | import matplotlib.pyplot as plt 36 | from keras.utils.vis_utils import plot_model 37 | import pandas as pd 38 | from keras.datasets import mnist 39 | >>>>>>> develop2 40 | 41 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 42 | 43 | 44 | #x_train = x_train.reshape(60000, 784) 45 | #x_test = x_test.reshape(10000, 784) 46 | img_rows, img_cols = 28, 28 47 | 48 | if K.image_data_format() == 'channels_first': # Theano backend 49 | x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols) 50 | x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols) 51 | input_shape = (1, img_rows, img_cols) 52 | else: # Tensorflow backend 53 | x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) 54 | x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) 55 | input_shape = (img_rows, img_cols, 1) 56 | 57 | x_train = x_train.astype('float32') / 255 58 | x_test = x_test.astype('float32') / 255 59 | 60 | y_train = to_categorical(y_train, 10) 61 | y_test = to_categorical(y_test, 10) 62 | 63 | batch_size = 128 64 | num_classes = 10 65 | epochs = 10 66 | 67 | 68 | <<<<<<< HEAD 69 | 70 | model = Sequential() 71 | model.add(Conv2D(32, kernel_size=(3, 3), 72 | activation='relu', 73 | input_shape=input_shape)) 74 | model.add(Conv2D(64, (3, 3), activation='relu')) 75 | model.add(MaxPooling2D(pool_size=(2, 2))) 76 | model.add(Dropout(0.25)) 77 | model.add(Flatten()) 78 | model.add(Dense(128, activation='relu')) 79 | model.add(Dropout(0.5)) 80 | model.add(Dense(num_classes, activation='softmax')) 81 | 82 | model.compile(loss=categorical_crossentropy, 83 | optimizer= Adadelta(), 84 | metrics=['accuracy']) 85 | 86 | ======= 87 | input_tensor = Input(shape=(img_rows, img_cols, 1)) 88 | base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 89 | 90 | # change only the output layer to a FC that it's output is a softmax layer 91 | top_model = Sequential() 92 | top_model.add(Flatten(input_shape=base_model.output_shape[1:])) 93 | top_model.add(Dense(1024,activation="relu")) 94 | top_model.add(Dropout(0.5)) 95 | top_model.add(Dense(num_classes, activation="softmax")) 96 | 97 | model = Model(input=base_model.input, output=top_model(base_model.output)) 98 | 99 | for layer in base_model.layers: 100 | layer.trainable = False 101 | 102 | opt = SGD(lr=0.01, momentum=0.9) 103 | 104 | 105 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['top_3_categorical_accuracy']) 106 | >>>>>>> develop2 107 | model.fit(x_train, y_train, 108 | batch_size=batch_size, 109 | epochs = epochs, 110 | verbose=1, 111 | validation_data=(x_test, y_test)) 112 | 113 | score = model.evaluate(x_test, y_test, verbose=0) 114 | print('Test loss:', score[0]) 115 | print('Test accuracy:', score[1]) -------------------------------------------------------------------------------- /mobilenet_without_MAMC.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Mon Oct 29 08:18:52 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | #!/usr/bin/env python3 10 | # -*- coding: utf-8 -*- 11 | """ 12 | Created on Tue Oct 23 08:45:20 2018 13 | 14 | @author: n-kamiya 15 | """ 16 | 17 | 18 | from keras.applications.mobilenet import MobileNet 19 | from keras.models import Sequential, Model 20 | from keras.layers import Input, Dense, Dropout, Activation, Flatten 21 | from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, AveragePooling2D 22 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 23 | from keras.optimizers import SGD 24 | from keras.utils.np_utils import to_categorical 25 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array 26 | from keras.regularizers import l2 27 | import matplotlib.image as mpimg 28 | import numpy as np 29 | import keras.backend as K 30 | import pathlib 31 | from PIL import Image 32 | import cv2 33 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 34 | import matplotlib.pyplot as plt 35 | from keras.utils.vis_utils import plot_model 36 | import pandas as pd 37 | #%% 38 | 39 | K.clear_session() 40 | BATCH_SIZE = 16 41 | test_nb = 5794 42 | train_nb = 5994 43 | num_classes = 200 44 | img_size= 448 45 | 46 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 47 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 48 | #%% create data generator 49 | train_datagen = ImageDataGenerator(rescale = 1./img_size, 50 | zoom_range=[0.5,1], 51 | rotation_range=90, 52 | horizontal_flip=True, 53 | vertical_flip=True) 54 | test_datagen = ImageDataGenerator(rescale = 1./img_size) 55 | 56 | train_generator = train_datagen.flow_from_directory( 57 | train_path, 58 | target_size=(img_size, img_size), 59 | batch_size=BATCH_SIZE, 60 | seed = 13) 61 | 62 | validation_generator = test_datagen.flow_from_directory( 63 | test_path, 64 | target_size=(img_size, img_size), 65 | batch_size=BATCH_SIZE, 66 | seed = 13, 67 | shuffle=False) 68 | #%% finetuning resnet50 69 | 70 | input_tensor = Input(shape=(img_size, img_size, 3)) 71 | base_model = MobileNet(weights = "imagenet", include_top=False, input_tensor=input_tensor) 72 | 73 | # change only the output layer 74 | top_model = Sequential() 75 | top_model.add(GlobalAveragePooling2D(input_shape=base_model.output_shape[1:])) 76 | #top_model.add(Dense(256,activation='relu')) 77 | top_model.add(Dropout(0.5)) 78 | top_model.add(Dense(num_classes, activation='softmax')) 79 | 80 | model = Model(input=base_model.input, output=top_model(base_model.output)) 81 | 82 | opt = SGD(lr=0.0001, momentum=0.9) 83 | 84 | #base_model.trainable = False 85 | for layer in base_model.layers: 86 | layer.trainable = False 87 | 88 | 89 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 90 | 91 | #%% 92 | plot_model(model, to_file="model.png", show_shapes=True) 93 | 94 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 95 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_alpha.best_loss.hdf5', verbose=1, save_best_only=True) 96 | 97 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, 98 | patience=5, min_lr=0.000001) 99 | #%% fit_generator 100 | 101 | history = model.fit_generator(train_generator, 102 | steps_per_epoch=train_nb/BATCH_SIZE, 103 | epochs=10, 104 | validation_data=validation_generator, 105 | validation_steps=64, 106 | verbose=1, 107 | callbacks=[reduce_lr, checkpointer]) 108 | 109 | #%% plot results 110 | import datetime 111 | now = datetime.datetime.now() 112 | 113 | plt.plot(history.history['acc']) 114 | plt.plot(history.history['val_acc']) 115 | plt.title('model_without_MAMC accuracy') 116 | plt.ylabel('accuracy') 117 | plt.xlabel('epoch') 118 | plt.legend(['train', 'test'], loc='upper left') 119 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_{0:%d%m}-{0:%H%M%S}.png".format(now)) 120 | plt.show() 121 | 122 | #loss 123 | plt.plot(history.history['loss']) 124 | plt.plot(history.history['val_loss']) 125 | plt.title('model_without_MAMC loss') 126 | plt.ylabel('loss') 127 | plt.xlabel('epoch') 128 | plt.legend(['train', 'test'], loc='upper left') 129 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_{0:%d%m}-{0:%H%M%S}.png".format(now)) 130 | plt.show() 131 | 132 | -------------------------------------------------------------------------------- /model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xcnkx/fine_grained_classification/26797fb79d89242330f06ac405eb1bde925769e8/model.png -------------------------------------------------------------------------------- /model_OSME_miru.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 23 08:45:20 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | from keras.callbacks import EarlyStopping 10 | from keras.applications.resnet50 import ResNet50 11 | from keras.applications.inception_v3 import InceptionV3 12 | from keras.models import Sequential, Model 13 | from keras.layers import Input, Dense, Dropout, Activation, Flatten, Multiply, Lambda, concatenate, Add 14 | from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, AveragePooling2D 15 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 16 | from keras.optimizers import SGD 17 | from keras.utils.np_utils import to_categorical 18 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array 19 | from keras.regularizers import l2 20 | import matplotlib.image as mpimg 21 | import numpy as np 22 | import keras.backend as K 23 | import pathlib 24 | from PIL import Image 25 | import cv2 26 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 27 | import matplotlib.pyplot as plt 28 | from keras.utils.vis_utils import plot_model 29 | import pandas as pd 30 | import se_inception_v3_1 as se_inception_v3 31 | from se_inception_v3 import conv2d_bn, squeeze_excite_block 32 | import tensorflow as tf 33 | #%% 34 | 35 | K.clear_session() 36 | BATCH_SIZE = 16 37 | test_nb = 5794 38 | train_nb = 5994 39 | num_classes = 200 40 | img_size= 448 41 | classes = [] 42 | 43 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 44 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 45 | #%% 46 | 47 | 48 | with open("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/classes.txt") as f: 49 | for l in f.readlines(): 50 | data = l.split() 51 | classes.append(data[1]) 52 | 53 | 54 | #%% create data generator 55 | 56 | train_datagen = ImageDataGenerator(rescale = 1./255, 57 | zoom_range=[0.6,1], 58 | rotation_range=30, 59 | horizontal_flip=True) 60 | test_datagen = ImageDataGenerator(rescale = 1./255) 61 | 62 | train_generator = train_datagen.flow_from_directory( 63 | train_path, 64 | target_size=(img_size, img_size), 65 | batch_size=BATCH_SIZE, 66 | seed = 13, 67 | multi_outputs=True, 68 | out_n = 2 69 | ) 70 | 71 | validation_generator = test_datagen.flow_from_directory( 72 | test_path, 73 | target_size=(img_size, img_size), 74 | batch_size=BATCH_SIZE, 75 | seed = 13, 76 | multi_outputs=True, 77 | out_n = 2 78 | ) 79 | #%% finetuning resnet50 80 | 81 | input_tensor = Input(shape=(img_size, img_size, 3)) 82 | #base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 83 | base_model = se_inception_v3.se_inception_v3(include_top=False, input_tensor=input_tensor) 84 | #base_model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME.best_loss.hdf5",by_name=True) 85 | base_model.load_weights("/home/n-kamiya/.keras/models/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5", by_name=True) 86 | 87 | 88 | #for layer in base_model.layers: 89 | # layer.trainable = False 90 | #%% 91 | #%% 92 | 93 | channel_axis = 3 94 | 95 | def osme_block(in_block, ch, ratio=16, name=None): 96 | z = GlobalAveragePooling2D()(in_block) # 1 97 | x = Dense(ch//ratio, activation='relu')(z) # 2 98 | x = Dense(ch, activation='sigmoid', name=name)(x) # 3 99 | return Multiply()([in_block, x]) # 4 100 | 101 | attention = osme_block(base_model.output, base_model.output_shape[3], name='attention1') 102 | output_1 = osme_block(base_model.output, base_model.output_shape[3], name='output_1') 103 | 104 | output_1 = GlobalAveragePooling2D(name="output1")(output_1) 105 | 106 | x = Add()([base_model.output, attention]) 107 | 108 | # mixed 9: 8 x 8 x 2048 109 | def last_block(x): 110 | for i in range(2): 111 | branch1x1 = conv2d_bn(x, 320, 1, 1) 112 | 113 | branch3x3 = conv2d_bn(x, 384, 1, 1) 114 | branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3) 115 | branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1) 116 | branch3x3 = concatenate( 117 | [branch3x3_1, branch3x3_2], axis=channel_axis, name='mixed9_' + str(i)) 118 | 119 | branch3x3dbl = conv2d_bn(x, 448, 1, 1) 120 | branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3) 121 | branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3) 122 | branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1) 123 | branch3x3dbl = concatenate( 124 | [branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis) 125 | 126 | branch_pool = AveragePooling2D( 127 | (3, 3), strides=(1, 1), padding='same')(x) 128 | branch_pool = conv2d_bn(branch_pool, 192, 1, 1) 129 | x = concatenate( 130 | [branch1x1, branch3x3, branch3x3dbl, branch_pool], 131 | axis=channel_axis, 132 | name='mixed' + str(9 + i)) 133 | 134 | # squeeze and excite block 135 | x = squeeze_excite_block(x) 136 | return x 137 | 138 | x = last_block(x) 139 | output_2 = GlobalAveragePooling2D(name='output_2')(x) 140 | 141 | attention_branch = Dense(num_classes, activation='softmax', name='attention_branch')(output_1) 142 | perception_branch = Dense(num_classes, activation='softmax', name='perception_branch')(output_2) 143 | 144 | #%% 145 | ## for these 2 outputs run it requires a custom datagenerator that return a tuple of yield like (x_batch, [y_batch, y_batch]) 146 | model = Model(inputs=base_model.input, outputs=[attention_branch,perception_branch]) 147 | 148 | opt = SGD(lr=0.01, momentum=0.9, decay=0.0005) 149 | 150 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME.best_loss.hdf5") 151 | 152 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 153 | 154 | #%% 155 | plot_model(model, to_file="model_inceptv3_miru.png", show_shapes=True) 156 | 157 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 158 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME_SE_miru_448.best_loss.hdf5', verbose=1, save_best_only=True) 159 | 160 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, 161 | patience=3, min_lr=0.000001) 162 | 163 | #es_cb = EarlyStopping(patience=11) 164 | 165 | #%% fit_generator 166 | STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size 167 | STEP_SIZE_VALID=validation_generator.n//validation_generator.batch_size 168 | 169 | history = model.fit_generator(train_generator, 170 | steps_per_epoch=STEP_SIZE_TRAIN, 171 | epochs=40, 172 | validation_data=validation_generator, 173 | validation_steps=64, 174 | verbose=1, 175 | callbacks=[reduce_lr, checkpointer]) 176 | #%% 177 | 178 | import json 179 | import datetime 180 | now = datetime.datetime.now() 181 | 182 | class MyEncoder(json.JSONEncoder): 183 | def default(self, obj): 184 | if isinstance(obj, np.integer): 185 | return int(obj) 186 | elif isinstance(obj, np.floating): 187 | return float(obj) 188 | elif isinstance(obj, np.ndarray): 189 | return obj.tolist() 190 | else: 191 | return super(MyEncoder, self).default(obj) 192 | 193 | with open('/home/n-kamiya/models/model_without_MAMC/history_inceptv3_with_OSME_miru_448{0:%d%m}-{0:%H%M%S}.json'.format(now), 'w') as f: 194 | json.dump(history.history, f,cls = MyEncoder) 195 | 196 | 197 | #%% plot results 198 | 199 | #plt.plot(history.history['acc']) 200 | #plt.plot(history.history['val_acc']) 201 | #plt.title('model_without_MAMC accuracy') 202 | #plt.ylabel('accuracy') 203 | #plt.xlabel('epoch') 204 | #plt.legend(['train', 'test'], loc='upper left') 205 | #plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_inceptv3_with_OSME_SE_miru_448{0:%d%m}-{0:%H%M%S}.png".format(now)) 206 | ##plt.show() 207 | # 208 | ##loss 209 | #plt.plot(history.history['loss']) 210 | #plt.plot(history.history['val_loss']) 211 | #plt.title('model_without_MAMC loss') 212 | #plt.ylabel('loss') 213 | #plt.xlabel('epoch') 214 | #plt.legend(['train', 'test'], loc='upper left') 215 | #plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_inceptv3_without_OSME_SE_miru_448{0:%d%m}-{0:%H%M%S}.png".format(now)) 216 | ##plt.show() 217 | -------------------------------------------------------------------------------- /model_OSME_miru_beta.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 23 08:45:20 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | from keras.callbacks import EarlyStopping 10 | from keras.applications.resnet50 import ResNet50 11 | from keras.applications.inception_v3 import InceptionV3 12 | from keras.models import Sequential, Model 13 | from keras.layers import Input, Dense, Dropout, Activation, Flatten, Multiply, Lambda, concatenate, Add 14 | from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, AveragePooling2D 15 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 16 | from keras.optimizers import SGD 17 | from keras.utils.np_utils import to_categorical 18 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array 19 | from keras.regularizers import l2 20 | import matplotlib.image as mpimg 21 | import numpy as np 22 | import keras.backend as K 23 | import pathlib 24 | from PIL import Image 25 | import cv2 26 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 27 | import matplotlib.pyplot as plt 28 | from keras.utils.vis_utils import plot_model 29 | import pandas as pd 30 | import se_inception_v3_1 as se_inception_v3 31 | from se_inception_v3 import conv2d_bn, squeeze_excite_block 32 | import tensorflow as tf 33 | #%% 34 | 35 | K.clear_session() 36 | BATCH_SIZE = 16 37 | test_nb = 5794 38 | train_nb = 5994 39 | num_classes = 200 40 | img_size= 448 41 | classes = [] 42 | 43 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 44 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 45 | #%% 46 | 47 | 48 | with open("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/classes.txt") as f: 49 | for l in f.readlines(): 50 | data = l.split() 51 | classes.append(data[1]) 52 | 53 | 54 | #%% create data generator 55 | 56 | train_datagen = ImageDataGenerator(rescale = 1./255, 57 | zoom_range=[0.8,1], 58 | rotation_range=30, 59 | horizontal_flip=True) 60 | test_datagen = ImageDataGenerator(rescale = 1./255) 61 | 62 | train_generator = train_datagen.flow_from_directory( 63 | train_path, 64 | target_size=(img_size, img_size), 65 | batch_size=BATCH_SIZE, 66 | seed = 13, 67 | multi_outputs=True, 68 | out_n = 2 69 | ) 70 | 71 | validation_generator = test_datagen.flow_from_directory( 72 | test_path, 73 | target_size=(img_size, img_size), 74 | batch_size=BATCH_SIZE, 75 | seed = 13, 76 | multi_outputs=True, 77 | out_n = 2 78 | ) 79 | #%% finetuning resnet50 80 | 81 | input_tensor = Input(shape=(img_size, img_size, 3)) 82 | #base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 83 | base_model = se_inception_v3.se_inception_v3(include_top=False, input_tensor=input_tensor) 84 | #base_model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME.best_loss.hdf5",by_name=True) 85 | base_model.load_weights("/home/n-kamiya/.keras/models/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5", by_name=True) 86 | 87 | 88 | #for layer in base_model.layers: 89 | # layer.trainable = False 90 | #%% 91 | #%% 92 | 93 | channel_axis = 3 94 | 95 | def osme_block(in_block, ch, ratio=16, name=None): 96 | z = GlobalAveragePooling2D()(in_block) # 1 97 | x = Dense(ch//ratio, activation='relu')(z) # 2 98 | x = Dense(ch, activation='sigmoid', name=name)(x) # 3 99 | return Multiply()([in_block, x]) # 4 100 | 101 | attention_s1 = osme_block(base_model.output, base_model.output_shape[3], name='attention_s1') 102 | attention_s2 = osme_block(base_model.output, base_model.output_shape[3], name='attention_s2') 103 | 104 | s_1 = osme_block(base_model.output, base_model.output_shape[3], name='s1') 105 | s_2 = osme_block(base_model.output, base_model.output_shape[3], name='s2') 106 | 107 | 108 | fc1 = Flatten()(s_1) 109 | fc2 = Flatten()(s_2) 110 | 111 | fc1 = Dense(512, name='fc1')(fc1) 112 | fc2 = Dense(512, name='fc2')(fc2) 113 | 114 | fc = concatenate([fc1,fc2]) #fc1 + fc2 115 | 116 | x_1 = Add()([base_model.output, attention_s1]) 117 | x_2 = Add()([base_model.output, attention_s2]) 118 | 119 | 120 | # mixed 9: 8 x 8 x 2048 121 | def last_block(x): 122 | for i in range(2): 123 | branch1x1 = conv2d_bn(x, 320, 1, 1) 124 | 125 | branch3x3 = conv2d_bn(x, 384, 1, 1) 126 | branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3) 127 | branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1) 128 | branch3x3 = concatenate( 129 | [branch3x3_1, branch3x3_2], axis=channel_axis) 130 | 131 | branch3x3dbl = conv2d_bn(x, 448, 1, 1) 132 | branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3) 133 | branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3) 134 | branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1) 135 | branch3x3dbl = concatenate( 136 | [branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis) 137 | 138 | branch_pool = AveragePooling2D( 139 | (3, 3), strides=(1, 1), padding='same')(x) 140 | branch_pool = conv2d_bn(branch_pool, 192, 1, 1) 141 | x = concatenate( 142 | [branch1x1, branch3x3, branch3x3dbl, branch_pool], 143 | axis=channel_axis, 144 | ) 145 | 146 | # squeeze and excite block 147 | x = squeeze_excite_block(x) 148 | return x 149 | 150 | x_1 = last_block(x_1) 151 | x_2 = last_block(x_2) 152 | 153 | 154 | output_2_1 = GlobalAveragePooling2D(name='output_2_1')(x_1) 155 | output_2_2 = GlobalAveragePooling2D(name='output_2_2')(x_2) 156 | 157 | output_2 = concatenate([output_2_1,output_2_2]) 158 | 159 | attention_branch = Dense(num_classes, activation='softmax', name='attention_branch')(fc) 160 | perception_branch = Dense(num_classes, activation='softmax', name='perception_branch')(output_2) 161 | 162 | #%% 163 | ## for these 2 outputs run it requires a custom datagenerator that return a tuple of yield like (x_batch, [y_batch, y_batch]) 164 | model = Model(inputs=base_model.input, outputs=[attention_branch,perception_branch]) 165 | 166 | opt = SGD(lr=0.01, momentum=0.9, decay=0.0005) 167 | 168 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME.best_loss.hdf5") 169 | 170 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 171 | 172 | #%% 173 | plot_model(model, to_file="model_inceptv3_miru.png", show_shapes=True) 174 | 175 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 176 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_OSME_SE_miru_beta.best_loss.hdf5', verbose=1, save_best_only=True) 177 | 178 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, 179 | patience=3, min_lr=0.000001) 180 | 181 | #es_cb = EarlyStopping(patience=11) 182 | 183 | #%% fit_generator 184 | STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size 185 | STEP_SIZE_VALID=validation_generator.n//validation_generator.batch_size 186 | 187 | history = model.fit_generator(train_generator, 188 | steps_per_epoch=STEP_SIZE_TRAIN, 189 | epochs=60, 190 | validation_data=validation_generator, 191 | validation_steps=64, 192 | verbose=1, 193 | callbacks=[reduce_lr, checkpointer]) 194 | #%% 195 | 196 | import json 197 | import datetime 198 | now = datetime.datetime.now() 199 | 200 | class MyEncoder(json.JSONEncoder): 201 | def default(self, obj): 202 | if isinstance(obj, np.integer): 203 | return int(obj) 204 | elif isinstance(obj, np.floating): 205 | return float(obj) 206 | elif isinstance(obj, np.ndarray): 207 | return obj.tolist() 208 | else: 209 | return super(MyEncoder, self).default(obj) 210 | 211 | with open('/home/n-kamiya/models/model_without_MAMC/history_inceptv3_with_OSME_miru_beta{0:%d%m}-{0:%H%M%S}.json'.format(now), 'w') as f: 212 | json.dump(history.history, f,cls = MyEncoder) 213 | 214 | 215 | #%% plot results 216 | 217 | #plt.plot(history.history['acc']) 218 | #plt.plot(history.history['val_acc']) 219 | #plt.title('model_without_MAMC accuracy') 220 | #plt.ylabel('accuracy') 221 | #plt.xlabel('epoch') 222 | #plt.legend(['train', 'test'], loc='upper left') 223 | #plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_inceptv3_with_OSME_SE_miru_448{0:%d%m}-{0:%H%M%S}.png".format(now)) 224 | ##plt.show() 225 | # 226 | ##loss 227 | #plt.plot(history.history['loss']) 228 | #plt.plot(history.history['val_loss']) 229 | #plt.title('model_without_MAMC loss') 230 | #plt.ylabel('loss') 231 | #plt.xlabel('epoch') 232 | #plt.legend(['train', 'test'], loc='upper left') 233 | #plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_inceptv3_without_OSME_SE_miru_448{0:%d%m}-{0:%H%M%S}.png".format(now)) 234 | ##plt.show() 235 | -------------------------------------------------------------------------------- /model_OSME_miru_crop_dog_APN.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 23 08:45:20 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | from keras.callbacks import EarlyStopping 9 | from keras.applications.resnet50 import ResNet50 10 | from keras.applications.inception_v3 import InceptionV3 11 | from keras.models import Sequential, Model 12 | from keras.layers import Input, Dense, Dropout, Activation, Flatten, Multiply, Lambda, concatenate, Add 13 | from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, AveragePooling2D 14 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 15 | from keras.optimizers import SGD 16 | from keras.utils.np_utils import to_categorical 17 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array 18 | from keras.regularizers import l2 19 | import matplotlib.image as mpimg 20 | import numpy as np 21 | import keras.backend as K 22 | import pathlib 23 | from PIL import Image 24 | import cv2 25 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 26 | import matplotlib.pyplot as plt 27 | from keras.utils.vis_utils import plot_model 28 | import pandas as pd 29 | import se_inception_v3_1 as se_inception_v3 30 | from se_inception_v3 import conv2d_bn, squeeze_excite_block 31 | import tensorflow as tf 32 | from otsu import otsu 33 | 34 | #%% 35 | 36 | K.clear_session() 37 | BATCH_SIZE = 16 38 | train_nb = 18571 39 | num_classes = 120 40 | img_size= 448 41 | classes = [] 42 | 43 | 44 | train_path = "/home/n-kamiya/datasets/Standford_dogs_dataset/Images" 45 | 46 | #%% create data generator 47 | 48 | train_datagen = ImageDataGenerator(rescale = 1./255, 49 | zoom_range=[0.8,1], 50 | rotation_range=30, 51 | horizontal_flip=True, 52 | validation_split=0.1) 53 | test_datagen = ImageDataGenerator(rescale = 1./255) 54 | 55 | train_generator = train_datagen.flow_from_directory( 56 | train_path, 57 | target_size=(img_size, img_size), 58 | batch_size=BATCH_SIZE, 59 | seed = 13, 60 | multi_outputs=True, 61 | subset="training", 62 | out_n=2 63 | ) 64 | 65 | validation_generator = train_datagen.flow_from_directory( 66 | train_path, 67 | target_size=(img_size, img_size), 68 | batch_size=BATCH_SIZE, 69 | seed = 13, 70 | multi_outputs=True, 71 | subset="validation", 72 | out_n=2 73 | ) 74 | 75 | #%% add random crop function 76 | def random_crop(img, random_crop_size,seed): 77 | # Note: image_data_format is 'channel_last' 78 | np.random.seed(seed=seed) 79 | assert img.shape[2] == 3 80 | height, width = img.shape[0], img.shape[1] 81 | dy, dx = random_crop_size 82 | x = np.random.randint(0, width - dx + 1) 83 | y = np.random.randint(0, height - dy + 1) 84 | return img[y:(y+dy), x:(x+dx), :] 85 | 86 | 87 | def crop_generator(batches, crop_length , seed): 88 | """Take as input a Keras ImageGen (Iterator) and generate random 89 | crops from the image batches generated by the original iterator. 90 | """ 91 | while True: 92 | batch_x, batch_y = next(batches) 93 | batch_crops = np.zeros((batch_x.shape[0], crop_length, crop_length, 3)) 94 | for i in range(batch_x.shape[0]): 95 | batch_crops[i] = random_crop(batch_x[i], (crop_length, crop_length),seed) 96 | yield (batch_crops, batch_y) 97 | 98 | seed = 13 99 | crop_size = 392 100 | train_generator_cropped = crop_generator(train_generator, crop_size, seed) 101 | validation_generator_cropped = crop_generator(validation_generator, crop_size, seed) 102 | #%% finetuning resnet50 103 | 104 | input_tensor = Input(shape=(crop_size, crop_size, 3)) 105 | #base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 106 | base_model = se_inception_v3.se_inception_v3(include_top=False, input_tensor=input_tensor) 107 | #base_model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME.best_loss.hdf5",by_name=True) 108 | base_model.load_weights("/home/n-kamiya/.keras/models/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5", by_name=True) 109 | 110 | 111 | for layer in base_model.layers: 112 | layer.trainable = False 113 | #%% 114 | #%% 115 | 116 | channel_axis = 3 117 | 118 | def osme_block(in_block, ch, ratio=16, name=None): 119 | z = GlobalAveragePooling2D()(in_block) # 1 120 | x = Dense(ch//ratio, activation='relu')(z) # 2 121 | x = Dense(ch, activation='sigmoid', name=name)(x) # 3 122 | # print(x.shape) 123 | return Multiply()([in_block,x]) # 4 124 | 125 | attention = osme_block(base_model.output, base_model.output_shape[3], name='attention1') 126 | output_1 = osme_block(base_model.output, base_model.output_shape[3], name='output_1') 127 | 128 | #%% 129 | crop_branch = output_1 130 | output_1 = GlobalAveragePooling2D()(output_1) 131 | 132 | attention = Convolution2D(1,(1,1), name='attetion_map', activation='sigmoid')(attention) 133 | crop_branch = Convolution2D(1,(1,1), name='attetion_map_crop', activation='sigmoid')(crop_branch) 134 | 135 | #%% 136 | # TODO: implemetation of crop layer 137 | def APN(input_list): 138 | # get input tensor 139 | input_tensor = input_list[0] 140 | 141 | global crop_size 142 | # get input_image tensor 143 | input_image = input_list[1] 144 | 145 | # assert it is a square image 146 | assert input_tensor.get_shape()[1] == input_tensor.get_shape()[2] 147 | # get length 148 | length = int(input_tensor.get_shape()[1]) 149 | 150 | # create a zeros tensor for cropped image 151 | # cropped_imgs = K.zeros((0, int(crop_size/2), int(crop_size/2), 1)) 152 | cropped_imgs = [] 153 | 154 | tensor = K.reshape(input_tensor , shape=(tf.shape(input_tensor)[0], int(input_tensor.get_shape()[1])*(input_tensor.get_shape()[2]) )) 155 | # 156 | # max_nums = K.argmax(tensor[1]) 157 | 158 | def condition(i, x): 159 | # Tensor("while/Merge:0", shape=(), dtype=int32) Tensor("while/Merge_1:0", shape=(), dtype=int32) 160 | return i < BATCH_SIZE 161 | 162 | def crop(i,x): 163 | 164 | max_value = K.argmax(x[i]) 165 | 166 | _x = max_value % length 167 | _y = max_value / length 168 | 169 | x = tf.scalar_mul(int(crop_size/length), _x) # 2 * 3 170 | y = tf.scalar_mul(int(crop_size/length), _y) # 2 * 3 171 | 172 | x = tf.to_int32(x) 173 | y = tf.to_int32(y) 174 | 175 | l = crop_size/2 176 | margin = l/2 177 | 178 | margin = K.constant(int(margin), dtype=tf.int32) 179 | crop_s = K.constant(int(crop_size), dtype=tf.int32) 180 | 181 | diff = tf.subtract(crop_s, margin) 182 | 183 | # if x < margin or x > (crop_s-margin): 184 | # x = marginx = 185 | cond_1 = tf.math.logical_or(tf.math.greater(x, diff),tf.math.less(x, margin)) 186 | 187 | # 188 | # if y < margin or y > (crop_s-margin): 189 | # y = margin 190 | 191 | cond_2 = tf.math.logical_or(tf.math.less(y, margin),tf.math.greater(y, diff)) 192 | 193 | x = tf.cond(cond_1, lambda: margin , lambda: x) 194 | y = tf.cond(cond_2, lambda: margin, lambda: y) 195 | 196 | 197 | cropped_imgs.append(input_image[:, x-margin:(x+margin), y-margin:(y+margin), :]) 198 | 199 | i += 1 200 | 201 | tf.while_loop(crop, condition, (0, tensor)) 202 | 203 | 204 | 205 | #return batch tensor 206 | return cropped_imgs 207 | 208 | #%% 209 | #TODO: impementation of Lamda layer and shape 210 | _input_tensor = Input(shape=(crop_size, crop_size, 3)) 211 | 212 | cropped_input = Lambda(APN)([crop_branch, _input_tensor]) 213 | 214 | 215 | 216 | 217 | #%% 218 | #attention = otsu(attention) 219 | 220 | x = Add()([base_model.output, attention]) 221 | 222 | # mixed 9: 8 x 8 x 2048 223 | def last_block(x): 224 | for i in range(2): 225 | branch1x1 = conv2d_bn(x, 320, 1, 1) 226 | 227 | branch3x3 = conv2d_bn(x, 384, 1, 1) 228 | branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3) 229 | branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1) 230 | branch3x3 = concatenate( 231 | [branch3x3_1, branch3x3_2], axis=channel_axis, name='mixed9_' + str(i)) 232 | 233 | branch3x3dbl = conv2d_bn(x, 448, 1, 1) 234 | branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3) 235 | branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3) 236 | branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1) 237 | branch3x3dbl = concatenate( 238 | [branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis) 239 | 240 | branch_pool = AveragePooling2D( 241 | (3, 3), strides=(1, 1), padding='same')(x) 242 | branch_pool = conv2d_bn(branch_pool, 192, 1, 1) 243 | x = concatenate( 244 | [branch1x1, branch3x3, branch3x3dbl, branch_pool], 245 | axis=channel_axis, 246 | name='mixed' + str(9 + i)) 247 | 248 | # squeeze and excite block 249 | x = squeeze_excite_block(x) 250 | return x 251 | 252 | x = last_block(x) 253 | 254 | output_2 = GlobalAveragePooling2D(name='output_2')(x) 255 | 256 | attention_branch = Dense(num_classes, activation='softmax', name='attention_branch')(output_1) 257 | perception_branch = Dense(num_classes, activation='softmax', name='perception_branch')(output_2) 258 | 259 | 260 | 261 | #%% 262 | ## for these 2 outputs run it requires a custom datagenerator that return a tuple of yield like (x_batch, [y_batch, y_batch]) 263 | model = Model(inputs=base_model.input, outputs=[attention_branch,perception_branch]) 264 | 265 | opt = SGD(lr=0.01, momentum=0.9, decay=0.0005) 266 | 267 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME.best_loss.hdf5") 268 | 269 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 270 | 271 | #%% 272 | plot_model(model, to_file="model_inceptv3_miru.png", show_shapes=True) 273 | 274 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 275 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_inceptv3_OSME_SE_miru_dogs_test1.best_loss.hdf5', verbose=1, save_best_only=True) 276 | 277 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, 278 | patience=3, min_lr=0.000001) 279 | 280 | #es_cb = EarlyStopping(patience=11) 281 | 282 | #%% fit_generator 283 | STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size 284 | STEP_SIZE_VALID=validation_generator.n//validation_generator.batch_size 285 | 286 | history = model.fit_generator(train_generator_cropped, 287 | steps_per_epoch=STEP_SIZE_TRAIN, 288 | epochs=20, 289 | validation_data=validation_generator_cropped, 290 | validation_steps=STEP_SIZE_VALID, 291 | verbose=1, 292 | callbacks=[reduce_lr, checkpointer]) 293 | #%% 294 | 295 | import json 296 | import datetime 297 | now = datetime.datetime.now() 298 | 299 | class MyEncoder(json.JSONEncoder): 300 | def default(self, obj): 301 | if isinstance(obj, np.integer): 302 | return int(obj) 303 | elif isinstance(obj, np.floating): 304 | return float(obj) 305 | elif isinstance(obj, np.ndarray): 306 | return obj.tolist() 307 | else: 308 | return super(MyEncoder, self).default(obj) 309 | 310 | with open('/home/n-kamiya/models/model_without_MAMC/history_inceptv3_with_OSME_miru_dogs_cropped{0:%d%m}-{0:%H%M%S}.json'.format(now), 'w') as f: 311 | json.dump(history.history, f,cls = MyEncoder) 312 | 313 | 314 | #%% plot results 315 | 316 | #plt.plot(history.history['acc']) 317 | #plt.plot(history.history['val_acc']) 318 | #plt.title('model_without_MAMC accuracy') 319 | #plt.ylabel('accuracy') 320 | #plt.xlabel('epoch') 321 | #plt.legend(['train', 'test'], loc='upper left') 322 | #plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_inceptv3_with_OSME_SE_miru_448{0:%d%m}-{0:%H%M%S}.png".format(now)) 323 | ##plt.show() 324 | # 325 | ##loss 326 | #plt.plot(history.history['loss']) 327 | #plt.plot(history.history['val_loss']) 328 | #plt.title('model_without_MAMC loss') 329 | #plt.ylabel('loss') 330 | #plt.xlabel('epoch') 331 | #plt.legend(['train', 'test'], loc='upper left') 332 | #plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_inceptv3_without_OSME_SE_miru_448{0:%d%m}-{0:%H%M%S}.png".format(now)) 333 | ##plt.show() 334 | -------------------------------------------------------------------------------- /model_OSME_miru_crop_dog_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 23 08:45:20 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | from keras.callbacks import EarlyStopping 9 | from keras.applications.resnet50 import ResNet50 10 | from keras.applications.inception_v3 import InceptionV3 11 | from keras.models import Sequential, Model 12 | from keras.layers import Input, Dense, Dropout, Activation, Flatten, Multiply, Lambda, concatenate, Add 13 | from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, AveragePooling2D 14 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 15 | from keras.optimizers import SGD 16 | from keras.utils.np_utils import to_categorical 17 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array 18 | from keras.regularizers import l2 19 | import matplotlib.image as mpimg 20 | import numpy as np 21 | import keras.backend as K 22 | import pathlib 23 | from PIL import Image 24 | import cv2 25 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 26 | import matplotlib.pyplot as plt 27 | from keras.utils.vis_utils import plot_model 28 | import pandas as pd 29 | import se_inception_v3_1 as se_inception_v3 30 | from se_inception_v3 import conv2d_bn, squeeze_excite_block 31 | import tensorflow as tf 32 | from otsu import otsu 33 | 34 | #%% 35 | 36 | K.clear_session() 37 | BATCH_SIZE = 16 38 | train_nb = 18571 39 | num_classes = 120 40 | img_size= 448 41 | classes = [] 42 | 43 | 44 | train_path = "/home/n-kamiya/datasets/Standford_dogs_dataset/Images" 45 | 46 | #%% create data generator 47 | 48 | train_datagen = ImageDataGenerator(rescale = 1./255, 49 | zoom_range=[0.8,1], 50 | rotation_range=30, 51 | horizontal_flip=True, 52 | validation_split=0.1) 53 | test_datagen = ImageDataGenerator(rescale = 1./255) 54 | 55 | train_generator = train_datagen.flow_from_directory( 56 | train_path, 57 | target_size=(img_size, img_size), 58 | batch_size=BATCH_SIZE, 59 | seed = 13, 60 | multi_outputs=True, 61 | out_n = 2, 62 | subset="training" 63 | ) 64 | 65 | validation_generator = train_datagen.flow_from_directory( 66 | train_path, 67 | target_size=(img_size, img_size), 68 | batch_size=BATCH_SIZE, 69 | seed = 13, 70 | multi_outputs=True, 71 | subset="validation", 72 | out_n = 2 73 | ) 74 | 75 | #%% add random crop function 76 | def random_crop(img, random_crop_size,seed): 77 | # Note: image_data_format is 'channel_last' 78 | np.random.seed(seed=seed) 79 | assert img.shape[2] == 3 80 | height, width = img.shape[0], img.shape[1] 81 | dy, dx = random_crop_size 82 | x = np.random.randint(0, width - dx + 1) 83 | y = np.random.randint(0, height - dy + 1) 84 | return img[y:(y+dy), x:(x+dx), :] 85 | 86 | 87 | def crop_generator(batches, crop_length , seed): 88 | """Take as input a Keras ImageGen (Iterator) and generate random 89 | crops from the image batches generated by the original iterator. 90 | """ 91 | while True: 92 | batch_x, batch_y = next(batches) 93 | batch_crops = np.zeros((batch_x.shape[0], crop_length, crop_length, 3)) 94 | for i in range(batch_x.shape[0]): 95 | batch_crops[i] = random_crop(batch_x[i], (crop_length, crop_length),seed) 96 | yield (batch_crops, batch_y) 97 | 98 | seed = 13 99 | crop_size = 392 100 | train_generator_cropped = crop_generator(train_generator, crop_size, seed) 101 | validation_generator_cropped = crop_generator(validation_generator, crop_size, seed) 102 | 103 | #%% finetuning resnet50 104 | 105 | input_tensor = Input(shape=(crop_size, crop_size, 3)) 106 | #base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 107 | base_model = se_inception_v3.se_inception_v3(include_top=False, input_tensor=input_tensor) 108 | #base_model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME.best_loss.hdf5",by_name=True) 109 | base_model.load_weights("/home/n-kamiya/.keras/models/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5", by_name=True) 110 | 111 | ## 112 | #for layer in base_model.layers: 113 | # layer.trainable = False 114 | #%% 115 | #%% 116 | 117 | channel_axis = 3 118 | 119 | def osme_block(in_block, ch, ratio=16, name=None): 120 | z = GlobalAveragePooling2D()(in_block) # 1 121 | x = Dense(ch//ratio, activation='relu')(z) # 2 122 | x = Dense(ch, activation='sigmoid', name=name)(x) # 3 123 | # print(x.shape) 124 | return Multiply()([in_block,x]) # 4 125 | 126 | attention = osme_block(base_model.output, base_model.output_shape[3], name='attention1') 127 | output_1 = osme_block(base_model.output, base_model.output_shape[3], name='output_1') 128 | 129 | output_1 = GlobalAveragePooling2D()(output_1) 130 | 131 | attention = Convolution2D(1,(1,1), name='attention_map', activation='relu' )(attention) 132 | 133 | #%% 134 | #attention = otsu(attention) 135 | 136 | x = Add()([base_model.output, attention]) 137 | 138 | # mixed 9: 8 x 8 x 2048 139 | def last_block(x): 140 | for i in range(2): 141 | branch1x1 = conv2d_bn(x, 320, 1, 1) 142 | 143 | branch3x3 = conv2d_bn(x, 384, 1, 1) 144 | branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3) 145 | branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1) 146 | branch3x3 = concatenate( 147 | [branch3x3_1, branch3x3_2], axis=channel_axis, name='mixed9_' + str(i)) 148 | 149 | branch3x3dbl = conv2d_bn(x, 448, 1, 1) 150 | branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3) 151 | branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3) 152 | branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1) 153 | branch3x3dbl = concatenate( 154 | [branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis) 155 | 156 | branch_pool = AveragePooling2D( 157 | (3, 3), strides=(1, 1), padding='same')(x) 158 | branch_pool = conv2d_bn(branch_pool, 192, 1, 1) 159 | x = concatenate( 160 | [branch1x1, branch3x3, branch3x3dbl, branch_pool], 161 | axis=channel_axis, 162 | name='mixed' + str(9 + i)) 163 | 164 | # squeeze and excite block 165 | x = squeeze_excite_block(x) 166 | return x 167 | 168 | x = last_block(x) 169 | 170 | output_2 = GlobalAveragePooling2D(name='output_2')(x) 171 | 172 | attention_branch = Dense(num_classes, activation='softmax', name='attention_branch')(output_1) 173 | perception_branch = Dense(num_classes, activation='softmax', name='perception_branch')(output_2) 174 | 175 | 176 | 177 | #%% 178 | ## for these 2 outputs run it requires a custom datagenerator that return a tuple of yield like (x_batch, [y_batch, y_batch]) 179 | model = Model(inputs=base_model.input, outputs=[attention_branch,perception_branch]) 180 | 181 | opt = SGD(lr=0.01, momentum=0.9, decay=0.0005) 182 | 183 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME.best_loss.hdf5") 184 | 185 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 186 | 187 | #%% 188 | plot_model(model, to_file="model_inceptv3_miru.png", show_shapes=True) 189 | 190 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 191 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_inceptv3_OSME_SE_miru_dogs_test2.best_loss.hdf5', verbose=1, save_best_only=True) 192 | 193 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, 194 | patience=3, min_lr=0.000001) 195 | 196 | #es_cb = EarlyStopping(patience=11) 197 | 198 | #%% fit_generator 199 | STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size 200 | STEP_SIZE_VALID=validation_generator.n//validation_generator.batch_size 201 | 202 | history = model.fit_generator(train_generator_cropped, 203 | steps_per_epoch=STEP_SIZE_TRAIN, 204 | epochs=80, 205 | validation_data=validation_generator_cropped, 206 | validation_steps=STEP_SIZE_VALID, 207 | verbose=1, 208 | callbacks=[reduce_lr, checkpointer]) 209 | #%% 210 | 211 | import json 212 | import datetime 213 | now = datetime.datetime.now() 214 | 215 | class MyEncoder(json.JSONEncoder): 216 | def default(self, obj): 217 | if isinstance(obj, np.integer): 218 | return int(obj) 219 | elif isinstance(obj, np.floating): 220 | return float(obj) 221 | elif isinstance(obj, np.ndarray): 222 | return obj.tolist() 223 | else: 224 | return super(MyEncoder, self).default(obj) 225 | 226 | with open('/home/n-kamiya/models/model_without_MAMC/history_inceptv3_with_OSME_miru_dogs_cropped{0:%d%m}-{0:%H%M%S}.json'.format(now), 'w') as f: 227 | json.dump(history.history, f,cls = MyEncoder) 228 | 229 | 230 | #%% plot results 231 | 232 | #plt.plot(history.history['acc']) 233 | #plt.plot(history.history['val_acc']) 234 | #plt.title('model_without_MAMC accuracy') 235 | #plt.ylabel('accuracy') 236 | #plt.xlabel('epoch') 237 | #plt.legend(['train', 'test'], loc='upper left') 238 | #plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_inceptv3_with_OSME_SE_miru_448{0:%d%m}-{0:%H%M%S}.png".format(now)) 239 | ##plt.show() 240 | # 241 | ##loss 242 | #plt.plot(history.history['loss']) 243 | #plt.plot(history.history['val_loss']) 244 | #plt.title('model_without_MAMC loss') 245 | #plt.ylabel('loss') 246 | #plt.xlabel('epoch') 247 | #plt.legend(['train', 'test'], loc='upper left') 248 | #plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_inceptv3_without_OSME_SE_miru_448{0:%d%m}-{0:%H%M%S}.png".format(now)) 249 | ##plt.show() 250 | -------------------------------------------------------------------------------- /model_OSME_miru_dog.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 23 08:45:20 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | from keras.callbacks import EarlyStopping 10 | from keras.applications.resnet50 import ResNet50 11 | from keras.applications.inception_v3 import InceptionV3 12 | from keras.models import Sequential, Model 13 | from keras.layers import Input, Dense, Dropout, Activation, Flatten, Multiply, Lambda, concatenate, Add 14 | from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, AveragePooling2D 15 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 16 | from keras.optimizers import SGD 17 | from keras.utils.np_utils import to_categorical 18 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array 19 | from keras.regularizers import l2 20 | import matplotlib.image as mpimg 21 | import numpy as np 22 | import keras.backend as K 23 | import pathlib 24 | from PIL import Image 25 | import cv2 26 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 27 | import matplotlib.pyplot as plt 28 | from keras.utils.vis_utils import plot_model 29 | import pandas as pd 30 | import se_inception_v3_1 as se_inception_v3 31 | from se_inception_v3 import conv2d_bn, squeeze_excite_block 32 | import tensorflow as tf 33 | #%% 34 | 35 | K.clear_session() 36 | BATCH_SIZE = 16 37 | train_nb = 20580 38 | num_classes = 120 39 | img_size= 448 40 | classes = [] 41 | 42 | 43 | train_path = "/home/n-kamiya/datasets/Standford_dogs_dataset/Images" 44 | 45 | #%% create data generator 46 | 47 | train_datagen = ImageDataGenerator(rescale = 1./255, 48 | zoom_range=[0.8,1], 49 | # rotation_range=30, 50 | horizontal_flip=True, 51 | validation_split=0.1) 52 | test_datagen = ImageDataGenerator(rescale = 1./255) 53 | 54 | train_generator = train_datagen.flow_from_directory( 55 | train_path, 56 | target_size=(img_size, img_size), 57 | batch_size=BATCH_SIZE, 58 | seed = 13, 59 | multi_outputs=True, 60 | subset="training", 61 | ) 62 | 63 | validation_generator = train_datagen.flow_from_directory( 64 | train_path, 65 | target_size=(img_size, img_size), 66 | batch_size=BATCH_SIZE, 67 | seed = 13, 68 | multi_outputs=True, 69 | subset="validation", 70 | ) 71 | #%% finetuning resnet50 72 | 73 | input_tensor = Input(shape=(img_size, img_size, 3)) 74 | #base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 75 | base_model = se_inception_v3.se_inception_v3(include_top=False, input_tensor=input_tensor) 76 | #base_model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME.best_loss.hdf5",by_name=True) 77 | base_model.load_weights("/home/n-kamiya/.keras/models/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5", by_name=True) 78 | 79 | 80 | #for layer in base_model.layers: 81 | # layer.trainable = False 82 | #%% 83 | #%% 84 | 85 | channel_axis = 3 86 | 87 | def osme_block(in_block, ch, ratio=16, name=None): 88 | z = GlobalAveragePooling2D()(in_block) # 1 89 | x = Dense(ch//ratio, activation='relu')(z) # 2 90 | x = Dense(ch, activation='sigmoid', name=name)(x) # 3 91 | return x # 4 92 | 93 | attention = Multiply()([base_model.output,osme_block(base_model.output, base_model.output_shape[3], name='attention1')]) 94 | 95 | output_1 = osme_block(base_model.output, base_model.output_shape[3], name='output_1') 96 | output_1 = GlobalAveragePooling2D(name="output1")(output_1) 97 | 98 | x = Add()([base_model.output, attention]) 99 | 100 | # mixed 9: 8 x 8 x 2048 101 | def last_block(x): 102 | for i in range(2): 103 | branch1x1 = conv2d_bn(x, 320, 1, 1) 104 | 105 | branch3x3 = conv2d_bn(x, 384, 1, 1) 106 | branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3) 107 | branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1) 108 | branch3x3 = concatenate( 109 | [branch3x3_1, branch3x3_2], axis=channel_axis, name='mixed9_' + str(i)) 110 | 111 | branch3x3dbl = conv2d_bn(x, 448, 1, 1) 112 | branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3) 113 | branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3) 114 | branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1) 115 | branch3x3dbl = concatenate( 116 | [branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis) 117 | 118 | branch_pool = AveragePooling2D( 119 | (3, 3), strides=(1, 1), padding='same')(x) 120 | branch_pool = conv2d_bn(branch_pool, 192, 1, 1) 121 | x = concatenate( 122 | [branch1x1, branch3x3, branch3x3dbl, branch_pool], 123 | axis=channel_axis, 124 | name='mixed' + str(9 + i)) 125 | 126 | # squeeze and excite block 127 | x = squeeze_excite_block(x) 128 | return x 129 | 130 | x = last_block(x) 131 | output_2 = GlobalAveragePooling2D(name='output_2')(x) 132 | 133 | attention_branch = Dense(num_classes, activation='softmax', name='attention_branch')(output_1) 134 | perception_branch = Dense(num_classes, activation='softmax', name='perception_branch')(output_2) 135 | 136 | #%% 137 | ## for these 2 outputs run it requires a custom datagenerator that return a tuple of yield like (x_batch, [y_batch, y_batch]) 138 | model = Model(inputs=base_model.input, outputs=[attention_branch,perception_branch]) 139 | 140 | opt = SGD(lr=0.01, momentum=0.9, decay=0.0005) 141 | 142 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME.best_loss.hdf5") 143 | 144 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 145 | 146 | #%% 147 | plot_model(model, to_file="model_inceptv3_miru.png", show_shapes=True) 148 | 149 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 150 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_inceptv3_OSME_SE_miru_dogs_test1.best_loss.hdf5', verbose=1, save_best_only=True) 151 | 152 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, 153 | patience=3, min_lr=0.000001) 154 | 155 | #es_cb = EarlyStopping(patience=11) 156 | 157 | #%% fit_generator 158 | STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size 159 | STEP_SIZE_VALID=validation_generator.n//validation_generator.batch_size 160 | history = model.fit_generator(train_generator, 161 | steps_per_epoch=STEP_SIZE_TRAIN, 162 | epochs=20, 163 | validation_data=validation_generator, 164 | validation_steps=STEP_SIZE_VALID, 165 | verbose=1, 166 | callbacks=[reduce_lr, checkpointer]) 167 | #%% 168 | 169 | import json 170 | import datetime 171 | now = datetime.datetime.now() 172 | 173 | class MyEncoder(json.JSONEncoder): 174 | def default(self, obj): 175 | if isinstance(obj, np.integer): 176 | return int(obj) 177 | elif isinstance(obj, np.floating): 178 | return float(obj) 179 | elif isinstance(obj, np.ndarray): 180 | return obj.tolist() 181 | else: 182 | return super(MyEncoder, self).default(obj) 183 | 184 | with open('/home/n-kamiya/models/model_without_MAMC/history_inceptv3_with_OSME_miru_dogs_test1{0:%d%m}-{0:%H%M%S}.json'.format(now), 'w') as f: 185 | json.dump(history.history, f,cls = MyEncoder) 186 | 187 | 188 | #%% plot results 189 | 190 | #plt.plot(history.history['acc']) 191 | #plt.plot(history.history['val_acc']) 192 | #plt.title('model_without_MAMC accuracy') 193 | #plt.ylabel('accuracy') 194 | #plt.xlabel('epoch') 195 | #plt.legend(['train', 'test'], loc='upper left') 196 | #plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_inceptv3_with_OSME_SE_miru_448{0:%d%m}-{0:%H%M%S}.png".format(now)) 197 | ##plt.show() 198 | # 199 | ##loss 200 | #plt.plot(history.history['loss']) 201 | #plt.plot(history.history['val_loss']) 202 | #plt.title('model_without_MAMC loss') 203 | #plt.ylabel('loss') 204 | #plt.xlabel('epoch') 205 | #plt.legend(['train', 'test'], loc='upper left') 206 | #plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_inceptv3_without_OSME_SE_miru_448{0:%d%m}-{0:%H%M%S}.png".format(now)) 207 | ##plt.show() 208 | -------------------------------------------------------------------------------- /model_inceptv3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xcnkx/fine_grained_classification/26797fb79d89242330f06ac405eb1bde925769e8/model_inceptv3.png -------------------------------------------------------------------------------- /model_inceptv3_miru.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xcnkx/fine_grained_classification/26797fb79d89242330f06ac405eb1bde925769e8/model_inceptv3_miru.png -------------------------------------------------------------------------------- /model_miru.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 23 08:45:20 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | from keras.callbacks import EarlyStopping 10 | from keras.applications.resnet50 import ResNet50 11 | from keras.applications.inception_v3 import InceptionV3 12 | from keras.models import Sequential, Model 13 | from keras.layers import Input, Dense, Dropout, Activation, Flatten, Multiply, Lambda, concatenate, Add 14 | from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, AveragePooling2D 15 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 16 | from keras.optimizers import SGD 17 | from keras.utils.np_utils import to_categorical 18 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array 19 | from keras.regularizers import l2 20 | import matplotlib.image as mpimg 21 | import numpy as np 22 | import keras.backend as K 23 | import pathlib 24 | from PIL import Image 25 | import cv2 26 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 27 | import matplotlib.pyplot as plt 28 | from keras.utils.vis_utils import plot_model 29 | import pandas as pd 30 | import se_inception_v3_1 as se_inception_v3 31 | from se_inception_v3 import conv2d_bn, squeeze_excite_block 32 | import tensorflow as tf 33 | #%% 34 | 35 | K.clear_session() 36 | BATCH_SIZE = 16 37 | test_nb = 5794 38 | train_nb = 5994 39 | num_classes = 200 40 | img_size= 448 41 | classes = [] 42 | 43 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 44 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 45 | #%% 46 | 47 | 48 | with open("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/classes.txt") as f: 49 | for l in f.readlines(): 50 | data = l.split() 51 | classes.append(data[1]) 52 | 53 | 54 | #%% create data generator 55 | 56 | train_datagen = ImageDataGenerator(rescale = 1./255, 57 | zoom_range=[0.8,1], 58 | rotation_range=30, 59 | horizontal_flip=True) 60 | test_datagen = ImageDataGenerator(rescale = 1./255) 61 | 62 | train_generator = train_datagen.flow_from_directory( 63 | train_path, 64 | target_size=(img_size, img_size), 65 | batch_size=BATCH_SIZE, 66 | seed = 13, 67 | multi_outputs=True, 68 | out_n=2 69 | ) 70 | 71 | validation_generator = test_datagen.flow_from_directory( 72 | test_path, 73 | target_size=(img_size, img_size), 74 | batch_size=BATCH_SIZE, 75 | seed = 13, 76 | multi_outputs=True, 77 | out_n=2 78 | ) 79 | #%% finetuning resnet50 80 | 81 | input_tensor = Input(shape=(img_size, img_size, 3)) 82 | #base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 83 | base_model = se_inception_v3.se_inception_v3(include_top=False, input_tensor=input_tensor) 84 | #base_model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME.best_loss.hdf5",by_name=True) 85 | base_model.load_weights("/home/n-kamiya/.keras/models/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5", by_name=True) 86 | 87 | 88 | #for layer in base_model.layers: 89 | # layer.trainable = False 90 | #%% 91 | #%% 92 | 93 | channel_axis = 3 94 | # mixed 9: 8 x 8 x 2048 95 | 96 | 97 | def last_block(x, layer): 98 | for i in range(2): 99 | branch1x1 = conv2d_bn(x, 320, 1, 1) 100 | 101 | branch3x3 = conv2d_bn(x, 384, 1, 1) 102 | branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3) 103 | branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1) 104 | branch3x3 = concatenate( 105 | [branch3x3_1, branch3x3_2], axis=channel_axis, name='mixed9_' + layer + str(i)) 106 | 107 | branch3x3dbl = conv2d_bn(x, 448, 1, 1) 108 | branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3) 109 | branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3) 110 | branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1) 111 | branch3x3dbl = concatenate( 112 | [branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis) 113 | 114 | branch_pool = AveragePooling2D( 115 | (3, 3), strides=(1, 1), padding='same')(x) 116 | branch_pool = conv2d_bn(branch_pool, 192, 1, 1) 117 | x = concatenate( 118 | [branch1x1, branch3x3, branch3x3dbl, branch_pool], 119 | axis=channel_axis, 120 | name='mixed' + str(9 + i)+'_'+layer) 121 | 122 | # squeeze and excite block 123 | x = squeeze_excite_block(x) 124 | return x 125 | 126 | #attention branch 127 | def attention_block(in_block, ch, ratio=16, name=None): 128 | x = last_block(in_block,'attention') 129 | x = Convolution2D(num_classes,(1,1))(x) 130 | print(x.shape) 131 | 132 | return x 133 | 134 | 135 | 136 | x = attention_block(base_model.output, base_model.output_shape[3]) 137 | attention_map = Convolution2D(1,(1,1),activation='sigmoid', name="attention1")(x) 138 | output_1 = Convolution2D(num_classes,(1,1))(x) 139 | output_1 = GlobalAveragePooling2D(name="output1")(output_1) 140 | 141 | x = Multiply()([base_model.output, attention_map]) 142 | x = Add()([base_model.output, x]) 143 | 144 | x = last_block(x,'perception') 145 | output_2 = GlobalAveragePooling2D(name='output_2')(x) 146 | 147 | attention_branch = Dense(num_classes, activation='softmax', name='attention_branch')(output_1) 148 | perception_branch = Dense(num_classes, activation='softmax', name='perception_branch')(output_2) 149 | 150 | #%% 151 | ## for these 2 outputs run it requires a custom datagenerator that return a tuple of yield like (x_batch, [y_batch, y_batch]) 152 | model = Model(inputs=base_model.input, outputs=[attention_branch,perception_branch]) 153 | 154 | opt = SGD(lr=0.01, momentum=0.9, decay=0.0005) 155 | 156 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME.best_loss.hdf5") 157 | 158 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 159 | 160 | #%% 161 | plot_model(model, to_file="model_inceptv3_miru.png", show_shapes=True) 162 | 163 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 164 | #checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_inceptv3_miru_448.best_loss.hdf5', verbose=1, save_best_only=True) 165 | 166 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, 167 | patience=3, min_lr=0.000001) 168 | 169 | #es_cb = EarlyStopping(patience=11) 170 | 171 | #%% fit_generator 172 | 173 | history = model.fit_generator(train_generator, 174 | steps_per_epoch=train_nb/BATCH_SIZE, 175 | epochs=60, 176 | validation_data=validation_generator, 177 | validation_steps=64, 178 | verbose=1, 179 | callbacks=[reduce_lr]) 180 | #%% 181 | 182 | import json 183 | import datetime 184 | now = datetime.datetime.now() 185 | 186 | class MyEncoder(json.JSONEncoder): 187 | def default(self, obj): 188 | if isinstance(obj, np.integer): 189 | return int(obj) 190 | elif isinstance(obj, np.floating): 191 | return float(obj) 192 | elif isinstance(obj, np.ndarray): 193 | return obj.tolist() 194 | else: 195 | return super(MyEncoder, self).default(obj) 196 | 197 | with open('/home/n-kamiya/models/model_without_MAMC/history_inceptv3_miru_448{0:%d%m}-{0:%H%M%S}.json'.format(now), 'w') as f: 198 | json.dump(history.history, f,cls = MyEncoder) 199 | 200 | 201 | #%% plot results 202 | 203 | #plt.plot(history.history['acc']) 204 | #plt.plot(history.history['val_acc']) 205 | #plt.title('model_without_MAMC accuracy') 206 | #plt.ylabel('accuracy') 207 | #plt.xlabel('epoch') 208 | #plt.legend(['train', 'test'], loc='upper left') 209 | #plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_inceptv3_with_OSME_miru_448_{0:%d%m}-{0:%H%M%S}.png".format(now)) 210 | ##plt.show() 211 | # 212 | ##loss 213 | #plt.plot(history.history['loss']) 214 | #plt.plot(history.history['val_loss']) 215 | #plt.title('model_without_MAMC loss') 216 | #plt.ylabel('loss') 217 | #plt.xlabel('epoch') 218 | #plt.legend(['train', 'test'], loc='upper left') 219 | #plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_inceptv3_without_OSME_miru_448_{0:%d%m}-{0:%H%M%S}.png".format(now)) 220 | ##plt.show() 221 | -------------------------------------------------------------------------------- /model_resnet50_with_OSME.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Wed Oct 31 06:11:16 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | #!/usr/bin/env python3 10 | # -*- coding: utf-8 -*- 11 | """ 12 | Created on Tue Oct 23 08:45:20 2018 13 | 14 | @author: n-kamiya 15 | """ 16 | import keras 17 | from keras.callbacks import EarlyStopping 18 | from keras.applications.resnet50 import ResNet50 19 | from keras.applications.vgg19 import VGG19 20 | from keras.models import Model 21 | from keras.layers import Input, Dense, Flatten, Multiply, add 22 | from keras.layers import GlobalAveragePooling2D 23 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 24 | from keras.optimizers import SGD 25 | from keras.utils.np_utils import to_categorical 26 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array 27 | from keras.regularizers import l2 28 | import matplotlib.image as mpimg 29 | import numpy as np 30 | import keras.backend as K 31 | import pathlib 32 | from PIL import Image 33 | import cv2 34 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 35 | import matplotlib.pyplot as plt 36 | from keras.utils.vis_utils import plot_model 37 | import pandas as pd 38 | #%% 39 | 40 | K.clear_session() 41 | BATCH_SIZE = 16 42 | test_nb = 5794 43 | train_nb = 5994 44 | num_classes = 200 45 | img_size= 448 46 | classes = [] 47 | 48 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 49 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 50 | #%% 51 | 52 | 53 | with open("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/classes.txt") as f: 54 | for l in f.readlines(): 55 | data = l.split() 56 | classes.append(data[1]) 57 | 58 | 59 | #%% create data generator 60 | 61 | train_datagen = ImageDataGenerator(rescale = 1.0/255, 62 | zoom_range=[1.0,2.0], 63 | rotation_range=90, 64 | horizontal_flip=True, 65 | vertical_flip=True) 66 | test_datagen = ImageDataGenerator(rescale = 1.0/255) 67 | 68 | train_generator = train_datagen.flow_from_directory( 69 | train_path, 70 | target_size=(img_size, img_size), 71 | batch_size=BATCH_SIZE, 72 | seed = 13) 73 | 74 | validation_generator = test_datagen.flow_from_directory( 75 | test_path, 76 | target_size=(img_size, img_size), 77 | batch_size=BATCH_SIZE, 78 | seed = 13) 79 | #%% finetuning resnet50 80 | 81 | input_tensor = Input(shape=(img_size, img_size, 3)) 82 | #base_model = VGG19(weights = "imagenet", include_top=False, input_tensor=input_tensor) 83 | base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 84 | 85 | #for layer in base_model.layers: 86 | # layer.trainable = False 87 | # if isinstance(layer, keras.layers.normalization.BatchNormalization): 88 | # layer._per_input_updates = {} 89 | 90 | #%% Implementation of OSME module 91 | 92 | def osme_block(in_block, ch, ratio=16): 93 | z = GlobalAveragePooling2D()(in_block) # 1 94 | x = Dense(ch//ratio, activation='relu')(z) # 2 95 | x = Dense(ch, activation='sigmoid')(x) # 3 96 | return Multiply()([in_block, x]) # 4 97 | 98 | s_1 = osme_block(base_model.output, base_model.output_shape[3]) 99 | s_2 = osme_block(base_model.output, base_model.output_shape[3]) 100 | 101 | fc1 = Flatten()(s_1) 102 | fc2 = Flatten()(s_2) 103 | 104 | fc1 = Dense(1024, name='fc1')(fc1) 105 | fc2 = Dense(1024, name='fc2')(fc2) 106 | 107 | 108 | fc = add([fc1,fc2]) # fc1 + fc2 109 | 110 | prediction = Dense(num_classes, activation='softmax', name='prediction')(fc) 111 | 112 | 113 | model = Model(inputs=base_model.input, outputs=prediction) 114 | 115 | opt = SGD(lr=0.001, momentum=0.9, decay=0.0001) 116 | 117 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_osme_vgg_imagenet.best_loss.hdf5") 118 | 119 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 120 | 121 | #%% 122 | plot_model(model, to_file="model.png", show_shapes=True) 123 | 124 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 125 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_osme_resnet50.best_loss.hdf5', verbose=1, save_best_only=True) 126 | 127 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, 128 | patience=5, min_lr=0.0000001) 129 | 130 | #es_cb = EarlyStopping(patience=11) 131 | 132 | #%% fit_generator 133 | 134 | history = model.fit_generator(train_generator, 135 | steps_per_epoch=train_nb/BATCH_SIZE, 136 | epochs=60, 137 | validation_data=validation_generator, 138 | validation_steps=64, 139 | verbose=1, 140 | callbacks=[reduce_lr, checkpointer]) 141 | 142 | #%% plot results 143 | import datetime 144 | now = datetime.datetime.now() 145 | 146 | plt.plot(history.history['acc']) 147 | plt.plot(history.history['val_acc']) 148 | plt.title('model_without_MAMC accuracy') 149 | plt.ylabel('accuracy') 150 | plt.xlabel('epoch') 151 | plt.legend(['train', 'test'], loc='upper left') 152 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_osme_resnet50{0:%d%m}-{0:%H%M%S}.png".format(now)) 153 | #plt.show() 154 | 155 | #loss 156 | plt.plot(history.history['loss']) 157 | plt.plot(history.history['val_loss']) 158 | plt.title('model_without_MAMC loss') 159 | plt.ylabel('loss') 160 | plt.xlabel('epoch') 161 | plt.legend(['train', 'test'], loc='upper left') 162 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_osme_resnet50{0:%d%m}-{0:%H%M%S}.png".format(now)) 163 | #plt.show() 164 | -------------------------------------------------------------------------------- /model_resnet50_with_rescale_no_trainable.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Wed Oct 31 06:13:06 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | #!/usr/bin/env python3 10 | # -*- coding: utf-8 -*- 11 | """ 12 | Created on Tue Oct 23 08:45:20 2018 13 | 14 | @author: n-kamiya 15 | """ 16 | import keras 17 | from keras.callbacks import EarlyStopping 18 | from keras.applications.resnet50 import ResNet50 19 | from keras.applications.vgg19 import VGG19 20 | from keras.models import Model 21 | from keras.layers import Input, Dense, Flatten, Multiply, add 22 | from keras.layers import GlobalAveragePooling2D 23 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 24 | from keras.optimizers import SGD 25 | from keras.utils.np_utils import to_categorical 26 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array 27 | from keras.regularizers import l2 28 | import matplotlib.image as mpimg 29 | import numpy as np 30 | import keras.backend as K 31 | import pathlib 32 | from PIL import Image 33 | import cv2 34 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 35 | import matplotlib.pyplot as plt 36 | from keras.utils.vis_utils import plot_model 37 | import pandas as pd 38 | #%% 39 | 40 | K.clear_session() 41 | BATCH_SIZE = 16 42 | test_nb = 5794 43 | train_nb = 5994 44 | num_classes = 200 45 | img_size= 448 46 | classes = [] 47 | 48 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 49 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 50 | #%% 51 | 52 | 53 | with open("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/classes.txt") as f: 54 | for l in f.readlines(): 55 | data = l.split() 56 | classes.append(data[1]) 57 | 58 | 59 | #%% create data generator 60 | 61 | train_datagen = ImageDataGenerator(rescale = 1.0/255, 62 | zoom_range=[0.85,1], 63 | rotation_range=90, 64 | horizontal_flip=True, 65 | vertical_flip=True) 66 | test_datagen = ImageDataGenerator(rescale = 1.0/255) 67 | 68 | train_generator = train_datagen.flow_from_directory( 69 | train_path, 70 | target_size=(img_size, img_size), 71 | batch_size=BATCH_SIZE, 72 | seed = 13) 73 | 74 | validation_generator = test_datagen.flow_from_directory( 75 | test_path, 76 | target_size=(img_size, img_size), 77 | batch_size=BATCH_SIZE, 78 | seed = 13) 79 | #%% finetuning resnet50 80 | 81 | input_tensor = Input(shape=(img_size, img_size, 3)) 82 | #base_model = VGG19(weights = "imagenet", include_top=False, input_tensor=input_tensor) 83 | base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 84 | 85 | for layer in base_model.layers: 86 | layer.trainable = False 87 | 88 | #%% Implementation of OSME module 89 | 90 | def osme_block(in_block, ch, ratio=16): 91 | z = GlobalAveragePooling2D()(in_block) # 1 92 | x = Dense(ch//ratio, activation='relu')(z) # 2 93 | x = Dense(ch, activation='sigmoid')(x) # 3 94 | return Multiply()([in_block, x]) # 4 95 | 96 | s_1 = osme_block(base_model.output, base_model.output_shape[3]) 97 | s_2 = osme_block(base_model.output, base_model.output_shape[3]) 98 | 99 | fc1 = Dense(1024, name='fc1')(s_1) 100 | fc2 = Dense(1024, name='fc2')(s_2) 101 | 102 | fc1 = Flatten()(fc1) 103 | fc2 = Flatten()(fc2) 104 | 105 | fc = add([fc1,fc2]) # fc1 + fc2 106 | 107 | prediction = Dense(num_classes, activation='softmax', name='prediction')(fc) 108 | 109 | 110 | model = Model(inputs=base_model.input, outputs=prediction) 111 | 112 | opt = SGD(lr=0.001, momentum=0.9, decay=0.0001) 113 | 114 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_osme_vgg_imagenet.best_loss.hdf5") 115 | 116 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 117 | 118 | #%% 119 | plot_model(model, to_file="model.png", show_shapes=True) 120 | 121 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 122 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_osme_resnet_with_resc_no_trainable.best_loss.hdf5', verbose=1, save_best_only=True) 123 | 124 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, 125 | patience=2, min_lr=0.0000001) 126 | 127 | es_cb = EarlyStopping(patience=11) 128 | 129 | #%% fit_generator 130 | 131 | history = model.fit_generator(train_generator, 132 | steps_per_epoch=train_nb/BATCH_SIZE, 133 | epochs=60, 134 | validation_data=validation_generator, 135 | validation_steps=64, 136 | verbose=1, 137 | callbacks=[es_cb, reduce_lr, checkpointer]) 138 | 139 | #%% plot results 140 | import datetime 141 | now = datetime.datetime.now() 142 | 143 | plt.plot(history.history['acc']) 144 | plt.plot(history.history['val_acc']) 145 | plt.title('model_without_MAMC accuracy') 146 | plt.ylabel('accuracy') 147 | plt.xlabel('epoch') 148 | plt.legend(['train', 'test'], loc='upper left') 149 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_osme_{0:%d%m}-{0:%H%M%S}.png".format(now)) 150 | plt.show() 151 | 152 | #loss 153 | plt.plot(history.history['loss']) 154 | plt.plot(history.history['val_loss']) 155 | plt.title('model_without_MAMC loss') 156 | plt.ylabel('loss') 157 | plt.xlabel('epoch') 158 | plt.legend(['train', 'test'], loc='upper left') 159 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_osme_{0:%d%m}-{0:%H%M%S}.png".format(now)) 160 | plt.show() 161 | -------------------------------------------------------------------------------- /model_seinceptv3_with_OSME.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 23 08:45:20 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | import keras 9 | from keras.callbacks import EarlyStopping 10 | from keras.applications.resnet50 import ResNet50 11 | from keras.applications.vgg19 import VGG19 12 | from keras.applications.inception_v3 import InceptionV3 13 | from keras.models import Model 14 | from keras.layers import Input, Dense, Flatten, Multiply, add, concatenate 15 | from keras.layers import GlobalAveragePooling2D, Dropout 16 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 17 | from keras.optimizers import SGD, RMSprop 18 | from keras.utils.np_utils import to_categorical 19 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array, array_to_img 20 | from keras.regularizers import l2 21 | import matplotlib.image as mpimg 22 | import numpy as np 23 | import keras.backend as K 24 | import pathlib 25 | from PIL import Image 26 | import cv2 27 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 28 | import matplotlib.pyplot as plt 29 | from keras.utils.vis_utils import plot_model 30 | import pandas as pd 31 | import grad_cam 32 | from keras.models import load_model 33 | import os 34 | 35 | import se_inception_v3 36 | #%% 37 | import tensorflow as tf 38 | from keras.backend import tensorflow_backend 39 | 40 | K.clear_session() 41 | 42 | config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True)) 43 | session = tf.Session(config=config) 44 | tensorflow_backend.set_session(session) 45 | 46 | BATCH_SIZE = 16 47 | test_nb = 5794 48 | train_nb = 5994 49 | num_classes = 200 50 | img_size= 336 51 | classes = [] 52 | 53 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 54 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 55 | # 56 | # 57 | with open("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/classes.txt") as f: 58 | for l in f.readlines(): 59 | data = l.split() 60 | classes.append(data[1]) 61 | 62 | #%% create data generator 63 | 64 | train_datagen = ImageDataGenerator(rescale = 1.0/255, 65 | # featurewise_center=True, 66 | # featurewise_std_normalization=True, 67 | zoom_range=[0.7,1.0], 68 | rotation_range=30, 69 | # zca_whitening=True, 70 | horizontal_flip=True, 71 | vertical_flip=False, 72 | ) 73 | test_datagen = ImageDataGenerator(rescale = 1.0/255 74 | ) 75 | 76 | 77 | train_generator = train_datagen.flow_from_directory( 78 | train_path, 79 | target_size=(img_size, img_size), 80 | batch_size=BATCH_SIZE, 81 | seed = 13, 82 | #save_to_dir='/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/generated_images4/' 83 | ) 84 | 85 | validation_generator = test_datagen.flow_from_directory( 86 | test_path, 87 | target_size=(img_size, img_size), 88 | batch_size=BATCH_SIZE, 89 | seed = 13) 90 | 91 | #%% finetuning 92 | 93 | input_tensor = Input(shape=(img_size, img_size, 3)) 94 | #base_model = VGG19(weights = "imagenet", include_top=False, input_tensor=input_tensor) 95 | #base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 96 | base_model = se_inception_v3.se_inception_v3(weights = None, include_top=False, input_tensor=input_tensor) 97 | base_model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_osme_inceptv3_beta.best_loss.hdf5", by_name=True) 98 | 99 | #for layer in base_model.layers: 100 | # layer.trainable = False 101 | # 102 | #%% Implementation of OSME module 103 | 104 | def osme_block(in_block, ch, ratio=16, name=None): 105 | z = GlobalAveragePooling2D()(in_block) # 1 106 | x = Dense(ch//ratio, activation='relu')(z) # 2 107 | x = Dense(ch, activation='sigmoid', name=name)(x) # 3 108 | return Multiply()([in_block, x]) # 4 109 | 110 | s_1 = osme_block(base_model.output, base_model.output_shape[3], name='attention1') 111 | s_2 = osme_block(base_model.output, base_model.output_shape[3], name='attention2') 112 | 113 | fc1 = Flatten()(s_1) 114 | fc2 = Flatten()(s_2) 115 | 116 | fc1 = Dense(1024, name='fc1')(fc1) 117 | fc2 = Dense(1024, name='fc2')(fc2) 118 | 119 | #fc = fc1 120 | fc = concatenate([fc1,fc2]) #fc1 + fc2 121 | 122 | #fc = Dropout(0.5)(fc) #add dropout 123 | prediction = Dense(num_classes, activation='softmax', name='prediction')(fc) 124 | 125 | 126 | model = Model(inputs=base_model.input, outputs=prediction) 127 | 128 | opt = SGD(lr=0.001, momentum=0.9, decay=0.0005) 129 | #opt = RMSprop(lr=0.01) 130 | 131 | 132 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_osme_inceptv3_beta.best_loss.hdf5", by_name=True) 133 | 134 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 135 | 136 | #%% 137 | plot_model(model, to_file="model.png", show_shapes=True) 138 | 139 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 140 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_osme_se_inceptv3.best_loss.hdf5', verbose=1, save_best_only=True) 141 | 142 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, 143 | patience=3, min_lr=0.0000001) 144 | 145 | #es_cb = EarlyStopping(patience=11) 146 | 147 | #%% fit_generator 148 | 149 | history = model.fit_generator(train_generator, 150 | steps_per_epoch=train_nb/BATCH_SIZE, 151 | epochs=15, 152 | validation_data=validation_generator, 153 | validation_steps=64, 154 | verbose=1, 155 | callbacks=[reduce_lr, checkpointer]) 156 | 157 | #%% plot results 158 | import datetime 159 | now = datetime.datetime.now() 160 | 161 | plt.plot(history.history['acc']) 162 | plt.plot(history.history['val_acc']) 163 | plt.title('model_without_MAMC accuracy') 164 | plt.ylabel('accuracy') 165 | plt.xlabel('epoch') 166 | plt.legend(['train', 'test'], loc='upper left') 167 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_seinceptv3_with_OSME{0:%d%m}-{0:%H%M%S}.png".format(now)) 168 | plt.show() 169 | 170 | #loss 171 | plt.plot(history.history['loss']) 172 | plt.plot(history.history['val_loss']) 173 | plt.title('model_without_MAMC loss') 174 | plt.ylabel('loss') 175 | plt.xlabel('epoch') 176 | plt.legend(['train', 'test'], loc='upper left') 177 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_seinceptv3_gamma_with_OSME{0:%d%m}-{0:%H%M%S}.png".format(now)) 178 | plt.show() 179 | 180 | 181 | #%%Visualize with grad_cam 182 | # 183 | model = load_model("/home/n-kamiya/models/model_without_MAMC/model_osme_se_inceptv3_gamma.best_loss_.hdf5", custom_objects={"tf":tf}) 184 | #%% 185 | import copy 186 | img_size = 336 187 | for i, file in enumerate(os.listdir("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/013.Bobolink")): 188 | x = img_to_array(load_img(file, target_size=(img_size,img_size))) 189 | _x = copy.copy(x) 190 | _x /= 255 191 | y_proba = model.predict(_x.reshape([-1,img_size,img_size,3])) 192 | print(classes[int(y_proba.argmax(axis = -1))]) 193 | image1 = grad_cam.Grad_Cam(model, x, "multiply_1", img_size) 194 | image2 = grad_cam.Grad_Cam(model, x, "multiply_2", img_size) 195 | 196 | image1 = array_to_img(image1) 197 | image2 = array_to_img(image2) 198 | 199 | image1.save("/home/n-kamiya/images/inceptv3_alpha_with_OSME/%s_%d_attention1.png"%(file, i)) 200 | image2.save("/home/n-kamiya/images/inceptv3_alpha_with_OSME/%s_%d_attention2.png"%(file, i)) 201 | 202 | model.evaluate_generator(generator=validation_generator, steps = test_nb/BATCH_SIZE, verbose = 1) 203 | #%% 204 | -------------------------------------------------------------------------------- /model_with_OSME.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 23 08:45:20 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | import keras 9 | from keras.callbacks import EarlyStopping 10 | from keras.applications.resnet50 import ResNet50 11 | from keras.applications.vgg19 import VGG19 12 | from keras.applications.inception_v3 import InceptionV3 13 | from keras.models import Model 14 | from keras.layers import Input, Dense, Flatten, Multiply, add, concatenate, Lambda 15 | from keras.layers import GlobalAveragePooling2D, Dropout 16 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 17 | from keras.optimizers import SGD, RMSprop 18 | from keras.utils.np_utils import to_categorical 19 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array, array_to_img 20 | from keras.regularizers import l2 21 | import matplotlib.image as mpimg 22 | import numpy as np 23 | import keras.backend as K 24 | import pathlib 25 | from PIL import Image 26 | import cv2 27 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 28 | import matplotlib.pyplot as plt 29 | from keras.utils.vis_utils import plot_model 30 | import pandas as pd 31 | import grad_cam 32 | from keras.models import load_model 33 | import os 34 | #%% 35 | import tensorflow as tf 36 | from keras.backend import tensorflow_backend 37 | 38 | K.clear_session() 39 | 40 | config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True)) 41 | session = tf.Session(config=config) 42 | tensorflow_backend.set_session(session) 43 | 44 | BATCH_SIZE = 16 45 | test_nb = 5794 46 | train_nb = 5994 47 | num_classes = 200 48 | img_size= 336 49 | classes = [] 50 | 51 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 52 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 53 | # 54 | # 55 | with open("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/classes.txt") as f: 56 | for l in f.readlines(): 57 | data = l.split() 58 | classes.append(data[1]) 59 | 60 | #%% add random crop function 61 | #def random_crop(img, random_crop_size): 62 | # # Note: image_data_format is 'channel_last' 63 | # assert img.shape[2] == 3 64 | # height, width = img.shape[0], img.shape[1] 65 | # dy, dx = random_crop_size 66 | # x = np.random.randint(0, width - dx + 1) 67 | # y = np.random.randint(0, height - dy + 1) 68 | # return img[y:(y+dy), x:(x+dx), :] 69 | # 70 | # 71 | #def crop_generator(batches, crop_length , seed): 72 | # """Take as input a Keras ImageGen (Iterator) and generate random 73 | # crops from the image batches generated by the original iterator. 74 | # """ 75 | # while True: 76 | # batch_x, batch_y = next(batches) 77 | # batch_crops = np.zeros((batch_x.shape[0], crop_length, crop_length, 3)) 78 | # for i in range(batch_x.shape[0]): 79 | # batch_crops[i] = random_crop(batch_x[i], (crop_length, crop_length)) 80 | # yield (batch_crops, batch_y) 81 | 82 | #%% create data generator 83 | 84 | train_datagen = ImageDataGenerator(rescale = 1.0/255, 85 | # featurewise_center=True, 86 | # featurewise_std_normalization=True, 87 | zoom_range=[0.7,1.0], 88 | rotation_range=30, 89 | # zca_whitening=True, 90 | horizontal_flip=True, 91 | vertical_flip=False, 92 | ) 93 | test_datagen = ImageDataGenerator(rescale = 1.0/255 94 | ) 95 | 96 | 97 | train_generator = train_datagen.flow_from_directory( 98 | train_path, 99 | target_size=(img_size, img_size), 100 | batch_size=BATCH_SIZE, 101 | seed = 13, 102 | #save_to_dir='/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/generated_images4/' 103 | ) 104 | 105 | validation_generator = test_datagen.flow_from_directory( 106 | test_path, 107 | target_size=(img_size, img_size), 108 | batch_size=BATCH_SIZE, 109 | seed = 13) 110 | 111 | #seed = 13 112 | #crop_size = 352 113 | #train_crops = crop_generator(train_generator, crop_size, seed) 114 | 115 | #%% finetuning resnet50 116 | 117 | input_tensor = Input(shape=(img_size, img_size, 3)) 118 | #base_model = VGG19(weights = "imagenet", include_top=False, input_tensor=input_tensor) 119 | #base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 120 | base_model = InceptionV3(weights = "imagenet", include_top=False, input_tensor=input_tensor) 121 | 122 | #for layer in base_model.layers: 123 | # layer.trainable = False 124 | # 125 | #%% Implementation of OSME module 126 | 127 | split = Lambda( lambda x: tf.split(x,num_or_size_splits=2,axis=3))(base_model.output) 128 | #%% 129 | def osme_block(in_block, ch, ratio=16, name=None): 130 | z = GlobalAveragePooling2D()(in_block) # 1 131 | x = Dense(ch//ratio, activation='relu')(z) # 2 132 | x = Dense(ch, activation='sigmoid', name=name)(x) # 3 133 | return Multiply()([in_block, x]) # 4 134 | 135 | s_1 = osme_block(split[0], split[0].shape[3].value, name='attention1') 136 | s_2 = osme_block(split[1], split[1].shape[3].value, name='attention2') 137 | 138 | fc1 = Flatten()(s_1) 139 | fc2 = Flatten()(s_2) 140 | 141 | fc1 = Dense(1024, name='fc1')(fc1) 142 | fc2 = Dense(1024, name='fc2')(fc2) 143 | 144 | #fc = fc1 145 | fc = concatenate([fc1,fc2]) #fc1 + fc2 146 | 147 | #fc = Dropout(0.5)(fc) #add dropout 148 | prediction = Dense(num_classes, activation='softmax', name='prediction')(fc) 149 | 150 | 151 | model = Model(inputs=base_model.input, outputs=prediction) 152 | 153 | opt = SGD(lr=0.001, momentum=0.9, decay=0.0005) 154 | #opt = RMSprop(lr=0.001) 155 | 156 | 157 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_osme_inceptv3_beta.best_loss.hdf5") 158 | 159 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 160 | 161 | #%% 162 | plot_model(model, to_file="model.png", show_shapes=True) 163 | 164 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 165 | import datetime 166 | now = datetime.datetime.now() 167 | 168 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_osme_inceptv3_gamma.best_loss_.hdf5', verbose=1, save_best_only=True) 169 | 170 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, 171 | patience=5, min_lr=0.0000001) 172 | 173 | #es_cb = EarlyStopping(patience=11) 174 | 175 | #%% fit_generator 176 | 177 | history = model.fit_generator(train_generator, 178 | steps_per_epoch=train_nb/BATCH_SIZE, 179 | epochs=60, 180 | validation_data=validation_generator, 181 | validation_steps=64, 182 | verbose=1, 183 | callbacks=[reduce_lr, checkpointer]) 184 | 185 | #%% plot results 186 | import datetime 187 | now = datetime.datetime.now() 188 | 189 | plt.plot(history.history['acc']) 190 | plt.plot(history.history['val_acc']) 191 | plt.title('model_without_MAMC accuracy') 192 | plt.ylabel('accuracy') 193 | plt.xlabel('epoch') 194 | plt.legend(['train', 'test'], loc='upper left') 195 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_inceptv3_gamma_with_OSME{0:%d%m}-{0:%H%M%S}.png".format(now)) 196 | #plt.show() 197 | 198 | #loss 199 | plt.plot(history.history['loss']) 200 | plt.plot(history.history['val_loss']) 201 | plt.title('model_without_MAMC loss') 202 | plt.ylabel('loss') 203 | plt.xlabel('epoch') 204 | plt.legend(['train', 'test'], loc='upper left') 205 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_inceptv3_gamma_with_OSME{0:%d%m}-{0:%H%M%S}.png".format(now)) 206 | #plt.show() 207 | 208 | 209 | #%% 210 | 211 | model = load_model("/home/n-kamiya/models/model_without_MAMC/model_osme_inceptv3_gamma.best_loss.hdf5", custom_objects={"tf": tf}) 212 | #%% 213 | for file in os.listdir("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/013.Bobolink"): 214 | x = img_to_array(load_img(file, target_size=(336,336))) 215 | 216 | y_proba = model.predict(x.reshape([-1,336,336,3])) 217 | print(classes[int(y_proba.argmax(axis = -1))]) 218 | #%%Model Evaluation 219 | model.evaluate_generator(generator=validation_generator, steps = test_nb/BATCH_SIZE, verbose = 1) 220 | #%% 221 | image1 = grad_cam.Grad_Cam(model, x, "multiply_1", 336) 222 | image2 = grad_cam.Grad_Cam(model, x, "multiply_2", 336) 223 | 224 | image1 = array_to_img(image1) 225 | image2 = array_to_img(image2) 226 | 227 | image1.show() 228 | image2.show() 229 | 230 | -------------------------------------------------------------------------------- /model_with_OSME_SE.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 23 08:45:20 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | import keras 9 | from keras.callbacks import EarlyStopping 10 | from keras.applications.resnet50 import ResNet50 11 | from keras.applications.vgg19 import VGG19 12 | from keras.applications.inception_v3 import InceptionV3 13 | from keras.models import Model 14 | from keras.layers import Input, Dense, Flatten, Multiply, add, concatenate, Lambda 15 | from keras.layers import GlobalAveragePooling2D, Dropout 16 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 17 | from keras.optimizers import SGD, RMSprop 18 | from keras.utils.np_utils import to_categorical 19 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array, array_to_img 20 | from keras.regularizers import l2 21 | import matplotlib.image as mpimg 22 | import numpy as np 23 | import keras.backend as K 24 | import pathlib 25 | from PIL import Image 26 | import cv2 27 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 28 | import matplotlib.pyplot as plt 29 | from keras.utils.vis_utils import plot_model 30 | import pandas as pd 31 | #import grad_cam 32 | from keras.models import load_model 33 | import os 34 | 35 | from se_inception_v3 import se_inception_v3 36 | #%% 37 | import tensorflow as tf 38 | from keras.backend import tensorflow_backend 39 | 40 | K.clear_session() 41 | 42 | config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True)) 43 | session = tf.Session(config=config) 44 | tensorflow_backend.set_session(session) 45 | 46 | BATCH_SIZE = 16 47 | test_nb = 5794 48 | train_nb = 5994 49 | num_classes = 200 50 | img_size= 336 51 | classes = [] 52 | 53 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 54 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 55 | # 56 | # 57 | with open("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/classes.txt") as f: 58 | for l in f.readlines(): 59 | data = l.split() 60 | classes.append(data[1]) 61 | 62 | #%% add random crop function 63 | #def random_crop(img, random_crop_size): 64 | # # Note: image_data_format is 'channel_last' 65 | # assert img.shape[2] == 3 66 | # height, width = img.shape[0], img.shape[1] 67 | # dy, dx = random_crop_size 68 | # x = np.random.randint(0, width - dx + 1) 69 | # y = np.random.randint(0, height - dy + 1) 70 | # return img[y:(y+dy), x:(x+dx), :] 71 | # 72 | # 73 | #def crop_generator(batches, crop_length , seed): 74 | # """Take as input a Keras ImageGen (Iterator) and generate random 75 | # crops from the image batches generated by the original iterator. 76 | # """ 77 | # while True: 78 | # batch_x, batch_y = next(batches) 79 | # batch_crops = np.zeros((batch_x.shape[0], crop_length, crop_length, 3)) 80 | # for i in range(batch_x.shape[0]): 81 | # batch_crops[i] = random_crop(batch_x[i], (crop_length, crop_length)) 82 | # yield (batch_crops, batch_y) 83 | 84 | #%% create data generator 85 | 86 | train_datagen = ImageDataGenerator(rescale = 1.0/255, 87 | # featurewise_center=True, 88 | # featurewise_std_normalization=True, 89 | zoom_range=[0.8,1.0], 90 | rotation_range=30, 91 | # zca_whitening=True, 92 | horizontal_flip=True, 93 | vertical_flip=False, 94 | ) 95 | test_datagen = ImageDataGenerator(rescale = 1.0/255 96 | ) 97 | 98 | 99 | train_generator = train_datagen.flow_from_directory( 100 | train_path, 101 | target_size=(img_size, img_size), 102 | batch_size=BATCH_SIZE, 103 | seed = 13, 104 | #save_to_dir='/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/generated_images4/' 105 | ) 106 | 107 | validation_generator = test_datagen.flow_from_directory( 108 | test_path, 109 | target_size=(img_size, img_size), 110 | batch_size=BATCH_SIZE, 111 | seed = 13) 112 | 113 | #seed = 13 114 | #crop_size = 352 115 | #train_crops = crop_generator(train_generator, crop_size, seed) 116 | 117 | #%% finetuning resnet50 118 | 119 | input_tensor = Input(shape=(img_size, img_size, 3)) 120 | #base_model = VGG19(weights = "imagenet", include_top=False, input_tensor=input_tensor) 121 | #base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 122 | #base_model = InceptionV3(weights = "imagenet", include_top=False, input_tensor=input_tensor) 123 | base_model = se_inception_v3(include_top=False, input_tensor=input_tensor) 124 | base_model.load_weights("/home/n-kamiya/.keras/models/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5", by_name=True) 125 | #for layer in base_model.layers: 126 | # layer.trainable = False 127 | # 128 | #%% Implementation of OSME module 129 | 130 | def osme_block(in_block, ch, ratio=16, name=None): 131 | z = GlobalAveragePooling2D()(in_block) # 1 132 | x = Dense(ch//ratio, activation='relu')(z) # 2 133 | x = Dense(ch, activation='sigmoid', name=name)(x) # 3 134 | return Multiply()([in_block, x]) # 4 135 | 136 | s_1 = osme_block(base_model.output, base_model.output_shape[3], name='attention1') 137 | s_2 = osme_block(base_model.output, base_model.output_shape[3], name='attention2') 138 | 139 | fc1 = Flatten()(s_1) 140 | fc2 = Flatten()(s_2) 141 | 142 | fc1 = Dense(1024, name='fc1')(fc1) 143 | fc2 = Dense(1024, name='fc2')(fc2) 144 | 145 | #fc = fc1 146 | fc = concatenate([fc1,fc2]) #fc1 + fc2 147 | 148 | #fc = Dropout(0.5)(fc) #add dropout 149 | prediction = Dense(num_classes, activation='softmax', name='prediction')(fc) 150 | 151 | 152 | model = Model(inputs=base_model.input, outputs=prediction) 153 | 154 | opt = SGD(lr=0.001, momentum=0.9, decay=0.0005) 155 | #opt = RMSprop(lr=0.001) 156 | 157 | 158 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_osme_inceptv3_beta.best_loss.hdf5") 159 | 160 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 161 | 162 | #%% 163 | plot_model(model, to_file="model.png", show_shapes=True) 164 | 165 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 166 | import datetime 167 | now = datetime.datetime.now() 168 | 169 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_osme_se_inceptv3_drop.best_loss_.hdf5', verbose=1, save_best_only=True) 170 | 171 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, 172 | patience=5, min_lr=0.0000001) 173 | 174 | #es_cb = EarlyStopping(patience=11) 175 | 176 | #%% fit_generator 177 | 178 | history = model.fit_generator(train_generator, 179 | steps_per_epoch=train_nb/BATCH_SIZE, 180 | epochs=60, 181 | validation_data=validation_generator, 182 | validation_steps=64, 183 | verbose=1, 184 | callbacks=[reduce_lr, checkpointer]) 185 | 186 | #%% plot results 187 | import datetime 188 | now = datetime.datetime.now() 189 | 190 | plt.plot(history.history['acc']) 191 | plt.plot(history.history['val_acc']) 192 | plt.title('model_without_MAMC accuracy') 193 | plt.ylabel('accuracy') 194 | plt.xlabel('epoch') 195 | plt.legend(['train', 'test'], loc='upper left') 196 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_se_inceptv3_with_OSME_drop{0:%d%m}-{0:%H%M%S}.png".format(now)) 197 | 198 | #loss 199 | plt.plot(history.history['loss']) 200 | plt.plot(history.history['val_loss']) 201 | plt.title('model_without_MAMC loss') 202 | plt.ylabel('loss') 203 | plt.xlabel('epoch') 204 | plt.legend(['train', 'test'], loc='upper left') 205 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_se_inceptv3_with_OSME_drop{0:%d%m}-{0:%H%M%S}.png".format(now)) 206 | 207 | 208 | 209 | #%%model_evaluation 210 | 211 | model_dogs = load_model("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_OSME_SE_miru_dogs_mod.best_loss.hdf5") 212 | #%% 213 | for file in os.listdir("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/013.Bobolink/"): 214 | x = img_to_array(load_img(file, target_size=(448,448))) 215 | 216 | y_proba = model_dogs.predict(x.reshape([-1,448,448,3])) 217 | print(classes[int(y_proba.argmax(axis = -1))]) 218 | #%%Model Evaluation 219 | model_dogs.evaluate_generator(generator=validation_generator, steps = 10, verbose = 1) 220 | #%% 221 | 222 | image1 = grad_cam.Grad_Cam(model_dogs, x, "multiply_1",336) 223 | image2 = grad_cam.Grad_Cam(model_dogs, x, "multiply_2",336) 224 | 225 | image1 = array_to_img(image1) 226 | image2 = array_to_img(image2) 227 | 228 | image1.show() 229 | image1.save("/home/n-kamiya/images/inceptv3_se_osme_miru_multiply_1.png") 230 | image2.show() 231 | image2.save("/home/n-kamiya/images/inceptv3_se_osme_miru_multiply_2.png") 232 | -------------------------------------------------------------------------------- /model_with_OSME_SE_alpha.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 23 08:45:20 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | import keras 9 | from keras.callbacks import EarlyStopping 10 | from keras.applications.resnet50 import ResNet50 11 | from keras.applications.vgg19 import VGG19 12 | from keras.applications.inception_v3 import InceptionV3 13 | from keras.models import Model 14 | from keras.layers import Input, Dense, Flatten, Multiply, add, concatenate, Lambda 15 | from keras.layers import GlobalAveragePooling2D, Dropout 16 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 17 | from keras.optimizers import SGD, RMSprop 18 | from keras.utils.np_utils import to_categorical 19 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array, array_to_img 20 | from keras.regularizers import l2 21 | import matplotlib.image as mpimg 22 | import numpy as np 23 | import keras.backend as K 24 | import pathlib 25 | from PIL import Image 26 | import cv2 27 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 28 | import matplotlib.pyplot as plt 29 | from keras.utils.vis_utils import plot_model 30 | import pandas as pd 31 | import grad_cam 32 | from keras.models import load_model 33 | import os 34 | 35 | from se_inception_v3 import se_inception_v3, preprocess_input 36 | #%% 37 | import tensorflow as tf 38 | from keras.backend import tensorflow_backend 39 | 40 | K.clear_session() 41 | 42 | config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True)) 43 | session = tf.Session(config=config) 44 | tensorflow_backend.set_session(session) 45 | 46 | BATCH_SIZE = 16 47 | test_nb = 5794 48 | train_nb = 5994 49 | num_classes = 200 50 | img_size= 336 51 | classes = [] 52 | 53 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 54 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 55 | # 56 | # 57 | with open("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/classes.txt") as f: 58 | for l in f.readlines(): 59 | data = l.split() 60 | classes.append(data[1]) 61 | 62 | #%% add random crop function 63 | #def random_crop(img, random_crop_size): 64 | # # Note: image_data_format is 'channel_last' 65 | # assert img.shape[2] == 3 66 | # height, width = img.shape[0], img.shape[1] 67 | # dy, dx = random_crop_size 68 | # x = np.random.randint(0, width - dx + 1) 69 | # y = np.random.randint(0, height - dy + 1) 70 | # return img[y:(y+dy), x:(x+dx), :] 71 | # 72 | # 73 | #def crop_generator(batches, crop_length , seed): 74 | # """Take as input a Keras ImageGen (Iterator) and generate random 75 | # crops from the image batches generated by the original iterator. 76 | # """ 77 | # while True: 78 | # batch_x, batch_y = next(batches) 79 | # batch_crops = np.zeros((batch_x.shape[0], crop_length, crop_length, 3)) 80 | # for i in range(batch_x.shape[0]): 81 | # batch_crops[i] = random_crop(batch_x[i], (crop_length, crop_length)) 82 | # yield (batch_crops, batch_y) 83 | 84 | #%% create data generator 85 | 86 | train_datagen = ImageDataGenerator(#rescale = 1.0/255, 87 | # featurewise_center=True, 88 | # featurewise_std_normalization=True, 89 | zoom_range=[0.7,1.0], 90 | rotation_range=30, 91 | # zca_whitening=True, 92 | horizontal_flip=True, 93 | vertical_flip=False, 94 | preprocessing_function=preprocess_input 95 | ) 96 | test_datagen = ImageDataGenerator(#rescale = 1.0/255 97 | preprocessing_function=preprocess_input 98 | 99 | ) 100 | 101 | 102 | train_generator = train_datagen.flow_from_directory( 103 | train_path, 104 | target_size=(img_size, img_size), 105 | batch_size=BATCH_SIZE, 106 | seed = 13, 107 | #save_to_dir='/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/generated_images4/' 108 | ) 109 | 110 | validation_generator = test_datagen.flow_from_directory( 111 | test_path, 112 | target_size=(img_size, img_size), 113 | batch_size=BATCH_SIZE, 114 | seed = 13) 115 | 116 | #seed = 13 117 | #crop_size = 352 118 | #train_crops = crop_generator(train_generator, crop_size, seed) 119 | 120 | #%% finetuning resnet50 121 | 122 | input_tensor = Input(shape=(img_size, img_size, 3)) 123 | #base_model = VGG19(weights = "imagenet", include_top=False, input_tensor=input_tensor) 124 | #base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 125 | #base_model = InceptionV3(weights = "imagenet", include_top=False, input_tensor=input_tensor) 126 | base_model = se_inception_v3(include_top=False, input_tensor=input_tensor) 127 | base_model.load_weights("/home/n-kamiya/.keras/models/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5", by_name=True) 128 | #for layer in base_model.layers: 129 | # layer.trainable = False 130 | # 131 | #%% Implementation of OSME module 132 | 133 | split = Lambda( lambda x: tf.split(x,num_or_size_splits=2,axis=3))(base_model.output) 134 | #%% 135 | def osme_block(in_block, ch, ratio=16, name=None): 136 | z = GlobalAveragePooling2D()(in_block) # 1 137 | x = Dense(ch//ratio, activation='relu')(z) # 2 138 | x = Dense(ch, activation='sigmoid', name=name)(x) # 3 139 | return Multiply()([in_block, x]) # 4 140 | 141 | s_1 = osme_block(split[0], split[0].shape[3].value, name='attention1') 142 | s_2 = osme_block(split[1], split[1].shape[3].value, name='attention2') 143 | 144 | fc1 = Flatten()(s_1) 145 | fc2 = Flatten()(s_2) 146 | 147 | fc1 = Dense(1024, name='fc1')(fc1) 148 | fc2 = Dense(1024, name='fc2')(fc2) 149 | 150 | #fc = fc1 151 | fc = concatenate([fc1,fc2]) #fc1 + fc2 152 | 153 | #fc = Dropout(0.5)(fc) #add dropout 154 | prediction = Dense(num_classes, activation='softmax', name='prediction')(fc) 155 | 156 | 157 | model = Model(inputs=base_model.input, outputs=prediction) 158 | 159 | opt = SGD(lr=0.001, momentum=0.9, decay=0.0005) 160 | #opt = RMSprop(lr=0.001) 161 | 162 | 163 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_osme_inceptv3_beta.best_loss.hdf5") 164 | 165 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 166 | 167 | #%% 168 | plot_model(model, to_file="model.png", show_shapes=True) 169 | 170 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 171 | import datetime 172 | now = datetime.datetime.now() 173 | 174 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_osme_se_inceptv3_alpha.best_loss_.hdf5', verbose=1, save_best_only=True) 175 | 176 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, 177 | patience=5, min_lr=0.0000001) 178 | 179 | #es_cb = EarlyStopping(patience=11) 180 | 181 | #%% fit_generator 182 | 183 | history = model.fit_generator(train_generator, 184 | steps_per_epoch=train_nb/BATCH_SIZE, 185 | epochs=60, 186 | validation_data=validation_generator, 187 | validation_steps=64, 188 | verbose=1, 189 | callbacks=[reduce_lr, checkpointer]) 190 | 191 | #%% plot results 192 | import datetime 193 | now = datetime.datetime.now() 194 | 195 | plt.plot(history.history['acc']) 196 | plt.plot(history.history['val_acc']) 197 | plt.title('model_without_MAMC accuracy') 198 | plt.ylabel('accuracy') 199 | plt.xlabel('epoch') 200 | plt.legend(['train', 'test'], loc='upper left') 201 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_se_inceptv3_alpha_with_OSME{0:%d%m}-{0:%H%M%S}.png".format(now)) 202 | #plt.show() 203 | 204 | #loss 205 | plt.plot(history.history['loss']) 206 | plt.plot(history.history['val_loss']) 207 | plt.title('model_without_MAMC loss') 208 | plt.ylabel('loss') 209 | plt.xlabel('epoch') 210 | plt.legend(['train', 'test'], loc='upper left') 211 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_se_inceptv3_alpha_with_OSME{0:%d%m}-{0:%H%M%S}.png".format(now)) 212 | #plt.show() 213 | 214 | 215 | #%% 216 | 217 | #model = load_model("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME_SE.best_loss.hdf5", custom_objects={"tf": tf}) 218 | ##%% 219 | #for file in os.listdir("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/013.Bobolink/"): 220 | # x = img_to_array(load_img(file, target_size=(448,448))) 221 | # 222 | # y_proba = model.predict(x.reshape([-1,448,448,3])) 223 | # print(classes[int(y_proba.argmax(axis = -1))]) 224 | ##%%Model Evaluation 225 | #model._generator(generator=validation_generator, steps = test_nb/BATCH_SIZE, verbose = 1) 226 | ##%% 227 | #image1 = grad_cam.Grad_Cam(model, x, "multiply_1") 228 | #image2 = grad_cam.Grad_Cam(model, x, "multiply_2") 229 | # 230 | #image1 = array_to_img(image1) 231 | #image2 = array_to_img(image2) 232 | # 233 | #image1.show() 234 | #image1.save("/home/n-kamiya/images/inceptv3_beta_multiply_1.png") 235 | #image2.show() 236 | #image2.save("/home/n-kamiya/images/inceptv3_beta_multiply_2.png") 237 | -------------------------------------------------------------------------------- /model_with_OSME_SE_alpha_p.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 23 08:45:20 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | import keras 9 | from keras.callbacks import EarlyStopping 10 | from keras.applications.resnet50 import ResNet50 11 | from keras.applications.vgg19 import VGG19 12 | from keras.applications.inception_v3 import InceptionV3 13 | from keras.models import Model 14 | from keras.layers import Input, Dense, Flatten, Multiply, add, concatenate, Lambda 15 | from keras.layers import GlobalAveragePooling2D, Dropout 16 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 17 | from keras.optimizers import SGD, RMSprop 18 | from keras.utils.np_utils import to_categorical 19 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array, array_to_img 20 | from keras.regularizers import l2 21 | import matplotlib.image as mpimg 22 | import numpy as np 23 | import keras.backend as K 24 | import pathlib 25 | from PIL import Image 26 | import cv2 27 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 28 | import matplotlib.pyplot as plt 29 | from keras.utils.vis_utils import plot_model 30 | import pandas as pd 31 | import grad_cam 32 | from keras.models import load_model 33 | import os 34 | 35 | from se_inception_v3 import se_inception_v3, preprocess_input 36 | #%% 37 | import tensorflow as tf 38 | from keras.backend import tensorflow_backend 39 | 40 | K.clear_session() 41 | 42 | config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True)) 43 | session = tf.Session(config=config) 44 | tensorflow_backend.set_session(session) 45 | 46 | BATCH_SIZE = 16 47 | test_nb = 5794 48 | train_nb = 5994 49 | num_classes = 200 50 | img_size= 336 51 | classes = [] 52 | 53 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 54 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 55 | # 56 | # 57 | with open("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/classes.txt") as f: 58 | for l in f.readlines(): 59 | data = l.split() 60 | classes.append(data[1]) 61 | 62 | #%% add random crop function 63 | #def random_crop(img, random_crop_size): 64 | # # Note: image_data_format is 'channel_last' 65 | # assert img.shape[2] == 3 66 | # height, width = img.shape[0], img.shape[1] 67 | # dy, dx = random_crop_size 68 | # x = np.random.randint(0, width - dx + 1) 69 | # y = np.random.randint(0, height - dy + 1) 70 | # return img[y:(y+dy), x:(x+dx), :] 71 | # 72 | # 73 | #def crop_generator(batches, crop_length , seed): 74 | # """Take as input a Keras ImageGen (Iterator) and generate random 75 | # crops from the image batches generated by the original iterator. 76 | # """ 77 | # while True: 78 | # batch_x, batch_y = next(batches) 79 | # batch_crops = np.zeros((batch_x.shape[0], crop_length, crop_length, 3)) 80 | # for i in range(batch_x.shape[0]): 81 | # batch_crops[i] = random_crop(batch_x[i], (crop_length, crop_length)) 82 | # yield (batch_crops, batch_y) 83 | 84 | #%% create data generator 85 | 86 | train_datagen = ImageDataGenerator(#rescale = 1.0/255, 87 | # featurewise_center=True, 88 | # featurewise_std_normalization=True, 89 | zoom_range=[0.7,1.0], 90 | rotation_range=30, 91 | # zca_whitening=True, 92 | horizontal_flip=True, 93 | vertical_flip=False, 94 | preprocessing_function=preprocess_input 95 | ) 96 | test_datagen = ImageDataGenerator(#rescale = 1.0/255 97 | preprocessing_function=preprocess_input 98 | 99 | ) 100 | 101 | 102 | train_generator = train_datagen.flow_from_directory( 103 | train_path, 104 | target_size=(img_size, img_size), 105 | batch_size=BATCH_SIZE, 106 | seed = 13, 107 | #save_to_dir='/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/generated_images4/' 108 | ) 109 | 110 | validation_generator = test_datagen.flow_from_directory( 111 | test_path, 112 | target_size=(img_size, img_size), 113 | batch_size=BATCH_SIZE, 114 | seed = 13) 115 | 116 | #seed = 13 117 | #crop_size = 352 118 | #train_crops = crop_generator(train_generator, crop_size, seed) 119 | 120 | #%% finetuning resnet50 121 | 122 | input_tensor = Input(shape=(img_size, img_size, 3)) 123 | #base_model = VGG19(weights = "imagenet", include_top=False, input_tensor=input_tensor) 124 | #base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 125 | #base_model = InceptionV3(weights = "imagenet", include_top=False, input_tensor=input_tensor) 126 | base_model = se_inception_v3(include_top=False, input_tensor=input_tensor) 127 | base_model.load_weights("/home/n-kamiya/.keras/models/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5", by_name=True) 128 | #for layer in base_model.layers: 129 | # layer.trainable = False 130 | # 131 | #%% Implementation of OSME module 132 | 133 | split = Lambda( lambda x: tf.split(x,num_or_size_splits=2,axis=3))(base_model.output) 134 | #%% 135 | def osme_block(in_block, ch, ratio=16, name=None): 136 | z = GlobalAveragePooling2D()(in_block) # 1 137 | x = Dense(ch//ratio, activation='relu')(z) # 2 138 | x = Dense(ch, activation='sigmoid', name=name)(x) # 3 139 | return Multiply()([in_block, x]) # 4 140 | 141 | s_1 = osme_block(split[0], split[0].shape[3].value, name='attention1') 142 | s_2 = osme_block(split[1], split[1].shape[3].value, name='attention2') 143 | 144 | fc1 = Flatten()(s_1) 145 | fc2 = Flatten()(s_2) 146 | 147 | fc1 = Dense(1024, name='fc1')(fc1) 148 | fc2 = Dense(1024, name='fc2')(fc2) 149 | 150 | #fc = fc1 151 | fc = concatenate([fc1,fc2]) #fc1 + fc2 152 | 153 | #fc = Dropout(0.5)(fc) #add dropout 154 | prediction = Dense(num_classes, activation='softmax', name='prediction')(fc) 155 | 156 | 157 | model = Model(inputs=base_model.input, outputs=prediction) 158 | 159 | opt = SGD(lr=0.001, momentum=0.9, decay=0.0005) 160 | #opt = RMSprop(lr=0.001) 161 | 162 | 163 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_osme_inceptv3_beta.best_loss.hdf5") 164 | 165 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 166 | 167 | #%% 168 | plot_model(model, to_file="model.png", show_shapes=True) 169 | 170 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 171 | import datetime 172 | now = datetime.datetime.now() 173 | 174 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_osme_se_inceptv3_alpha_p.best_loss_.hdf5', verbose=1, save_best_only=True) 175 | 176 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, 177 | patience=5, min_lr=0.0000001) 178 | 179 | #es_cb = EarlyStopping(patience=11) 180 | 181 | #%% fit_generator 182 | 183 | history = model.fit_generator(train_generator, 184 | steps_per_epoch=train_nb/BATCH_SIZE, 185 | epochs=60, 186 | validation_data=validation_generator, 187 | validation_steps=64, 188 | verbose=1, 189 | callbacks=[reduce_lr, checkpointer]) 190 | 191 | #%% plot results 192 | import datetime 193 | now = datetime.datetime.now() 194 | 195 | plt.plot(history.history['acc']) 196 | plt.plot(history.history['val_acc']) 197 | plt.title('model_without_MAMC accuracy') 198 | plt.ylabel('accuracy') 199 | plt.xlabel('epoch') 200 | plt.legend(['train', 'test'], loc='upper left') 201 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_se_inceptv3_alpha_with_OSME_p{0:%d%m}-{0:%H%M%S}.png".format(now)) 202 | #plt.show() 203 | 204 | #loss 205 | plt.plot(history.history['loss']) 206 | plt.plot(history.history['val_loss']) 207 | plt.title('model_without_MAMC loss') 208 | plt.ylabel('loss') 209 | plt.xlabel('epoch') 210 | plt.legend(['train', 'test'], loc='upper left') 211 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_se_inceptv3_alpha_with_OSME_p{0:%d%m}-{0:%H%M%S}.png".format(now)) 212 | #plt.show() 213 | 214 | 215 | #%% 216 | 217 | #model = load_model("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME_SE.best_loss.hdf5", custom_objects={"tf": tf}) 218 | ##%% 219 | #for file in os.listdir("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/013.Bobolink/"): 220 | # x = img_to_array(load_img(file, target_size=(448,448))) 221 | # 222 | # y_proba = model.predict(x.reshape([-1,448,448,3])) 223 | # print(classes[int(y_proba.argmax(axis = -1))]) 224 | ##%%Model Evaluation 225 | #model._generator(generator=validation_generator, steps = test_nb/BATCH_SIZE, verbose = 1) 226 | ##%% 227 | #image1 = grad_cam.Grad_Cam(model, x, "multiply_1") 228 | #image2 = grad_cam.Grad_Cam(model, x, "multiply_2") 229 | # 230 | #image1 = array_to_img(image1) 231 | #image2 = array_to_img(image2) 232 | # 233 | #image1.show() 234 | #image1.save("/home/n-kamiya/images/inceptv3_beta_multiply_1.png") 235 | #image2.show() 236 | #image2.save("/home/n-kamiya/images/inceptv3_beta_multiply_2.png") 237 | -------------------------------------------------------------------------------- /model_with_OSME_crop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 23 08:45:20 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | import keras 9 | from keras.callbacks import EarlyStopping 10 | from keras.applications.resnet50 import ResNet50 11 | from keras.applications.vgg19 import VGG19 12 | from keras.applications.inception_v3 import InceptionV3 13 | from keras.models import Model 14 | from keras.layers import Input, Dense, Flatten, Multiply, add 15 | from keras.layers import GlobalAveragePooling2D, Dropout 16 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 17 | from keras.optimizers import SGD, RMSprop 18 | from keras.utils.np_utils import to_categorical 19 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array 20 | from keras.regularizers import l2 21 | import matplotlib.image as mpimg 22 | import numpy as np 23 | import keras.backend as K 24 | import pathlib 25 | from PIL import Image 26 | import cv2 27 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 28 | import matplotlib.pyplot as plt 29 | from keras.utils.vis_utils import plot_model 30 | import pandas as pd 31 | #%% 32 | 33 | K.clear_session() 34 | BATCH_SIZE = 16 35 | test_nb = 5794 36 | train_nb = 5994 37 | num_classes = 200 38 | img_size= 448 39 | classes = [] 40 | 41 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 42 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 43 | # 44 | # 45 | #with open("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/classes.txt") as f: 46 | # for l in f.readlines(): 47 | # data = l.split() 48 | # classes.append(data[1]) 49 | 50 | #%% add random crop function 51 | def random_crop(img, random_crop_size,seed): 52 | # Note: image_data_format is 'channel_last' 53 | np.random.seed(seed=seed) 54 | assert img.shape[2] == 3 55 | height, width = img.shape[0], img.shape[1] 56 | dy, dx = random_crop_size 57 | x = np.random.randint(0, width - dx + 1) 58 | y = np.random.randint(0, height - dy + 1) 59 | return img[y:(y+dy), x:(x+dx), :] 60 | 61 | 62 | def crop_generator(batches, crop_length , seed): 63 | """Take as input a Keras ImageGen (Iterator) and generate random 64 | crops from the image batches generated by the original iterator. 65 | """ 66 | while True: 67 | batch_x, batch_y = next(batches) 68 | batch_crops = np.zeros((batch_x.shape[0], crop_length, crop_length, 3)) 69 | for i in range(batch_x.shape[0]): 70 | batch_crops[i] = random_crop(batch_x[i], (crop_length, crop_length),seed) 71 | yield (batch_crops, batch_y) 72 | 73 | #%% create data generator 74 | 75 | train_datagen = ImageDataGenerator(rescale = 1.0/255, 76 | zoom_range=[0.9,1.0], 77 | rotation_range=, 78 | horizontal_flip=True, 79 | vertical_flip=True) 80 | test_datagen = ImageDataGenerator(rescale = 1.0/255) 81 | 82 | train_generator = train_datagen.flow_from_directory( 83 | train_path, 84 | target_size=(img_size, img_size), 85 | batch_size=BATCH_SIZE, 86 | seed = 13) 87 | 88 | validation_generator = test_datagen.flow_from_directory( 89 | test_path, 90 | target_size=(352, 352), 91 | batch_size=BATCH_SIZE, 92 | seed = 13) 93 | 94 | seed = 13 95 | crop_size = 352 96 | train_crops = crop_generator(train_generator, crop_size, seed) 97 | 98 | #%% finetuning resnet50 99 | 100 | input_tensor = Input(shape=(crop_size, crop_size, 3)) 101 | #base_model = VGG19(weights = "imagenet", include_top=False, input_tensor=input_tensor) 102 | #base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 103 | base_model = InceptionV3(weights = "imagenet", include_top=False, input_tensor=input_tensor) 104 | 105 | for layer in base_model.layers: 106 | layer.trainable = False 107 | # if isinstance(layer, keras.layers.normalization.BatchNormalization): 108 | # layer._per_input_updates = {} 109 | 110 | #%% Implementation of OSME module 111 | 112 | def osme_block(in_block, ch, ratio=16): 113 | z = GlobalAveragePooling2D()(in_block) # 1 114 | x = Dense(ch//ratio, activation='relu')(z) # 2 115 | x = Dense(ch, activation='sigmoid')(x) # 3 116 | return Multiply()([in_block, x]) # 4 117 | 118 | s_1 = osme_block(base_model.output, base_model.output_shape[3]) 119 | s_2 = osme_block(base_model.output, base_model.output_shape[3]) 120 | 121 | fc1 = Dense(1024, name='fc1')(s_1) 122 | fc2 = Dense(1024, name='fc2')(s_2) 123 | 124 | fc1 = Flatten()(fc1) 125 | fc2 = Flatten()(fc2) 126 | 127 | fc = add([fc1,fc2]) # fc1 + fc2 128 | 129 | #fc = Dropout(0.5)(fc) #add dropout 130 | prediction = Dense(num_classes, activation='softmax', name='prediction')(fc) 131 | 132 | 133 | model = Model(inputs=base_model.input, outputs=prediction) 134 | 135 | opt = SGD(lr=0.001, momentum=0.9, decay=0.0001) 136 | #opt = RMSprop(lr=0.001) 137 | 138 | 139 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_osme_vgg_imagenet.best_loss.hdf5") 140 | 141 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 142 | 143 | #%% 144 | plot_model(model, to_file="model.png", show_shapes=True) 145 | 146 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 147 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_osme_inceptv3_cropped_alpha.best_loss.hdf5', verbose=1, save_best_only=True) 148 | 149 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, 150 | patience=5, min_lr=0.0000001) 151 | 152 | #es_cb = EarlyStopping(patience=11) 153 | 154 | #%% fit_generator 155 | 156 | history = model.fit_generator(train_crops, 157 | steps_per_epoch=train_nb/BATCH_SIZE, 158 | epochs=60, 159 | validation_data=validation_generator, 160 | validation_steps=64, 161 | verbose=1, 162 | callbacks=[reduce_lr, checkpointer]) 163 | 164 | #%% plot results 165 | import datetime 166 | now = datetime.datetime.now() 167 | 168 | plt.plot(history.history['acc']) 169 | plt.plot(history.history['val_acc']) 170 | plt.title('model_without_MAMC accuracy') 171 | plt.ylabel('accuracy') 172 | plt.xlabel('epoch') 173 | plt.legend(['train', 'test'], loc='upper left') 174 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_inceptv3_with_OSME_cropped_alpha{0:%d%m}-{0:%H%M%S}.png".format(now)) 175 | plt.show() 176 | 177 | #loss 178 | plt.plot(history.history['loss']) 179 | plt.plot(history.history['val_loss']) 180 | plt.title('model_without_MAMC loss') 181 | plt.ylabel('loss') 182 | plt.xlabel('epoch') 183 | plt.legend(['train', 'test'], loc='upper left') 184 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_inceptv3_with_OSME_cropped_alpha{0:%d%m}-{0:%H%M%S}.png".format(now)) 185 | plt.show() 186 | -------------------------------------------------------------------------------- /model_with_OSME_se.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 23 08:45:20 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | from keras.callbacks import EarlyStopping 10 | from keras.applications.resnet50 import ResNet50 11 | from keras.applications.inception_v3 import InceptionV3 12 | from keras.models import Sequential, Model, load_model 13 | from keras.layers import Input, Dense, Dropout, Activation, Flatten, Multiply, Lambda, concatenate 14 | from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, AveragePooling2D 15 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 16 | from keras.optimizers import SGD 17 | from keras.utils.np_utils import to_categorical 18 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array 19 | from keras.regularizers import l2 20 | import matplotlib.image as mpimg 21 | import numpy as np 22 | import keras.backend as K 23 | import pathlib 24 | from PIL import Image 25 | import cv2 26 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 27 | import matplotlib.pyplot as plt 28 | from keras.utils.vis_utils import plot_model 29 | import pandas as pd 30 | import se_inception_v3 31 | import tensorflow as tf 32 | #%% 33 | 34 | K.clear_session() 35 | BATCH_SIZE = 16 36 | test_nb = 5794 37 | train_nb = 5994 38 | num_classes = 200 39 | img_size= 336 40 | classes = [] 41 | 42 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 43 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 44 | #%% 45 | 46 | 47 | with open("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/classes.txt") as f: 48 | for l in f.readlines(): 49 | data = l.split() 50 | classes.append(data[1]) 51 | 52 | 53 | #%% create data generator 54 | 55 | train_datagen = ImageDataGenerator(rescale = 1./255, 56 | zoom_range=[0.8,1], 57 | rotation_range=30, 58 | horizontal_flip=True) 59 | test_datagen = ImageDataGenerator(rescale = 1./255) 60 | 61 | train_generator = train_datagen.flow_from_directory( 62 | train_path, 63 | target_size=(img_size, img_size), 64 | batch_size=BATCH_SIZE, 65 | seed = 13, 66 | ) 67 | 68 | validation_generator = test_datagen.flow_from_directory( 69 | test_path, 70 | target_size=(img_size, img_size), 71 | batch_size=BATCH_SIZE, 72 | seed = 13, 73 | ) 74 | #%% finetuning resnet50 75 | 76 | input_tensor = Input(shape=(img_size, img_size, 3)) 77 | #base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 78 | base_model = se_inception_v3.se_inception_v3(include_top=False, input_tensor=input_tensor) 79 | base_model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME.best_loss.hdf5",by_name=True) 80 | 81 | 82 | #for layer in base_model.layers: 83 | # layer.trainable = False 84 | #%% 85 | split = Lambda( lambda x: tf.split(x,num_or_size_splits=2,axis=3))(base_model.output) 86 | #%% 87 | def osme_block(in_block, ch, ratio=16, name=None): 88 | z = GlobalAveragePooling2D()(in_block) # 1 89 | x = Dense(ch//ratio, activation='relu')(z) # 2 90 | x = Dense(ch, activation='sigmoid', name=name)(x) # 3 91 | return Multiply()([in_block, x]) # 4 92 | 93 | s_1 = osme_block(split[0], split[0].shape[3].value, name='attention1') 94 | s_2 = osme_block(split[1], split[1].shape[3].value, name='attention2') 95 | 96 | fc1 = Flatten()(s_1) 97 | fc2 = Flatten()(s_2) 98 | 99 | fc1 = Dense(1024, name='fc1')(fc1) 100 | fc2 = Dense(1024, name='fc2')(fc2) 101 | 102 | #fc = fc1 103 | fc = concatenate([fc1,fc2]) #fc1 + fc2 104 | 105 | x = GlobalAveragePooling2D(name='avg_pool')(base_model.output) 106 | x = Dense(1024, activation="relu")(x) 107 | predictions = Dense(num_classes, activation='softmax', name='predictions')(x) 108 | 109 | #%% 110 | 111 | model = Model(inputs=base_model.input, outputs=predictions) 112 | 113 | opt = SGD(lr=0.01, momentum=0.9, decay=0.0005) 114 | 115 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME.best_loss.hdf5") 116 | 117 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 118 | 119 | #%% 120 | plot_model(model, to_file="model_inceptv3.png", show_shapes=True) 121 | 122 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 123 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_inceptv3_with_OSME_SE.best_loss.hdf5', verbose=1, save_best_only=True) 124 | 125 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, 126 | patience=3, min_lr=0.000001) 127 | 128 | #es_cb = EarlyStopping(patience=11) 129 | 130 | #%% fit_generator 131 | 132 | history = model.fit_generator(train_generator, 133 | steps_per_epoch=train_nb/BATCH_SIZE, 134 | epochs=15, 135 | validation_data=validation_generator, 136 | validation_steps=64, 137 | verbose=1, 138 | callbacks=[reduce_lr, checkpointer]) 139 | 140 | #%% plot results 141 | import datetime 142 | now = datetime.datetime.now() 143 | 144 | plt.plot(history.history['acc']) 145 | plt.plot(history.history['val_acc']) 146 | plt.title('model_without_MAMC accuracy') 147 | plt.ylabel('accuracy') 148 | plt.xlabel('epoch') 149 | plt.legend(['train', 'test'], loc='upper left') 150 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_inceptv3_with_OSME_SE_{0:%d%m}-{0:%H%M%S}.png".format(now)) 151 | plt.show() 152 | 153 | #loss 154 | plt.plot(history.history['loss']) 155 | plt.plot(history.history['val_loss']) 156 | plt.title('model_without_MAMC loss') 157 | plt.ylabel('loss') 158 | plt.xlabel('epoch') 159 | plt.legend(['train', 'test'], loc='upper left') 160 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_inceptv3_with_OSME_SE_{0:%d%m}-{0:%H%M%S}.png".format(now)) 161 | plt.show() 162 | -------------------------------------------------------------------------------- /model_with_OSME_vmmrdb.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Thu Nov 1 05:27:24 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | #!/usr/bin/env python3 10 | # -*- coding: utf-8 -*- 11 | """ 12 | Created on Tue Oct 23 08:45:20 2018 13 | 14 | @author: n-kamiya 15 | """ 16 | import keras 17 | from keras.callbacks import EarlyStopping 18 | from keras.applications.resnet50 import ResNet50 19 | from keras.applications.vgg19 import VGG19 20 | from keras.applications.inception_v3 import InceptionV3 21 | from keras.models import Model 22 | from keras.layers import Input, Dense, Flatten, Multiply, add 23 | from keras.layers import GlobalAveragePooling2D 24 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 25 | from keras.optimizers import SGD, RMSprop 26 | from keras.utils.np_utils import to_categorical 27 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array 28 | from keras.regularizers import l2 29 | import matplotlib.image as mpimg 30 | import numpy as np 31 | import keras.backend as K 32 | import pathlib 33 | from PIL import Image 34 | import cv2 35 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 36 | import matplotlib.pyplot as plt 37 | from keras.utils.vis_utils import plot_model 38 | import pandas as pd 39 | #%% 40 | 41 | K.clear_session() 42 | BATCH_SIZE = 16 43 | test_nb = 5794 44 | train_nb = 5994 45 | num_classes = 3040 46 | img_size= 448 47 | classes = [] 48 | 49 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 50 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 51 | 52 | train_path = "/home/n-kamiya/datasets/VMMRdb_3040/" 53 | # 54 | # 55 | #with open("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/classes.txt") as f: 56 | # for l in f.readlines(): 57 | # data = l.split() 58 | # classes.append(data[1]) 59 | 60 | #%% create data generator 61 | 62 | train_datagen = ImageDataGenerator(rescale = 1.0/255, 63 | zoom_range=[1.2,2.0], 64 | rotation_range=90, 65 | horizontal_flip=True, 66 | vertical_flip=True, 67 | validation_split=0.3) 68 | test_datagen = ImageDataGenerator(rescale = 1.0/255 69 | ) 70 | 71 | train_generator = train_datagen.flow_from_directory( 72 | train_path, 73 | target_size=(img_size, img_size), 74 | batch_size=BATCH_SIZE, 75 | seed = 13) 76 | 77 | validation_generator = test_datagen.flow_from_directory( 78 | train_path, 79 | target_size=(img_size, img_size), 80 | batch_size=BATCH_SIZE, 81 | seed = 13) 82 | #%% finetuning resnet50 83 | 84 | input_tensor = Input(shape=(img_size, img_size, 3)) 85 | #base_model = VGG19(weights = "imagenet", include_top=False, input_tensor=input_tensor) 86 | #base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 87 | base_model = InceptionV3(weights = "imagenet", include_top=False, input_tensor=input_tensor) 88 | 89 | for layer in base_model.layers: 90 | layer.trainable = False 91 | # if isinstance(layer, keras.layers.normalization.BatchNormalization): 92 | # layer._per_input_updates = {} 93 | 94 | #%% Implementation of OSME module 95 | 96 | def osme_block(in_block, ch, ratio=16): 97 | z = GlobalAveragePooling2D()(in_block) # 1 98 | x = Dense(ch//ratio, activation='relu')(z) # 2 99 | x = Dense(ch, activation='sigmoid')(x) # 3 100 | return Multiply()([in_block, x]) # 4 101 | 102 | s_1 = osme_block(base_model.output, base_model.output_shape[3]) 103 | s_2 = osme_block(base_model.output, base_model.output_shape[3]) 104 | 105 | fc1 = Dense(1024, name='fc1')(s_1) 106 | fc2 = Dense(1024, name='fc2')(s_2) 107 | 108 | fc1 = Flatten()(fc1) 109 | fc2 = Flatten()(fc2) 110 | 111 | fc = add([fc1,fc2]) # fc1 + fc2 112 | 113 | prediction = Dense(num_classes, activation='softmax', name='prediction')(fc) 114 | 115 | 116 | model = Model(inputs=base_model.input, outputs=prediction) 117 | 118 | opt = SGD(lr=0.001, momentum=0.9, decay=0.0005) 119 | #opt = RMSprop(lr=0.001) 120 | 121 | 122 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_osme_vgg_imagenet.best_loss.hdf5") 123 | 124 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 125 | 126 | #%% 127 | plot_model(model, to_file="model.png", show_shapes=True) 128 | 129 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 130 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_osme_inceptv3_imagenet_vmmrdb.best_loss.hdf5', verbose=1, save_best_only=True) 131 | 132 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, 133 | patience=5, min_lr=0.0000001) 134 | 135 | #es_cb = EarlyStopping(patience=11) 136 | 137 | #%% fit_generator 138 | 139 | history = model.fit_generator(train_generator, 140 | steps_per_epoch=train_nb/BATCH_SIZE, 141 | epochs=60, 142 | validation_data=validation_generator, 143 | validation_steps=64, 144 | verbose=1, 145 | callbacks=[reduce_lr, checkpointer]) 146 | 147 | #%% plot results 148 | import datetime 149 | now = datetime.datetime.now() 150 | 151 | plt.plot(history.history['acc']) 152 | plt.plot(history.history['val_acc']) 153 | plt.title('model_without_MAMC accuracy') 154 | plt.ylabel('accuracy') 155 | plt.xlabel('epoch') 156 | plt.legend(['train', 'test'], loc='upper left') 157 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_inceptv3_with_OSME_vmmmrdb{0:%d%m}-{0:%H%M%S}.png".format(now)) 158 | #plt.show() 159 | 160 | #loss 161 | plt.plot(history.history['loss']) 162 | plt.plot(history.history['val_loss']) 163 | plt.title('model_without_MAMC loss') 164 | plt.ylabel('loss') 165 | plt.xlabel('epoch') 166 | plt.legend(['train', 'test'], loc='upper left') 167 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_inceptv3_with_OSME_vmmmrdb{0:%d%m}-{0:%H%M%S}.png".format(now)) 168 | #plt.show() 169 | -------------------------------------------------------------------------------- /model_without_MAMC.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 23 08:45:20 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | from keras.applications.resnet50 import ResNet50 10 | from keras.applications.resnet50 import preprocess_input 11 | from keras.models import Sequential, Model 12 | from keras.layers import Dense, Dropout, Activation, Flatten 13 | from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, AveragePooling2D 14 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 15 | from keras.optimizers import SGD 16 | from keras.utils.np_utils import to_categorical 17 | from keras.preprocessing.image import ImageDataGenerator 18 | from keras.regularizers import l2 19 | import matplotlib.image as mpimg 20 | import numpy as np 21 | import keras.backend as K 22 | import pathlib 23 | from PIL import Image 24 | import cv2 25 | 26 | 27 | K.clear_session() 28 | 29 | num_classes = 200 30 | img_size=488 31 | image_txt = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011" 32 | images_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/images" 33 | 34 | #%% 35 | #def load_images(root): 36 | # all_imgs = [] 37 | # all_classes = [] 38 | # img_path = [] 39 | # Xtrain = [] 40 | # ytrain = [] 41 | # Xtest = [] 42 | # ytest = [] 43 | # s = [] 44 | # 45 | # with open("%s/images.txt" % root) as f: 46 | # for line in f.readlines(): 47 | # data = line.split() 48 | # img_path.append(data[1]) 49 | # class_name = data[1].split(".") 50 | # all_classes.append(class_name[0]) 51 | # 52 | # with open("%s/train_test_split.txt" % root) as f: 53 | # for line in f.readlines(): 54 | # data = line.split() 55 | # s.append(data) 56 | # 57 | # for image in img_path: 58 | # path = images_path +'/'+ image 59 | # img = cv2.imread(path) 60 | # resize_img_ar = cv2.resize(img,(img_size, img_size)) 61 | # all_imgs.append(resize_img_ar) 62 | # 63 | # for img in s: 64 | # if img[1] == "1": 65 | # Xtrain.append(all_imgs[int(img[0])-1]) 66 | # ytrain.append(all_classes[int(img[0])-1]) 67 | # elif img[1] == "0": 68 | # Xtest.append(all_imgs[int(img[0])-1]) 69 | # ytest.append(all_classes[int(img[0])-1]) 70 | # 71 | # 72 | # 73 | # return np.array(Xtrain), np.array(ytrain), np.array(Xtest), np.array(ytest) 74 | 75 | #%% 76 | #X_train, y_train, X_test, y_test = load_images(image_txt) 77 | #%% 78 | train_datagen = ImageDataGenerator(rescale=1./img_size) 79 | 80 | test_datagen = ImageDataGenerator(rescale=1./img_size) 81 | 82 | train_generator = train_datagen.flow_from_directory(images_path ,X_train, y_train, batch_size=64, seed = 13) 83 | validation_generator = train_datagen.flow_from_directory(images_path ,X_train, y_train, batch_size=64, seed = 13) 84 | #%% 85 | base_model = ResNet50(weights = "imagenet", include_top=False) 86 | 87 | x = base_model.output 88 | x = GlobalAveragePooling2D()(x) 89 | predictions = Dense(1, kernel_initializer="glorot_uniform", activation="sigmoid", kernel_regularizer=l2(.0005))(x) 90 | model = Model(inputs=base_model.input, outputs=predictions) 91 | 92 | for layer in base_model.layers: 93 | layer.trainable = False 94 | 95 | opt = SGD(lr=.01, momentum=.9) 96 | model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy']) 97 | #%% 98 | checkpointer = ModelCheckpoint(filepath='model.{epoch:02d}-{val_loss:.2f}.hdf5', verbose=1, save_best_only=True) 99 | 100 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, 101 | patience=5, min_lr=0.001) 102 | 103 | history = model.fit_generator(train_generator, 104 | steps_per_epoch=2000, 105 | epochs=10, 106 | validation_data=test_generator, 107 | validation_steps=800, 108 | verbose=1, 109 | callbacks=[reduce_lr, checkpointer]) 110 | -------------------------------------------------------------------------------- /model_without_OSME.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 23 08:45:20 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | from keras.callbacks import EarlyStopping 10 | from keras.applications.resnet50 import ResNet50 11 | from keras.applications.inception_v3 import InceptionV3 12 | from keras.models import Sequential, Model 13 | from keras.layers import Input, Dense, Dropout, Activation, Flatten, Multiply 14 | from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, AveragePooling2D 15 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 16 | from keras.optimizers import SGD 17 | from keras.utils.np_utils import to_categorical 18 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array 19 | from keras.regularizers import l2 20 | import matplotlib.image as mpimg 21 | import numpy as np 22 | import keras.backend as K 23 | import pathlib 24 | from PIL import Image 25 | import cv2 26 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 27 | import matplotlib.pyplot as plt 28 | from keras.utils.vis_utils import plot_model 29 | import pandas as pd 30 | #%% 31 | 32 | K.clear_session() 33 | BATCH_SIZE = 16 34 | test_nb = 5794 35 | train_nb = 5994 36 | num_classes = 200 37 | img_size= 336 38 | classes = [] 39 | 40 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 41 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 42 | #%% 43 | 44 | 45 | with open("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/classes.txt") as f: 46 | for l in f.readlines(): 47 | data = l.split() 48 | classes.append(data[1]) 49 | 50 | 51 | #%% create data generator 52 | 53 | train_datagen = ImageDataGenerator(rescale = 1./255, 54 | zoom_range=[0.8,1], 55 | rotation_range=30, 56 | horizontal_flip=True) 57 | test_datagen = ImageDataGenerator(rescale = 1./255) 58 | 59 | train_generator = train_datagen.flow_from_directory( 60 | train_path, 61 | target_size=(img_size, img_size), 62 | batch_size=BATCH_SIZE, 63 | seed = 13, 64 | ) 65 | 66 | validation_generator = test_datagen.flow_from_directory( 67 | test_path, 68 | target_size=(img_size, img_size), 69 | batch_size=BATCH_SIZE, 70 | seed = 13, 71 | ) 72 | #%% finetuning resnet50 73 | 74 | input_tensor = Input(shape=(img_size, img_size, 3)) 75 | #base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 76 | base_model = InceptionV3(weights = "imagenet", include_top=False, input_tensor=input_tensor) 77 | 78 | 79 | #for layer in base_model.layers: 80 | # layer.trainable = False 81 | 82 | #%% 83 | # change only the output layer 84 | top_model = Sequential() 85 | #top_model.add(Flatten(input_shape=base_model.output_shape[1:])) 86 | top_model.add(GlobalAveragePooling2D(input_shape=base_model.output_shape[1:])) 87 | top_model.add(Dense(1024,activation='relu')) 88 | top_model.add(Dense(num_classes, activation='softmax')) 89 | 90 | model = Model(inputs=base_model.input, outputs=top_model(base_model.output)) 91 | 92 | opt = SGD(lr=0.001, momentum=0.9, decay=0.0005) 93 | 94 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME.best_loss.hdf5") 95 | 96 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 97 | 98 | #%% 99 | plot_model(model, to_file="model_inceptv3.png", show_shapes=True) 100 | 101 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 102 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME_336.best_loss.hdf5', verbose=1, save_best_only=True) 103 | 104 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, 105 | patience=3, min_lr=0.000001) 106 | 107 | #es_cb = EarlyStopping(patience=11) 108 | 109 | #%% fit_generator 110 | 111 | history = model.fit_generator(train_generator, 112 | steps_per_epoch=train_nb/BATCH_SIZE, 113 | epochs=15, 114 | validation_data=validation_generator, 115 | validation_steps=64, 116 | verbose=1, 117 | callbacks=[reduce_lr, checkpointer]) 118 | 119 | #%% plot results 120 | import datetime 121 | now = datetime.datetime.now() 122 | 123 | plt.plot(history.history['acc']) 124 | plt.plot(history.history['val_acc']) 125 | plt.title('model_without_MAMC accuracy') 126 | plt.ylabel('accuracy') 127 | plt.xlabel('epoch') 128 | plt.legend(['train', 'test'], loc='upper left') 129 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_inceptv3_without_OSME_336{0:%d%m}-{0:%H%M%S}.png".format(now)) 130 | plt.show() 131 | 132 | #loss 133 | plt.plot(history.history['loss']) 134 | plt.plot(history.history['val_loss']) 135 | plt.title('model_without_MAMC loss') 136 | plt.ylabel('loss') 137 | plt.xlabel('epoch') 138 | plt.legend(['train', 'test'], loc='upper left') 139 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_inceptv3_without_OSME_336{0:%d%m}-{0:%H%M%S}.png".format(now)) 140 | plt.show() 141 | -------------------------------------------------------------------------------- /otsu.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Mon Nov 19 03:33:03 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | import cv2 10 | 11 | def otsu(input_tensor): 12 | 13 | gray = cv2.cvtColor(input_tensor, cv2.COLOR_RGB2GRAY) 14 | 15 | 16 | ret, th2 = cv2.threshold(gray, 0, 255, cv2.THRESH_OTSU) 17 | 18 | 19 | cv2.imwrite("th2.jpg", th2) 20 | 21 | return th2 22 | 23 | 24 | if __name__ == "__main__": 25 | otsu() -------------------------------------------------------------------------------- /res-net-101.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Wed Oct 17 01:17:40 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | # -*- coding: utf-8 -*- 10 | 11 | import cv2 12 | import numpy as np 13 | 14 | 15 | from keras.layers import Input, Dense, Convolution2D, MaxPooling2D, AveragePooling2D, ZeroPadding2D, Dropout, Flatten, merge, Reshape, Activation, Lambda, GlobalAveragePooling2D, Merge 16 | from keras.optimizers import SGD 17 | from keras.layers.normalization import BatchNormalization 18 | from keras.models import Model 19 | from keras import initializations 20 | from keras.engine import Layer, InputSpec 21 | from keras import backend as K 22 | 23 | import sys 24 | sys.setrecursionlimit(3000) 25 | 26 | class Scale(Layer): 27 | '''Learns a set of weights and biases used for scaling the input data. 28 | the output consists simply in an element-wise multiplication of the input 29 | and a sum of a set of constants: 30 | 31 | out = in * gamma + beta, 32 | 33 | where 'gamma' and 'beta' are the weights and biases larned. 34 | 35 | # Arguments 36 | axis: integer, axis along which to normalize in mode 0. For instance, 37 | if your input tensor has shape (samples, channels, rows, cols), 38 | set axis to 1 to normalize per feature map (channels axis). 39 | momentum: momentum in the computation of the 40 | exponential average of the mean and standard deviation 41 | of the data, for feature-wise normalization. 42 | weights: Initialization weights. 43 | List of 2 Numpy arrays, with shapes: 44 | `[(input_shape,), (input_shape,)]` 45 | beta_init: name of initialization function for shift parameter 46 | (see [initializations](../initializations.md)), or alternatively, 47 | Theano/TensorFlow function to use for weights initialization. 48 | This parameter is only relevant if you don't pass a `weights` argument. 49 | gamma_init: name of initialization function for scale parameter (see 50 | [initializations](../initializations.md)), or alternatively, 51 | Theano/TensorFlow function to use for weights initialization. 52 | This parameter is only relevant if you don't pass a `weights` argument. 53 | gamma_init: name of initialization function for scale parameter (see 54 | [initializations](../initializations.md)), or alternatively, 55 | Theano/TensorFlow function to use for weights initialization. 56 | This parameter is only relevant if you don't pass a `weights` argument. 57 | ''' 58 | def __init__(self, weights=None, axis=-1, momentum = 0.9, beta_init='zero', gamma_init='one', **kwargs): 59 | self.momentum = momentum 60 | self.axis = axis 61 | self.beta_init = initializations.get(beta_init) 62 | self.gamma_init = initializations.get(gamma_init) 63 | self.initial_weights = weights 64 | super(Scale, self).__init__(**kwargs) 65 | 66 | def build(self, input_shape): 67 | self.input_spec = [InputSpec(shape=input_shape)] 68 | shape = (int(input_shape[self.axis]),) 69 | 70 | self.gamma = self.gamma_init(shape, name='{}_gamma'.format(self.name)) 71 | self.beta = self.beta_init(shape, name='{}_beta'.format(self.name)) 72 | self.trainable_weights = [self.gamma, self.beta] 73 | if self.initial_weights is not None: 74 | self.set_weights(self.initial_weights) 75 | del self.initial_weights 76 | 77 | def call(self, x, mask=None): 78 | input_shape = self.input_spec[0].shape 79 | broadcast_shape = [1] * len(input_shape) 80 | broadcast_shape[self.axis] = input_shape[self.axis] 81 | 82 | out = K.reshape(self.gamma, broadcast_shape) * x + K.reshape(self.beta, broadcast_shape) 83 | return out 84 | 85 | def get_config(self): 86 | config = {"momentum": self.momentum, "axis": self.axis} 87 | base_config = super(Scale, self).get_config() 88 | return dict(list(base_config.items()) + list(config.items())) 89 | 90 | def identity_block(input_tensor, kernel_size, filters, stage, block): 91 | '''The identity_block is the block that has no conv layer at shortcut 92 | # Arguments 93 | input_tensor: input tensor 94 | kernel_size: defualt 3, the kernel size of middle conv layer at main path 95 | filters: list of integers, the nb_filters of 3 conv layer at main path 96 | stage: integer, current stage label, used for generating layer names 97 | block: 'a','b'..., current block label, used for generating layer names 98 | ''' 99 | eps = 1.1e-5 100 | nb_filter1, nb_filter2, nb_filter3 = filters 101 | conv_name_base = 'res' + str(stage) + block + '_branch' 102 | bn_name_base = 'bn' + str(stage) + block + '_branch' 103 | scale_name_base = 'scale' + str(stage) + block + '_branch' 104 | 105 | x = Convolution2D(nb_filter1, 1, 1, name=conv_name_base + '2a', bias=False)(input_tensor) 106 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2a')(x) 107 | x = Scale(axis=bn_axis, name=scale_name_base + '2a')(x) 108 | x = Activation('relu', name=conv_name_base + '2a_relu')(x) 109 | 110 | x = ZeroPadding2D((1, 1), name=conv_name_base + '2b_zeropadding')(x) 111 | x = Convolution2D(nb_filter2, kernel_size, kernel_size, 112 | name=conv_name_base + '2b', bias=False)(x) 113 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2b')(x) 114 | x = Scale(axis=bn_axis, name=scale_name_base + '2b')(x) 115 | x = Activation('relu', name=conv_name_base + '2b_relu')(x) 116 | 117 | x = Convolution2D(nb_filter3, 1, 1, name=conv_name_base + '2c', bias=False)(x) 118 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2c')(x) 119 | x = Scale(axis=bn_axis, name=scale_name_base + '2c')(x) 120 | 121 | x = merge([x, input_tensor], mode='sum', name='res' + str(stage) + block) 122 | x = Activation('relu', name='res' + str(stage) + block + '_relu')(x) 123 | return x 124 | 125 | def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)): 126 | '''conv_block is the block that has a conv layer at shortcut 127 | # Arguments 128 | input_tensor: input tensor 129 | kernel_size: defualt 3, the kernel size of middle conv layer at main path 130 | filters: list of integers, the nb_filters of 3 conv layer at main path 131 | stage: integer, current stage label, used for generating layer names 132 | block: 'a','b'..., current block label, used for generating layer names 133 | Note that from stage 3, the first conv layer at main path is with subsample=(2,2) 134 | And the shortcut should have subsample=(2,2) as well 135 | ''' 136 | eps = 1.1e-5 137 | nb_filter1, nb_filter2, nb_filter3 = filters 138 | conv_name_base = 'res' + str(stage) + block + '_branch' 139 | bn_name_base = 'bn' + str(stage) + block + '_branch' 140 | scale_name_base = 'scale' + str(stage) + block + '_branch' 141 | 142 | x = Convolution2D(nb_filter1, 1, 1, subsample=strides, 143 | name=conv_name_base + '2a', bias=False)(input_tensor) 144 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2a')(x) 145 | x = Scale(axis=bn_axis, name=scale_name_base + '2a')(x) 146 | x = Activation('relu', name=conv_name_base + '2a_relu')(x) 147 | 148 | x = ZeroPadding2D((1, 1), name=conv_name_base + '2b_zeropadding')(x) 149 | x = Convolution2D(nb_filter2, kernel_size, kernel_size, 150 | name=conv_name_base + '2b', bias=False)(x) 151 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2b')(x) 152 | x = Scale(axis=bn_axis, name=scale_name_base + '2b')(x) 153 | x = Activation('relu', name=conv_name_base + '2b_relu')(x) 154 | 155 | x = Convolution2D(nb_filter3, 1, 1, name=conv_name_base + '2c', bias=False)(x) 156 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2c')(x) 157 | x = Scale(axis=bn_axis, name=scale_name_base + '2c')(x) 158 | 159 | shortcut = Convolution2D(nb_filter3, 1, 1, subsample=strides, 160 | name=conv_name_base + '1', bias=False)(input_tensor) 161 | shortcut = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '1')(shortcut) 162 | shortcut = Scale(axis=bn_axis, name=scale_name_base + '1')(shortcut) 163 | 164 | x = merge([x, shortcut], mode='sum', name='res' + str(stage) + block) 165 | x = Activation('relu', name='res' + str(stage) + block + '_relu')(x) 166 | return x 167 | 168 | def resnet101_model(weights_path=None): 169 | '''Instantiate the ResNet101 architecture, 170 | # Arguments 171 | weights_path: path to pretrained weight file 172 | # Returns 173 | A Keras model instance. 174 | ''' 175 | eps = 1.1e-5 176 | 177 | # Handle Dimension Ordering for different backends 178 | global bn_axis 179 | if K.image_dim_ordering() == 'tf': 180 | bn_axis = 3 181 | img_input = Input(shape=(224, 224, 3), name='data') 182 | else: 183 | bn_axis = 1 184 | img_input = Input(shape=(3, 224, 224), name='data') 185 | 186 | x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input) 187 | x = Convolution2D(64, 7, 7, subsample=(2, 2), name='conv1', bias=False)(x) 188 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name='bn_conv1')(x) 189 | x = Scale(axis=bn_axis, name='scale_conv1')(x) 190 | x = Activation('relu', name='conv1_relu')(x) 191 | x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x) 192 | 193 | x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) 194 | x = identity_block(x, 3, [64, 64, 256], stage=2, block='b') 195 | x = identity_block(x, 3, [64, 64, 256], stage=2, block='c') 196 | 197 | x = conv_block(x, 3, [128, 128, 512], stage=3, block='a') 198 | for i in range(1,3): 199 | x = identity_block(x, 3, [128, 128, 512], stage=3, block='b'+str(i)) 200 | 201 | x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a') 202 | for i in range(1,23): 203 | x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b'+str(i)) 204 | 205 | x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a') 206 | x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b') 207 | x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c') 208 | 209 | x_fc = AveragePooling2D((7, 7), name='avg_pool')(x) 210 | x_fc = Flatten()(x_fc) 211 | x_fc = Dense(1000, activation='softmax', name='fc1000')(x_fc) 212 | 213 | model = Model(img_input, x_fc) 214 | 215 | # load weights 216 | if weights_path: 217 | model.load_weights(weights_path, by_name=True) 218 | 219 | return model 220 | 221 | if __name__ == '__main__': 222 | 223 | im = cv2.resize(cv2.imread('cat.jpg'), (224, 224)).astype(np.float32) 224 | 225 | # Remove train image mean 226 | im[:,:,0] -= 103.939 227 | im[:,:,1] -= 116.779 228 | im[:,:,2] -= 123.68 229 | 230 | # Transpose image dimensions (Theano uses the channels as the 1st dimension) 231 | if K.image_dim_ordering() == 'th': 232 | im = im.transpose((2,0,1)) 233 | 234 | # Use pre-trained weights for Theano backend 235 | weights_path = 'resnet101_weights_th.h5' 236 | else: 237 | # Use pre-trained weights for Tensorflow backend 238 | weights_path = 'resnet101_weights_tf.h5' 239 | 240 | # Insert a new dimension for the batch_size 241 | im = np.expand_dims(im, axis=0) 242 | 243 | # Test pretrained model 244 | model = resnet101_model(weights_path) 245 | sgd = SGD(lr=1e-2, decay=1e-6, momentum=0.9, nesterov=True) 246 | model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy']) 247 | 248 | out = model.predict(im) 249 | print(np.argmax(out)) -------------------------------------------------------------------------------- /resnet50_alpha.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 23 08:45:20 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | import keras 9 | from keras.callbacks import EarlyStopping 10 | from keras.applications.resnet50 import ResNet50 11 | from keras.applications.vgg19 import VGG19 12 | from keras.applications.inception_v3 import InceptionV3 13 | from keras.models import Model 14 | from keras.layers import Input, Dense, Flatten, Multiply, add, concatenate, Lambda 15 | from keras.layers import GlobalAveragePooling2D, Dropout 16 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 17 | from keras.optimizers import SGD, RMSprop 18 | from keras.utils.np_utils import to_categorical 19 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array, array_to_img 20 | from keras.regularizers import l2 21 | import matplotlib.image as mpimg 22 | import numpy as np 23 | import keras.backend as K 24 | import pathlib 25 | from PIL import Image 26 | import cv2 27 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 28 | import matplotlib.pyplot as plt 29 | from keras.utils.vis_utils import plot_model 30 | import pandas as pd 31 | import grad_cam 32 | from keras.models import load_model 33 | import os 34 | 35 | from se_inception_v3 import se_inception_v3 36 | #%% 37 | import tensorflow as tf 38 | from keras.backend import tensorflow_backend 39 | 40 | K.clear_session() 41 | 42 | config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True)) 43 | session = tf.Session(config=config) 44 | tensorflow_backend.set_session(session) 45 | 46 | BATCH_SIZE = 16 47 | test_nb = 5794 48 | train_nb = 5994 49 | num_classes = 200 50 | img_size= 336 51 | classes = [] 52 | 53 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 54 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 55 | # 56 | # 57 | with open("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/classes.txt") as f: 58 | for l in f.readlines(): 59 | data = l.split() 60 | classes.append(data[1]) 61 | 62 | #%% add random crop function 63 | #def random_crop(img, random_crop_size): 64 | # # Note: image_data_format is 'channel_last' 65 | # assert img.shape[2] == 3 66 | # height, width = img.shape[0], img.shape[1] 67 | # dy, dx = random_crop_size 68 | # x = np.random.randint(0, width - dx + 1) 69 | # y = np.random.randint(0, height - dy + 1) 70 | # return img[y:(y+dy), x:(x+dx), :] 71 | # 72 | # 73 | #def crop_generator(batches, crop_length , seed): 74 | # """Take as input a Keras ImageGen (Iterator) and generate random 75 | # crops from the image batches generated by the original iterator. 76 | # """ 77 | # while True: 78 | # batch_x, batch_y = next(batches) 79 | # batch_crops = np.zeros((batch_x.shape[0], crop_length, crop_length, 3)) 80 | # for i in range(batch_x.shape[0]): 81 | # batch_crops[i] = random_crop(batch_x[i], (crop_length, crop_length)) 82 | # yield (batch_crops, batch_y) 83 | 84 | #%% create data generator 85 | 86 | train_datagen = ImageDataGenerator(rescale = 1.0/255, 87 | # featurewise_center=True, 88 | # featurewise_std_normalization=True, 89 | zoom_range=[0.7,1.0], 90 | rotation_range=30, 91 | # zca_whitening=True, 92 | horizontal_flip=True, 93 | vertical_flip=False, 94 | ) 95 | test_datagen = ImageDataGenerator(rescale = 1.0/255 96 | ) 97 | 98 | 99 | train_generator = train_datagen.flow_from_directory( 100 | train_path, 101 | target_size=(img_size, img_size), 102 | batch_size=BATCH_SIZE, 103 | seed = 13, 104 | #save_to_dir='/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/generated_images4/' 105 | ) 106 | 107 | validation_generator = test_datagen.flow_from_directory( 108 | test_path, 109 | target_size=(img_size, img_size), 110 | batch_size=BATCH_SIZE, 111 | seed = 13) 112 | 113 | #seed = 13 114 | #crop_size = 352 115 | #train_crops = crop_generator(train_generator, crop_size, seed) 116 | 117 | #%% finetuning resnet50 118 | 119 | input_tensor = Input(shape=(img_size, img_size, 3)) 120 | #base_model = VGG19(weights = "imagenet", include_top=False, input_tensor=input_tensor) 121 | base_model = ResNet50(weights = "imagenet", include_top=False, input_tensor=input_tensor) 122 | #base_model = InceptionV3(weights = "imagenet", include_top=False, input_tensor=input_tensor) 123 | #base_model = se_inception_v3(include_top=False, input_tensor=input_tensor) 124 | #base_model.load_weights("/home/n-kamiya/.keras/models/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5", by_name=True) 125 | #for layer in base_model.layers: 126 | # layer.trainable = False 127 | #%% 128 | plot_model(base_model, to_file="resnet50_model.png", show_shapes=True) 129 | 130 | #%% Implementation of OSME module 131 | 132 | split = Lambda( lambda x: tf.split(x,num_or_size_splits=2,axis=3))(base_model.output) 133 | #%% 134 | def osme_block(in_block, ch, ratio=16, name=None): 135 | z = GlobalAveragePooling2D()(in_block) # 1 136 | x = Dense(ch//ratio, activation='relu')(z) # 2 137 | x = Dense(ch, activation='sigmoid', name=name)(x) # 3 138 | return Multiply()([in_block, x]) # 4 139 | 140 | s_1 = osme_block(split[0], split[0].shape[3].value, name='attention1') 141 | s_2 = osme_block(split[1], split[1].shape[3].value, name='attention2') 142 | 143 | fc1 = Flatten()(s_1) 144 | fc2 = Flatten()(s_2) 145 | 146 | fc1 = Dense(1024, name='fc1')(fc1) 147 | fc2 = Dense(1024, name='fc2')(fc2) 148 | 149 | #fc = fc1 150 | fc = concatenate([fc1,fc2]) #fc1 + fc2 151 | 152 | #fc = Dropout(0.5)(fc) #add dropout 153 | prediction = Dense(num_classes, activation='softmax', name='prediction')(fc) 154 | 155 | 156 | model = Model(inputs=base_model.input, outputs=prediction) 157 | 158 | opt = SGD(lr=0.001, momentum=0.9, decay=0.0005) 159 | #opt = RMSprop(lr=0.001) 160 | 161 | 162 | #model.load_weights("/home/n-kamiya/models/model_without_MAMC/model_osme_inceptv3_beta.best_loss.hdf5") 163 | 164 | model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 165 | 166 | #%% 167 | plot_model(model, to_file="model.png", show_shapes=True) 168 | 169 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 170 | import datetime 171 | now = datetime.datetime.now() 172 | 173 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_osme_resnet50_alpha.best_loss_.hdf5', verbose=1, save_best_only=True) 174 | 175 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, 176 | patience=5, min_lr=0.0000001) 177 | 178 | #es_cb = EarlyStopping(patience=11) 179 | 180 | #%% fit_generator 181 | 182 | history = model.fit_generator(train_generator, 183 | steps_per_epoch=train_nb/BATCH_SIZE, 184 | epochs=15, 185 | validation_data=validation_generator, 186 | validation_steps=64, 187 | verbose=1, 188 | callbacks=[reduce_lr, checkpointer]) 189 | 190 | #%% plot results 191 | import datetime 192 | now = datetime.datetime.now() 193 | 194 | plt.plot(history.history['acc']) 195 | plt.plot(history.history['val_acc']) 196 | plt.title('model_without_MAMC accuracy') 197 | plt.ylabel('accuracy') 198 | plt.xlabel('epoch') 199 | plt.legend(['train', 'test'], loc='upper left') 200 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_se_resnet50_alpha_with_OSME{0:%d%m}-{0:%H%M%S}.png".format(now)) 201 | plt.show() 202 | 203 | #loss 204 | plt.plot(history.history['loss']) 205 | plt.plot(history.history['val_loss']) 206 | plt.title('model_without_MAMC loss') 207 | plt.ylabel('loss') 208 | plt.xlabel('epoch') 209 | plt.legend(['train', 'test'], loc='upper left') 210 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_se_resnet50_alpha_with_OSME{0:%d%m}-{0:%H%M%S}.png".format(now)) 211 | plt.show() 212 | 213 | 214 | #%% 215 | 216 | #model = load_model("/home/n-kamiya/models/model_without_MAMC/model_inceptv3_without_OSME_SE.best_loss.hdf5", custom_objects={"tf": tf}) 217 | ##%% 218 | #for file in os.listdir("/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/013.Bobolink/"): 219 | # x = img_to_array(load_img(file, target_size=(448,448))) 220 | # 221 | # y_proba = model.predict(x.reshape([-1,448,448,3])) 222 | # print(classes[int(y_proba.argmax(axis = -1))]) 223 | ##%%Model Evaluation 224 | #model._generator(generator=validation_generator, steps = test_nb/BATCH_SIZE, verbose = 1) 225 | ##%% 226 | #image1 = grad_cam.Grad_Cam(model, x, "multiply_1") 227 | #image2 = grad_cam.Grad_Cam(model, x, "multiply_2") 228 | # 229 | #image1 = array_to_img(image1) 230 | #image2 = array_to_img(image2) 231 | # 232 | #image1.show() 233 | #image1.save("/home/n-kamiya/images/inceptv3_beta_multiply_1.png") 234 | #image2.show() 235 | #image2.save("/home/n-kamiya/images/inceptv3_beta_multiply_2.png") 236 | -------------------------------------------------------------------------------- /resnet50_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xcnkx/fine_grained_classification/26797fb79d89242330f06ac405eb1bde925769e8/resnet50_model.png -------------------------------------------------------------------------------- /se_block.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Wed Nov 7 07:48:47 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | 10 | # Implementation of Squeeze and Excite block 11 | 12 | 13 | from keras.layers import GlobalAveragePooling2D, Reshape, Dense, multiply 14 | 15 | def squeeze_excite_block(input, ratio = 16): 16 | init = input 17 | channel_axis = -1 # Since we are using Tensorflow 18 | filters = init._keras_shape[channel_axis] 19 | shape = (1, 1, filters) 20 | 21 | se = GlobalAveragePooling2D()(init) 22 | se = Reshape(shape)(se) 23 | se = Dense(filters // ratio, activation = "relu", kernel_initializer = "he_normal", use_bias = False)(se) 24 | se = Dense(filters, activation = "sigmoid", kernel_initializer = "he_normal", use_bias = False)(se) 25 | 26 | output = multiply([init, se]) 27 | 28 | return output -------------------------------------------------------------------------------- /se_inception_v3.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Wed Nov 7 07:31:40 2018 5 | 6 | @author: n-kamiya 7 | 8 | Squeeze and Excite Inception V3 model 9 | Major portions of this code is adapted from the applications folder of Keras. 10 | Note that the input image format for this model is different than for 11 | the VGG16 and ResNet models (299x299 instead of 224x224), 12 | and that the input preprocessing function is also different (same as Xception). 13 | # Reference 14 | - [Rethinking the Inception Architecture for Computer Vision](http://arxiv.org/abs/1512.00567) 15 | - []() # added when paper is published on Arxiv 16 | """ 17 | from __future__ import print_function 18 | from __future__ import absolute_import 19 | 20 | from keras.models import Model 21 | from keras import layers 22 | from keras.layers import Activation 23 | from keras.layers import Dense 24 | from keras.layers import Input 25 | from keras.layers import BatchNormalization 26 | from keras.layers import Conv2D 27 | from keras.layers import MaxPooling2D 28 | from keras.layers import AveragePooling2D 29 | from keras.layers import GlobalAveragePooling2D 30 | from keras.layers import GlobalMaxPooling2D 31 | from keras.engine.topology import get_source_inputs 32 | from keras import backend as K 33 | 34 | from keras_applications.imagenet_utils import _obtain_input_shape 35 | 36 | from se_block import squeeze_excite_block 37 | 38 | WEIGHTS_PATH = '' 39 | WEIGHTS_PATH_NO_TOP = '' 40 | 41 | 42 | def conv2d_bn(x, 43 | filters, 44 | num_row, 45 | num_col, 46 | padding='same', 47 | strides=(1, 1), 48 | name=None): 49 | """Utility function to apply conv + BN. 50 | # Arguments 51 | x: input tensor. 52 | filters: filters in `Conv2D`. 53 | num_row: height of the convolution kernel. 54 | num_col: width of the convolution kernel. 55 | padding: padding mode in `Conv2D`. 56 | strides: strides in `Conv2D`. 57 | name: name of the ops; will become `name + '_conv'` 58 | for the convolution and `name + '_bn'` for the 59 | batch norm layer. 60 | # Returns 61 | Output tensor after applying `Conv2D` and `BatchNormalization`. 62 | """ 63 | if name is not None: 64 | bn_name = name + '_bn' 65 | conv_name = name + '_conv' 66 | else: 67 | bn_name = None 68 | conv_name = None 69 | if K.image_data_format() == 'channels_first': 70 | bn_axis = 1 71 | else: 72 | bn_axis = 3 73 | x = Conv2D( 74 | filters, (num_row, num_col), 75 | strides=strides, 76 | padding=padding, 77 | use_bias=False, 78 | name=conv_name)(x) 79 | x = BatchNormalization(axis=bn_axis, scale=False, name=bn_name)(x) 80 | x = Activation('relu', name=name)(x) 81 | return x 82 | 83 | 84 | def se_inception_v3(include_top=True, 85 | weights=None, 86 | input_tensor=None, 87 | input_shape=None, 88 | pooling=None, 89 | classes=1000): 90 | """Instantiates the Squeeze and Excite Inception v3 architecture. 91 | # Arguments 92 | include_top: whether to include the fully-connected 93 | layer at the top of the network. 94 | weights: one of `None` (random initialization) 95 | or "imagenet" (pre-training on ImageNet). 96 | input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) 97 | to use as image input for the model. 98 | input_shape: optional shape tuple, only to be specified 99 | if `include_top` is False (otherwise the input shape 100 | has to be `(299, 299, 3)` (with `channels_last` data format) 101 | or `(3, 299, 299)` (with `channels_first` data format). 102 | It should have exactly 3 inputs channels, 103 | and width and height should be no smaller than 139. 104 | E.g. `(150, 150, 3)` would be one valid value. 105 | pooling: Optional pooling mode for feature extraction 106 | when `include_top` is `False`. 107 | - `None` means that the output of the model will be 108 | the 4D tensor output of the 109 | last convolutional layer. 110 | - `avg` means that global average pooling 111 | will be applied to the output of the 112 | last convolutional layer, and thus 113 | the output of the model will be a 2D tensor. 114 | - `max` means that global max pooling will 115 | be applied. 116 | classes: optional number of classes to classify images 117 | into, only to be specified if `include_top` is True, and 118 | if no `weights` argument is specified. 119 | # Returns 120 | A Keras model instance. 121 | # Raises 122 | ValueError: in case of invalid argument for `weights`, 123 | or invalid input shape. 124 | """ 125 | if weights not in {'imagenet', None}: 126 | raise ValueError('The `weights` argument should be either ' 127 | '`None` (random initialization) or `imagenet` ' 128 | '(pre-training on ImageNet).') 129 | 130 | if weights == 'imagenet' and include_top and classes != 1000: 131 | raise ValueError('If using `weights` as imagenet with `include_top`' 132 | ' as true, `classes` should be 1000') 133 | 134 | # Determine proper input shape 135 | input_shape = _obtain_input_shape( 136 | input_shape, 137 | default_size=299, 138 | min_size=139, 139 | data_format=K.image_data_format(), 140 | require_flatten=include_top) 141 | 142 | if input_tensor is None: 143 | img_input = Input(shape=input_shape) 144 | else: 145 | if not K.is_keras_tensor(input_tensor): 146 | img_input = Input(tensor=input_tensor, shape=input_shape) 147 | else: 148 | img_input = input_tensor 149 | 150 | if K.image_data_format() == 'channels_first': 151 | channel_axis = 1 152 | else: 153 | channel_axis = 3 154 | 155 | x = conv2d_bn(img_input, 32, 3, 3, strides=(2, 2), padding='valid') 156 | x = conv2d_bn(x, 32, 3, 3, padding='valid') 157 | x = conv2d_bn(x, 64, 3, 3) 158 | x = MaxPooling2D((3, 3), strides=(2, 2))(x) 159 | 160 | x = conv2d_bn(x, 80, 1, 1, padding='valid') 161 | x = conv2d_bn(x, 192, 3, 3, padding='valid') 162 | x = MaxPooling2D((3, 3), strides=(2, 2))(x) 163 | 164 | # mixed 0, 1, 2: 35 x 35 x 256 165 | branch1x1 = conv2d_bn(x, 64, 1, 1) 166 | 167 | branch5x5 = conv2d_bn(x, 48, 1, 1) 168 | branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) 169 | 170 | branch3x3dbl = conv2d_bn(x, 64, 1, 1) 171 | branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) 172 | branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) 173 | 174 | branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) 175 | branch_pool = conv2d_bn(branch_pool, 32, 1, 1) 176 | x = layers.concatenate( 177 | [branch1x1, branch5x5, branch3x3dbl, branch_pool], 178 | axis=channel_axis, 179 | name='mixed0') 180 | 181 | # squeeze and excite block 182 | x = squeeze_excite_block(x) 183 | 184 | # mixed 1: 35 x 35 x 256 185 | branch1x1 = conv2d_bn(x, 64, 1, 1) 186 | 187 | branch5x5 = conv2d_bn(x, 48, 1, 1) 188 | branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) 189 | 190 | branch3x3dbl = conv2d_bn(x, 64, 1, 1) 191 | branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) 192 | branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) 193 | 194 | branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) 195 | branch_pool = conv2d_bn(branch_pool, 64, 1, 1) 196 | x = layers.concatenate( 197 | [branch1x1, branch5x5, branch3x3dbl, branch_pool], 198 | axis=channel_axis, 199 | name='mixed1') 200 | 201 | # squeeze and excite block 202 | x = squeeze_excite_block(x) 203 | 204 | # mixed 2: 35 x 35 x 256 205 | branch1x1 = conv2d_bn(x, 64, 1, 1) 206 | 207 | branch5x5 = conv2d_bn(x, 48, 1, 1) 208 | branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) 209 | 210 | branch3x3dbl = conv2d_bn(x, 64, 1, 1) 211 | branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) 212 | branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) 213 | 214 | branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) 215 | branch_pool = conv2d_bn(branch_pool, 64, 1, 1) 216 | x = layers.concatenate( 217 | [branch1x1, branch5x5, branch3x3dbl, branch_pool], 218 | axis=channel_axis, 219 | name='mixed2') 220 | 221 | # squeeze and excite block 222 | x = squeeze_excite_block(x) 223 | 224 | # mixed 3: 17 x 17 x 768 225 | branch3x3 = conv2d_bn(x, 384, 3, 3, strides=(2, 2), padding='valid') 226 | 227 | branch3x3dbl = conv2d_bn(x, 64, 1, 1) 228 | branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) 229 | branch3x3dbl = conv2d_bn( 230 | branch3x3dbl, 96, 3, 3, strides=(2, 2), padding='valid') 231 | 232 | branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x) 233 | x = layers.concatenate( 234 | [branch3x3, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed3') 235 | 236 | # squeeze and excite block 237 | x = squeeze_excite_block(x) 238 | 239 | # mixed 4: 17 x 17 x 768 240 | branch1x1 = conv2d_bn(x, 192, 1, 1) 241 | 242 | branch7x7 = conv2d_bn(x, 128, 1, 1) 243 | branch7x7 = conv2d_bn(branch7x7, 128, 1, 7) 244 | branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) 245 | 246 | branch7x7dbl = conv2d_bn(x, 128, 1, 1) 247 | branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1) 248 | branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7) 249 | branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1) 250 | branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) 251 | 252 | branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) 253 | branch_pool = conv2d_bn(branch_pool, 192, 1, 1) 254 | x = layers.concatenate( 255 | [branch1x1, branch7x7, branch7x7dbl, branch_pool], 256 | axis=channel_axis, 257 | name='mixed4') 258 | 259 | # squeeze and excite block 260 | x = squeeze_excite_block(x) 261 | 262 | # mixed 5, 6: 17 x 17 x 768 263 | for i in range(2): 264 | branch1x1 = conv2d_bn(x, 192, 1, 1) 265 | 266 | branch7x7 = conv2d_bn(x, 160, 1, 1) 267 | branch7x7 = conv2d_bn(branch7x7, 160, 1, 7) 268 | branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) 269 | 270 | branch7x7dbl = conv2d_bn(x, 160, 1, 1) 271 | branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1) 272 | branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7) 273 | branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1) 274 | branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) 275 | 276 | branch_pool = AveragePooling2D( 277 | (3, 3), strides=(1, 1), padding='same')(x) 278 | branch_pool = conv2d_bn(branch_pool, 192, 1, 1) 279 | x = layers.concatenate( 280 | [branch1x1, branch7x7, branch7x7dbl, branch_pool], 281 | axis=channel_axis, 282 | name='mixed' + str(5 + i)) 283 | 284 | # squeeze and excite block 285 | x = squeeze_excite_block(x) 286 | 287 | # mixed 7: 17 x 17 x 768 288 | branch1x1 = conv2d_bn(x, 192, 1, 1) 289 | 290 | branch7x7 = conv2d_bn(x, 192, 1, 1) 291 | branch7x7 = conv2d_bn(branch7x7, 192, 1, 7) 292 | branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) 293 | 294 | branch7x7dbl = conv2d_bn(x, 192, 1, 1) 295 | branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1) 296 | branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) 297 | branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1) 298 | branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) 299 | 300 | branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) 301 | branch_pool = conv2d_bn(branch_pool, 192, 1, 1) 302 | x = layers.concatenate( 303 | [branch1x1, branch7x7, branch7x7dbl, branch_pool], 304 | axis=channel_axis, 305 | name='mixed7') 306 | 307 | # squeeze and excite block 308 | x = squeeze_excite_block(x) 309 | 310 | # mixed 8: 8 x 8 x 1280 311 | branch3x3 = conv2d_bn(x, 192, 1, 1) 312 | branch3x3 = conv2d_bn(branch3x3, 320, 3, 3, 313 | strides=(2, 2), padding='valid') 314 | 315 | branch7x7x3 = conv2d_bn(x, 192, 1, 1) 316 | branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7) 317 | branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1) 318 | branch7x7x3 = conv2d_bn( 319 | branch7x7x3, 192, 3, 3, strides=(2, 2), padding='valid') 320 | 321 | branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x) 322 | x = layers.concatenate( 323 | [branch3x3, branch7x7x3, branch_pool], axis=channel_axis, name='mixed8') 324 | 325 | # squeeze and excite block 326 | x = squeeze_excite_block(x) 327 | 328 | # mixed 9: 8 x 8 x 2048 329 | for i in range(2): 330 | branch1x1 = conv2d_bn(x, 320, 1, 1) 331 | 332 | branch3x3 = conv2d_bn(x, 384, 1, 1) 333 | branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3) 334 | branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1) 335 | branch3x3 = layers.concatenate( 336 | [branch3x3_1, branch3x3_2], axis=channel_axis, name='mixed9_' + str(i)) 337 | 338 | branch3x3dbl = conv2d_bn(x, 448, 1, 1) 339 | branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3) 340 | branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3) 341 | branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1) 342 | branch3x3dbl = layers.concatenate( 343 | [branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis) 344 | 345 | branch_pool = AveragePooling2D( 346 | (3, 3), strides=(1, 1), padding='same')(x) 347 | branch_pool = conv2d_bn(branch_pool, 192, 1, 1) 348 | x = layers.concatenate( 349 | [branch1x1, branch3x3, branch3x3dbl, branch_pool], 350 | axis=channel_axis, 351 | name='mixed' + str(9 + i)) 352 | 353 | # squeeze and excite block 354 | x = squeeze_excite_block(x) 355 | 356 | if include_top: 357 | # Classification block 358 | x = GlobalAveragePooling2D(name='avg_pool')(x) 359 | x = Dense(classes, activation='softmax', name='predictions')(x) 360 | else: 361 | if pooling == 'avg': 362 | x = GlobalAveragePooling2D()(x) 363 | elif pooling == 'max': 364 | x = GlobalMaxPooling2D()(x) 365 | 366 | # Ensure that the model takes into account 367 | # any potential predecessors of `input_tensor`. 368 | if input_tensor is not None: 369 | inputs = get_source_inputs(input_tensor) 370 | else: 371 | inputs = img_input 372 | # Create model. 373 | model = Model(inputs, x, name='SE_Inception_V3') 374 | 375 | return model 376 | 377 | 378 | def preprocess_input(x): 379 | x /= 255. 380 | x -= 0.5 381 | x *= 2. 382 | return x 383 | 384 | if __name__ == "__main__": 385 | se_inception_v3 = se_inception_v3() 386 | se_inception_v3.summary() 387 | -------------------------------------------------------------------------------- /test0.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 16 06:26:57 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | from keras.models import Model 10 | from keras.layers import Input, Activation, merge, Dense, Flatten, Reshape, Permute 11 | from keras.layers.convolutional import Convolution2D, MaxPooling2D, Conv2DTranspose, Cropping2D 12 | from keras.layers.normalization import BatchNormalization 13 | 14 | 15 | 16 | def create_model(): 17 | 18 | FCN_CLASSES = 21 19 | 20 | #(samples, channels, rows, cols) 21 | input_img = Input(shape=(3, 224, 224)) 22 | #(3*224*224) 23 | x = Convolution2D(64, 3, 3, activation='relu',padding='same')(input_img) 24 | x = Convolution2D(64, 3, 3, activation='relu',padding='same')(x) 25 | x = MaxPooling2D((2, 2), strides=(2, 2))(x) 26 | #(64*112*112) 27 | x = Convolution2D(128, 3, 3, activation='relu',padding='same')(x) 28 | x = Convolution2D(128, 3, 3, activation='relu',padding='same')(x) 29 | x = MaxPooling2D((2, 2), strides=(2, 2))(x) 30 | #(128*56*56) 31 | x = Convolution2D(256, 3, 3, activation='relu',padding='same')(x) 32 | x = Convolution2D(256, 3, 3, activation='relu',padding='same')(x) 33 | x = MaxPooling2D((2, 2), strides=(2, 2))(x) 34 | #(256*56*56) 35 | 36 | #split layer 37 | p3 = x 38 | p3 = Convolution2D(FCN_CLASSES, 1, 1,activation='relu')(p3) 39 | #(21*28*28) 40 | 41 | x = Convolution2D(512, 3, 3, activation='relu',padding='same')(x) 42 | x = Convolution2D(512, 3, 3, activation='relu',padding='same')(x) 43 | x = MaxPooling2D((2, 2), strides=(2, 2))(x) 44 | #(512*14*14) 45 | 46 | #split layer 47 | p4 = x 48 | p4 = Convolution2D(FCN_CLASSES, 1, 1, activation='relu')(p4) 49 | p4 = Conv2DTranspose(FCN_CLASSES, 4, 4, 50 | output_shape=(None, FCN_CLASSES, 30, 30), 51 | subsample=(2, 2), 52 | padding='valid')(p4) 53 | p4 = Cropping2D(cropping=((1, 1), (1, 1)))(p4) 54 | 55 | #(21*28*28) 56 | 57 | x = Convolution2D(512, 3, 3, activation='relu',padding='same')(x) 58 | x = Convolution2D(512, 3, 3, activation='relu',padding='same')(x) 59 | x = MaxPooling2D((2, 2), strides=(2, 2))(x) 60 | #(512*7*7) 61 | 62 | p5 = x 63 | p5 = Convolution2D(FCN_CLASSES, 1, 1, activation='relu')(p5) 64 | p5 = Conv2DTranspose(FCN_CLASSES, 8, 8, 65 | output_shape=(None, FCN_CLASSES, 32, 32), 66 | subsample=(4, 4), 67 | padding='valid')(p5) 68 | p5 = Cropping2D(cropping=((2, 2), (2, 2)))(p5) 69 | #(21*28*28) 70 | 71 | # merge scores 72 | merged = merge([p3, p4, p5], mode='sum') 73 | x = Conv2DTranspose(FCN_CLASSES, 16, 16, 74 | output_shape=(None, FCN_CLASSES, 232, 232), 75 | subsample=(8, 8), 76 | padding='valid')(merged) 77 | x = Cropping2D(cropping=((4, 4), (4, 4)))(x) 78 | x = Reshape((FCN_CLASSES,224*224))(x) 79 | x = Permute((2,1))(x) 80 | out = Activation("softmax")(x) 81 | #(21,224,224) 82 | model = Model(input_img, out) 83 | return model -------------------------------------------------------------------------------- /train_test_splitter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Thu Oct 25 07:04:49 2018 5 | 6 | Split CUB2002011 to test/train folders 7 | @author: n-kamiya 8 | """ 9 | import os 10 | import pandas as pd 11 | from database import cub2002011_db 12 | 13 | #%%create database 14 | con , cursor = cub2002011_db.create_cub_db() 15 | #%%load data frames 16 | df = pd.read_sql("select * from CUB2002011", con) 17 | df_train_path = pd.read_sql("select * from CUB2002011 where train = 1", con) 18 | df_test_path = pd.read_sql("select * from CUB2002011 where train = 0", con) 19 | #%% 20 | 21 | train_path = [] 22 | test_path = [] 23 | 24 | train_path=(df_train_path.iloc[:,3].values) 25 | test_path=(df_test_path.iloc[:,3].values) 26 | 27 | 28 | train_f = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 29 | test_f = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 30 | #%% remove files 31 | 32 | #for file in train_path: 33 | # os.remove(test_f+file) 34 | #for file in test_path: 35 | # os.remove(train_f+file) -------------------------------------------------------------------------------- /vgg16_model_withou_MAMC.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Mon Oct 29 07:21:15 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | #!/usr/bin/env python3 10 | # -*- coding: utf-8 -*- 11 | """ 12 | Created on Tue Oct 23 08:45:20 2018 13 | 14 | @author: n-kamiya 15 | """ 16 | 17 | 18 | from keras.applications.resnet50 import ResNet50 19 | from keras.applications.resnet50 import preprocess_input 20 | from keras.applications.vgg16 import preprocess_input 21 | from keras.applications.vgg16 import VGG16 22 | from keras.models import Sequential, Model 23 | from keras.layers import Input, Dense, Dropout, Activation, Flatten 24 | from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, AveragePooling2D 25 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 26 | from keras.optimizers import SGD 27 | from keras.utils.np_utils import to_categorical 28 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array 29 | from keras.regularizers import l2 30 | import matplotlib.image as mpimg 31 | import numpy as np 32 | import keras.backend as K 33 | import pathlib 34 | from PIL import Image 35 | import cv2 36 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 37 | import matplotlib.pyplot as plt 38 | from keras.utils.vis_utils import plot_model 39 | import pandas as pd 40 | #%% 41 | 42 | K.clear_session() 43 | BATCH_SIZE = 32 44 | test_nb = 5794 45 | train_nb = 5994 46 | num_classes = 200 47 | img_size= 448 48 | 49 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 50 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 51 | #%% create data generator 52 | train_datagen = ImageDataGenerator(rescale = 1./img_size, 53 | zoom_range=[0.5,1], 54 | rotation_range=90, 55 | horizontal_flip=True, 56 | vertical_flip=True) 57 | test_datagen = ImageDataGenerator(rescale = 1./img_size) 58 | 59 | train_generator = train_datagen.flow_from_directory( 60 | train_path, 61 | target_size=(img_size, img_size), 62 | batch_size=BATCH_SIZE, 63 | seed = 13) 64 | 65 | validation_generator = test_datagen.flow_from_directory( 66 | test_path, 67 | target_size=(img_size, img_size), 68 | batch_size=BATCH_SIZE, 69 | seed = 13) 70 | #%% finetuning vgg16 71 | 72 | input_tensor = Input(shape=(img_size, img_size, 3)) 73 | base_model_vgg16 = VGG16(weights = "imagenet", include_top=False, input_tensor=input_tensor) 74 | 75 | # change only the output layer 76 | 77 | top_model = Sequential() 78 | top_model.add(Flatten(input_shape=base_model_vgg16.output_shape[1:])) 79 | top_model.add(Dense(num_classes,activation='relu')) 80 | top_model.add(Dropout(0.5)) 81 | top_model.add(Dense(num_classes, activation='softmax')) 82 | 83 | model_vgg16 = Model(input=base_model_vgg16.input, output=top_model(base_model_vgg16.output)) 84 | 85 | opt = SGD(lr=0.001, momentum=0.9) 86 | # 87 | for layer in base_model_vgg16.layers: 88 | layer.trainable = False 89 | 90 | model_vgg16.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 91 | 92 | #%% 93 | plot_model(model_vgg16, to_file="model.png", show_shapes=True) 94 | 95 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 96 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_alpha.best_loss.hdf5', verbose=1, save_best_only=True) 97 | 98 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, 99 | patience=5, min_lr=0.000001) 100 | #%% fit_generator 101 | 102 | history = model_vgg16.fit_generator(train_generator, 103 | steps_per_epoch=train_nb/BATCH_SIZE, 104 | epochs=10, 105 | validation_data=validation_generator, 106 | validation_steps=64, 107 | verbose=1, 108 | callbacks=[reduce_lr, checkpointer]) 109 | #%%plot history 110 | 111 | import datetime 112 | now = datetime.datetime.now() 113 | 114 | plt.plot(history.history['acc']) 115 | plt.plot(history.history['val_acc']) 116 | plt.title('model_without_MAMC accuracy') 117 | plt.ylabel('accuracy') 118 | plt.xlabel('epoch') 119 | plt.legend(['train', 'test'], loc='upper left') 120 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_{0:%d%m}-{0:%H%M%S}.png".format(now)) 121 | plt.show() 122 | 123 | #loss 124 | plt.plot(history.history['loss']) 125 | plt.plot(history.history['val_loss']) 126 | plt.title('model_without_MAMC loss') 127 | plt.ylabel('loss') 128 | plt.xlabel('epoch') 129 | plt.legend(['train', 'test'], loc='upper left') 130 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_{0:%d%m}-{0:%H%M%S}.png".format(now)) 131 | plt.show() 132 | 133 | -------------------------------------------------------------------------------- /vgg19_model_without_MAMC.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Oct 30 02:52:05 2018 5 | 6 | @author: n-kamiya 7 | """ 8 | 9 | #!/usr/bin/env python3 10 | # -*- coding: utf-8 -*- 11 | """ 12 | Created on Mon Oct 29 07:21:15 2018 13 | 14 | @author: n-kamiya 15 | """ 16 | 17 | #!/usr/bin/env python3 18 | # -*- coding: utf-8 -*- 19 | """ 20 | Created on Tue Oct 23 08:45:20 2018 21 | 22 | @author: n-kamiya 23 | """ 24 | 25 | from keras.applications.vgg19 import preprocess_input 26 | from keras.applications.vgg19 import VGG19 27 | from keras.models import Sequential, Model 28 | from keras.layers import Input, Dense, Dropout, Activation, Flatten 29 | from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, AveragePooling2D 30 | from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau 31 | from keras.optimizers import SGD 32 | from keras.utils.np_utils import to_categorical 33 | from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array 34 | from keras.regularizers import l2 35 | import matplotlib.image as mpimg 36 | import numpy as np 37 | import keras.backend as K 38 | import pathlib 39 | from PIL import Image 40 | import cv2 41 | from sklearn.preprocessing import OneHotEncoder, LabelEncoder 42 | import matplotlib.pyplot as plt 43 | from keras.utils.vis_utils import plot_model 44 | import pandas as pd 45 | #%% 46 | 47 | K.clear_session() 48 | BATCH_SIZE = 16 49 | test_nb = 5794 50 | train_nb = 5994 51 | num_classes = 200 52 | img_size= 448 53 | 54 | train_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/train/" 55 | test_path = "/home/n-kamiya/datasets/CUB2002011/CUB_200_2011/test/" 56 | #%% create data generator 57 | train_datagen = ImageDataGenerator(rescale = 1./img_size, 58 | zoom_range=[0.5,1], 59 | rotation_range=90, 60 | horizontal_flip=True, 61 | vertical_flip=True) 62 | test_datagen = ImageDataGenerator(rescale = 1./img_size) 63 | 64 | train_generator = train_datagen.flow_from_directory( 65 | train_path, 66 | target_size=(img_size, img_size), 67 | batch_size=BATCH_SIZE, 68 | seed = 13) 69 | 70 | validation_generator = test_datagen.flow_from_directory( 71 | test_path, 72 | target_size=(img_size, img_size), 73 | batch_size=BATCH_SIZE, 74 | seed = 13) 75 | #%% finetuning vgg19 76 | 77 | input_tensor = Input(shape=(img_size, img_size, 3)) 78 | base_model_vgg19 = VGG19(weights = "imagenet", include_top=False, input_tensor=input_tensor) 79 | 80 | # change only the output layer 81 | 82 | top_model = Sequential() 83 | top_model.add(GlobalAveragePooling2D(input_shape=base_model_vgg19.output_shape[1:])) 84 | top_model.add(Dense(512,activation='relu')) 85 | top_model.add(Dropout(0.5)) 86 | top_model.add(Dense(num_classes, activation='softmax')) 87 | 88 | model_vgg19 = Model(input=base_model_vgg19.input, output=top_model(base_model_vgg19.output)) 89 | 90 | opt = SGD(lr=0.0001, momentum=0.9) 91 | # 92 | for layer in base_model_vgg19.layers[:-2]: 93 | layer.trainable = False 94 | 95 | 96 | model_vgg19.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) 97 | 98 | #%% 99 | plot_model(model_vgg19, to_file="model.png", show_shapes=True) 100 | 101 | #%% implement checkpointer and reduce_lr (to prevent overfitting) 102 | checkpointer = ModelCheckpoint(filepath='/home/n-kamiya/models/model_without_MAMC/model_alpha.best_loss.hdf5', verbose=1, save_best_only=True) 103 | 104 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, 105 | patience=5, min_lr=0.000001) 106 | #%% fit_generator 107 | 108 | history = model_vgg19.fit_generator(train_generator, 109 | steps_per_epoch=train_nb/BATCH_SIZE, 110 | epochs=10, 111 | validation_data=validation_generator, 112 | validation_steps=64, 113 | verbose=1, 114 | callbacks=[reduce_lr, checkpointer], 115 | class_weight="auto") 116 | #%%plot history 117 | 118 | import datetime 119 | now = datetime.datetime.now() 120 | 121 | plt.plot(history.history['acc']) 122 | plt.plot(history.history['val_acc']) 123 | plt.title('model_without_MAMC accuracy') 124 | plt.ylabel('accuracy') 125 | plt.xlabel('epoch') 126 | plt.legend(['train', 'test'], loc='upper left') 127 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/history_{0:%d%m}-{0:%H%M%S}.png".format(now)) 128 | plt.show() 129 | 130 | #loss 131 | plt.plot(history.history['loss']) 132 | plt.plot(history.history['val_loss']) 133 | plt.title('model_without_MAMC loss') 134 | plt.ylabel('loss') 135 | plt.xlabel('epoch') 136 | plt.legend(['train', 'test'], loc='upper left') 137 | plt.savefig("/home/n-kamiya/models/model_without_MAMC/loss_{0:%d%m}-{0:%H%M%S}.png".format(now)) 138 | plt.show() 139 | 140 | --------------------------------------------------------------------------------