├── README.md ├── cam.py ├── data.py ├── examples ├── building.jpg ├── debate.jpg ├── dog.jpg ├── mona_lisa.jpg ├── soccer.jpg └── traffic.jpg └── model.py /README.md: -------------------------------------------------------------------------------- 1 | ## Keras implementation of class activation mapping 2 | 3 | Paper / project page: http://cnnlocalization.csail.mit.edu 4 | 5 | Paper authors' code with Caffe / matcaffe interface: https://github.com/metalbubble/CAM 6 | 7 | 8 | Blog post on this repository: http://jacobcv.blogspot.com/2016/08/class-activation-maps-in-keras.html 9 | 10 | Checkpoint with person/not person weights: https://drive.google.com/open?id=0B1l5JSkBbENBdk95ZW1DOUhqQUE 11 | 12 | ![enter image description here](https://raw.githubusercontent.com/jacobgil/keras-cam/master/examples/mona_lisa.jpg) 13 | 14 | 15 | This project implements class activation maps with Keras. 16 | 17 | Class activation maps are a simple technique to get the image regions relevant to a certain class. 18 | 19 | This was fined tuned on VGG16 with images from here: 20 | http://pascal.inrialpes.fr/data/human 21 | 22 | The model in model.py is a two category classifier, used to classify person / not a person. 23 | 24 | python cam.py --model_path cam_checkpoint.hdf5 --image_path=image.jpg 25 | 26 | usage: cam.py [-h] [--train TRAIN] [--image_path IMAGE_PATH] 27 | [--output_path OUTPUT_PATH] [--model_path MODEL_PATH] 28 | [--dataset_path DATASET_PATH] 29 | 30 | optional arguments: 31 | -h, --help show this help message and exit 32 | --train TRAIN Train the network or visualize a CAM 33 | --image_path IMAGE_PATH 34 | Path of an image to run the network on 35 | --output_path OUTPUT_PATH 36 | Path of an image to run the network on 37 | --model_path MODEL_PATH 38 | Path of the trained model 39 | --dataset_path DATASET_PATH 40 | Path to image dataset. Should have pos/neg folders, 41 | like in the inria person dataset. 42 | http://pascal.inrialpes.fr/data/human/ 43 | 44 | -------------------------------------------------------------------------------- /cam.py: -------------------------------------------------------------------------------- 1 | from keras.models import * 2 | from keras.callbacks import * 3 | import keras.backend as K 4 | from model import * 5 | from data import * 6 | import cv2 7 | import argparse 8 | 9 | def train(dataset_path): 10 | model = get_model() 11 | X, y = load_inria_person(dataset_path) 12 | print "Training.." 13 | checkpoint_path="weights.{epoch:02d}-{val_loss:.2f}.hdf5" 14 | checkpoint = ModelCheckpoint(checkpoint_path, monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=False, mode='auto') 15 | model.fit(X, y, nb_epoch=40, batch_size=32, validation_split=0.2, verbose=1, callbacks=[checkpoint]) 16 | 17 | def visualize_class_activation_map(model_path, img_path, output_path): 18 | model = load_model(model_path) 19 | original_img = cv2.imread(img_path, 1) 20 | width, height, _ = original_img.shape 21 | 22 | #Reshape to the network input shape (3, w, h). 23 | img = np.array([np.transpose(np.float32(original_img), (2, 0, 1))]) 24 | 25 | #Get the 512 input weights to the softmax. 26 | class_weights = model.layers[-1].get_weights()[0] 27 | final_conv_layer = get_output_layer(model, "conv5_3") 28 | get_output = K.function([model.layers[0].input], [final_conv_layer.output, model.layers[-1].output]) 29 | [conv_outputs, predictions] = get_output([img]) 30 | conv_outputs = conv_outputs[0, :, :, :] 31 | 32 | #Create the class activation map. 33 | cam = np.zeros(dtype = np.float32, shape = conv_outputs.shape[1:3]) 34 | for i, w in enumerate(class_weights[:, 1]): 35 | cam += w * conv_outputs[i, :, :] 36 | print "predictions", predictions 37 | cam /= np.max(cam) 38 | cam = cv2.resize(cam, (height, width)) 39 | heatmap = cv2.applyColorMap(np.uint8(255*cam), cv2.COLORMAP_JET) 40 | heatmap[np.where(cam < 0.2)] = 0 41 | img = heatmap*0.5 + original_img 42 | cv2.imwrite(output_path, img) 43 | 44 | def get_args(): 45 | parser = argparse.ArgumentParser() 46 | parser.add_argument("--train", type = bool, default = False, help = 'Train the network or visualize a CAM') 47 | parser.add_argument("--image_path", type = str, help = "Path of an image to run the network on") 48 | parser.add_argument("--output_path", type = str, default = "heatmap.jpg", help = "Path of an image to run the network on") 49 | parser.add_argument("--model_path", type = str, help = "Path of the trained model") 50 | parser.add_argument("--dataset_path", type = str, help = \ 51 | 'Path to image dataset. Should have pos/neg folders, like in the inria person dataset. \ 52 | http://pascal.inrialpes.fr/data/human/') 53 | args = parser.parse_args() 54 | return args 55 | 56 | if __name__ == '__main__': 57 | args = get_args() 58 | if args.train: 59 | train(args.dataset_path) 60 | else: 61 | visualize_class_activation_map(args.model_path, args.image_path, args.output_path) 62 | -------------------------------------------------------------------------------- /data.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import glob 3 | import os 4 | import numpy as np 5 | from keras.utils.np_utils import to_categorical 6 | 7 | def load_inria_person(path): 8 | pos_path = os.path.join(path, "pos") 9 | neg_path = os.path.join(path, "/neg") 10 | pos_images = [cv2.resize(cv2.imread(x), (64, 128)) for x in glob.glob(pos_path + "/*.png")] 11 | pos_images = [np.transpose(img, (2, 0, 1)) for img in pos_images] 12 | neg_images = [cv2.resize(cv2.imread(x), (64, 128)) for x in glob.glob(neg_path + "/*.png")] 13 | neg_images = [np.transpose(img, (2, 0, 1)) for img in neg_images] 14 | y = [1] * len(pos_images) + [0] * len(neg_images) 15 | y = to_categorical(y, 2) 16 | X = np.float32(pos_images + neg_images) 17 | 18 | return X, y 19 | -------------------------------------------------------------------------------- /examples/building.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobgil/keras-cam/2b7ada2c5c819808a174444e82622315a07fa11e/examples/building.jpg -------------------------------------------------------------------------------- /examples/debate.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobgil/keras-cam/2b7ada2c5c819808a174444e82622315a07fa11e/examples/debate.jpg -------------------------------------------------------------------------------- /examples/dog.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobgil/keras-cam/2b7ada2c5c819808a174444e82622315a07fa11e/examples/dog.jpg -------------------------------------------------------------------------------- /examples/mona_lisa.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobgil/keras-cam/2b7ada2c5c819808a174444e82622315a07fa11e/examples/mona_lisa.jpg -------------------------------------------------------------------------------- /examples/soccer.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobgil/keras-cam/2b7ada2c5c819808a174444e82622315a07fa11e/examples/soccer.jpg -------------------------------------------------------------------------------- /examples/traffic.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobgil/keras-cam/2b7ada2c5c819808a174444e82622315a07fa11e/examples/traffic.jpg -------------------------------------------------------------------------------- /model.py: -------------------------------------------------------------------------------- 1 | from keras.models import Sequential 2 | from keras.layers import Convolution2D, ZeroPadding2D, MaxPooling2D 3 | from keras.layers.core import Flatten, Dense, Dropout, Lambda 4 | from keras import backend as K 5 | import h5py 6 | from keras.optimizers import SGD 7 | 8 | def global_average_pooling(x): 9 | return K.mean(x, axis = (2, 3)) 10 | 11 | def global_average_pooling_shape(input_shape): 12 | return input_shape[0:2] 13 | 14 | def VGG16_convolutions(): 15 | model = Sequential() 16 | model.add(ZeroPadding2D((1,1),input_shape=(3,None,None))) 17 | model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1')) 18 | model.add(ZeroPadding2D((1, 1))) 19 | model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_2')) 20 | model.add(MaxPooling2D((2, 2), strides=(2, 2))) 21 | 22 | model.add(ZeroPadding2D((1, 1))) 23 | model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1')) 24 | model.add(ZeroPadding2D((1, 1))) 25 | model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_2')) 26 | model.add(MaxPooling2D((2, 2), strides=(2, 2))) 27 | 28 | model.add(ZeroPadding2D((1, 1))) 29 | model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1')) 30 | model.add(ZeroPadding2D((1, 1))) 31 | model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_2')) 32 | model.add(ZeroPadding2D((1, 1))) 33 | model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_3')) 34 | model.add(MaxPooling2D((2, 2), strides=(2, 2))) 35 | 36 | model.add(ZeroPadding2D((1, 1))) 37 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1')) 38 | model.add(ZeroPadding2D((1, 1))) 39 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2')) 40 | model.add(ZeroPadding2D((1, 1))) 41 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_3')) 42 | model.add(MaxPooling2D((2, 2), strides=(2, 2))) 43 | 44 | model.add(ZeroPadding2D((1, 1))) 45 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1')) 46 | model.add(ZeroPadding2D((1, 1))) 47 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_2')) 48 | model.add(ZeroPadding2D((1, 1))) 49 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_3')) 50 | return model 51 | 52 | def get_model(): 53 | model = VGG16_convolutions() 54 | 55 | model = load_model_weights(model, "vgg16_weights.h5") 56 | 57 | model.add(Lambda(global_average_pooling, 58 | output_shape=global_average_pooling_shape)) 59 | model.add(Dense(2, activation = 'softmax', init='uniform')) 60 | sgd = SGD(lr=0.01, decay=1e-6, momentum=0.5, nesterov=True) 61 | model.compile(loss = 'categorical_crossentropy', optimizer = sgd, metrics=['accuracy']) 62 | return model 63 | 64 | def load_model_weights(model, weights_path): 65 | print 'Loading model.' 66 | f = h5py.File(weights_path) 67 | for k in range(f.attrs['nb_layers']): 68 | if k >= len(model.layers): 69 | # we don't look at the last (fully-connected) layers in the savefile 70 | break 71 | g = f['layer_{}'.format(k)] 72 | weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])] 73 | model.layers[k].set_weights(weights) 74 | model.layers[k].trainable = False 75 | f.close() 76 | print 'Model loaded.' 77 | return model 78 | 79 | def get_output_layer(model, layer_name): 80 | # get the symbolic outputs of each "key" layer (we gave them unique names). 81 | layer_dict = dict([(layer.name, layer) for layer in model.layers]) 82 | layer = layer_dict[layer_name] 83 | return layer --------------------------------------------------------------------------------