├── README.md ├── distracted_driver_detection_with_pretrained_VGG16_deep_CNN.py ├── distracted_driver_detection_with_small_CNN.py ├── img_7574.jpg └── vgg16_CNN.png /README.md: -------------------------------------------------------------------------------- 1 | # Distracted-Driver-Detection-with-Deep-Learning 2 | This project aims to detect the dangerous status of driving based on the images captured by the dashboard camera using deep learning. 3 | 4 | # Dataset 5 | 6 | The dataset is obtained from 7 | 8 | https://www.kaggle.com/c/state-farm-distracted-driver-detection/data 9 | 10 | The dataset contains 22,424 images which belongs to one of the 10 classes given below: 11 | 12 | c0: safe driving 13 | 14 | c1: texting - right 15 | 16 | c2: talking on the phone - right 17 | 18 | c3: texting - left 19 | 20 | c4: talking on the phone - left 21 | 22 | c5: operating the radio 23 | 24 | c6: drinking 25 | 26 | c7: reaching behind 27 | 28 | c8: hair and makeup 29 | 30 | c9: talking to passenger 31 | 32 | We split the data into two sets: training set containing 20,924 images, and validation set containing 1500 images (e.g., 150 images for each class). 33 | 34 | # Method 1: train a small Convolutional Neural Network (CNN) from the scatch 35 | 36 | Method 1 is implemented in "distracted_driver_detection_with_small_CNN.py". The small CNN consists of 3 convolutional layers with filter size of 3x3, each of which is followed by a max-pooling layer with pool isize of 2x2, and 2 fully-connected dense layers. 37 | 38 | 39 | # Method 2: train a two-layer dense neural network on top of a pre-trained VGG16 deep CNN 40 | 41 | VGG16 is a 16-layer CNN used by the VGG team in the ILSVRC-2014 competition. The VGG16 network structure can be seen in "vgg16_CNN.png". The weights of pre-trained VGG16 CNN can be found at: 42 | 43 | https://gist.github.com/baraldilorenzo/07d7802847aaad0a35d3 44 | 45 | Method 2 is implemented in "distracted_driver_detection_with_pretrained_VGG16_deep_CNN.py". The model consists of two parts: the lower part is a pre-trained VGG network with fozen weights, and the upper part is a two-layer dense network. The model is trained using the dataset. 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /distracted_driver_detection_with_pretrained_VGG16_deep_CNN.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This program is used to recognize the driver's status (one of the 10 statuses) based on the image using pre-trained VGG16 3 | deep convolutional neural network (CNN). 4 | 5 | This program is modified from the blog post: 6 | "Building powerful image classification models using very little data" from blog.keras.io. 7 | 8 | This program do fine tunning for a modified VGG16 net, which consists of two parts: 9 | the lower model: layer 0-layer24 of the original VGG16 net (frozen the first 4 blocks, train the weights of the 5-th block 10 | with our dataset) 11 | the upper model: newly added two layer dense net (train the weights using our dataset) 12 | ''' 13 | 14 | import os 15 | #The h5py package is a Pythonic interface to the HDF5 binary data format 16 | import h5py 17 | import numpy as np 18 | from keras.preprocessing.image import ImageDataGenerator 19 | from keras import optimizers 20 | from keras.models import Sequential 21 | from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D 22 | from keras.layers import Activation, Dropout, Flatten, Dense 23 | 24 | ''' path to the model weights file in HDF5 binary data format 25 | The vgg16 weights can be downloaded from the link below: 26 | https://drive.google.com/file/d/0Bz7KyqmuGsilT0J5dmRCM0ROVHc/view 27 | ''' 28 | weights_path = 'vgg16_weights.h5' 29 | 30 | # dimensions of the images 31 | img_width, img_height = 150, 150 32 | 33 | # the path to the training data 34 | train_data_dir = 'data/train' 35 | # the path to the validation data 36 | validation_data_dir = 'data/validation' 37 | 38 | # the number of training samples. We have 20924 training images, but actually we can set the 39 | # number of training samples can be augmented to much more, for example 2*20924 40 | nb_train_samples = 20924 41 | 42 | # We actually have 1500 validation samples, which can be augmented to much more 43 | nb_validation_samples = 1500 44 | 45 | # number of epoches for training 46 | nb_epoch = 10 47 | 48 | # build the VGG16 model 49 | model = Sequential() 50 | model.add(ZeroPadding2D((1, 1), input_shape=(3, img_width, img_height))) 51 | 52 | model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1')) 53 | model.add(ZeroPadding2D((1, 1))) 54 | model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_2')) 55 | model.add(MaxPooling2D((2, 2), strides=(2, 2))) 56 | 57 | model.add(ZeroPadding2D((1, 1))) 58 | model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1')) 59 | model.add(ZeroPadding2D((1, 1))) 60 | model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_2')) 61 | model.add(MaxPooling2D((2, 2), strides=(2, 2))) 62 | 63 | model.add(ZeroPadding2D((1, 1))) 64 | model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1')) 65 | model.add(ZeroPadding2D((1, 1))) 66 | model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_2')) 67 | model.add(ZeroPadding2D((1, 1))) 68 | model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_3')) 69 | model.add(MaxPooling2D((2, 2), strides=(2, 2))) 70 | 71 | model.add(ZeroPadding2D((1, 1))) 72 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1')) 73 | model.add(ZeroPadding2D((1, 1))) 74 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2')) 75 | model.add(ZeroPadding2D((1, 1))) 76 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_3')) 77 | model.add(MaxPooling2D((2, 2), strides=(2, 2))) 78 | 79 | model.add(ZeroPadding2D((1, 1))) 80 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1')) 81 | model.add(ZeroPadding2D((1, 1))) 82 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_2')) 83 | model.add(ZeroPadding2D((1, 1))) 84 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_3')) 85 | model.add(MaxPooling2D((2, 2), strides=(2, 2))) 86 | 87 | ''' 88 | # load the weights of the VGG16 networks (trained on ImageNet, won the ILSVRC competition in 2014) 89 | # note: when there is a complete match between your model definition 90 | # and your weight savefile, you can simply call model.load_weights(filename) 91 | ''' 92 | # load the weights for each layer 93 | assert os.path.exists(weights_path), 'Model weights not found (see "weights_path" variable in script).' 94 | f = h5py.File(weights_path) 95 | for k in range(f.attrs['nb_layers']): 96 | if k >= len(model.layers): 97 | # we don't look at the last (fully-connected) layers in the savefile 98 | break 99 | g = f['layer_{}'.format(k)] 100 | weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])] 101 | # set the weights to layer-k 102 | model.layers[k].set_weights(weights) 103 | f.close() 104 | print('VGG16 model weights have been successfully loaded.') 105 | 106 | # build a MLP classifier model to put on top of the VGG16 model 107 | top_model = Sequential() 108 | # flateen the output of VGG16 model to 2D Numpy matrix (n*D) 109 | top_model.add(Flatten(input_shape=model.output_shape[1:])) 110 | # hidden layer of 256 neurons 111 | top_model.add(Dense(256, activation='relu')) 112 | # add dropout for the dense layer 113 | top_model.add(Dropout(0.5)) 114 | # the output layer: we have 10 claases 115 | top_model.add(Dense(10, activation='softmax')) 116 | 117 | # connect the two models onto the VGG16 net 118 | model.add(top_model) 119 | 120 | # set the first 25 layers (up to the last conv block) of VGFG16 net to non-trainable (weights will not be updated) 121 | for layer in model.layers[:25]: 122 | layer.trainable=False 123 | 124 | # compile the model 125 | model.compile(loss = 'categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']) 126 | 127 | # augmentation configuration for training data 128 | train_datagen = ImageDataGenerator(rescale=1.0/255, shear_range=0.2, zoom_range=0.2, horizontal_flip=True) 129 | # augmentation configuration for validation data (actually we did no augmentation to teh validation images) 130 | test_datagen = ImageDataGenerator(rescale=1.0/255) 131 | 132 | # training data generator from folder 133 | train_generator = train_datagen.flow_from_directory(train_data_dir, target_size=(img_height, img_width), 134 | batch_size=32, class_mode='categorical') 135 | 136 | # validation data generator from folder 137 | validation_generator = train_datagen.flow_from_directory(validation_data_dir, target_size=(img_height, img_width), 138 | batch_size=32, class_mode='categorical') 139 | 140 | # fit the model 141 | model.fit_generator(train_generator, samples_per_epoch=nb_train_samples, nb_epoch=nb_epoch, 142 | validation_data=validation_generator, nb_val_samples=nb_validation_samples) 143 | 144 | # save the model weights 145 | # model.save_weights('VGG16_and_MLP_model.h5') 146 | 147 | -------------------------------------------------------------------------------- /distracted_driver_detection_with_small_CNN.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This program is used to detect the driver's status (10 statuses) by using a small convolutional neural network, which 3 | is trained fram scatch using the training images. 4 | ''' 5 | 6 | from keras.preprocessing.image import ImageDataGenerator 7 | from keras.models import Sequential 8 | from keras.layers import Convolution2D, MaxPooling2D 9 | from keras.layers import Activation, Dropout, Flatten, Dense 10 | 11 | # training s small convnet from scatch 12 | # convnet: a simple stack of 3 convolution layer with ReLU activation and followed by a max-pooling layers 13 | model=Sequential() 14 | model.add(Convolution2D(nb_filter=32, nb_row=3, nb_col=3, input_shape=(3,150,150))) 15 | model.add(Activation('relu')) 16 | model.add(MaxPooling2D(pool_size=(2,2))) 17 | 18 | model.add(Convolution2D(32, 3, 3)) 19 | model.add(Activation('relu')) 20 | model.add(MaxPooling2D(pool_size=(2,2))) 21 | 22 | model.add(Convolution2D(nb_filter=64, nb_row=3, nb_col=3)) 23 | model.add(Activation('relu')) 24 | model.add(MaxPooling2D(pool_size=(2,2))) 25 | 26 | # the model so far outputs 3D feature maps (height, width, features) 27 | model.add(Flatten()) # this converts our 3D feature maps to 1D feature vectors 28 | model.add(Dense(64)) # 64 neurons 29 | model.add(Activation('relu')) 30 | model.add(Dropout(0.5)) # drop 50% of neurons 31 | 32 | # output layer: classify to 10 driver's states 33 | model.add(Dense(10)) 34 | model.add(Activation('softmax')) 35 | 36 | # compile the model 37 | model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']) 38 | 39 | # the augmentation configuration for generating training data 40 | train_datagen=ImageDataGenerator(rescale=1.0/255, shear_range=0.2, zoom_range=0.2, horizontal_flip=True) 41 | 42 | # validation image is scaled by 1/255, no other augmentation on validation data 43 | test_datagen=ImageDataGenerator(rescale=1.0/255) 44 | 45 | #this is the generator that will read images found in sub-folders of 'data/train', 46 | #and indefinitely generate batches of augmented image data 47 | train_generator=train_datagen.flow_from_directory('data/train', target_size=(150,150), 48 | batch_size=32, class_mode='categorical') 49 | 50 | # this is the generator for validation data 51 | validation_generator=test_datagen.flow_from_directory('data/validation', target_size=(150,150), 52 | batch_size=32, class_mode='categorical') 53 | 54 | # train the convolutional neural network 55 | model.fit_generator(train_generator, samples_per_epoch=20924, nb_epoch=20, 56 | validation_data=validation_generator, nb_val_samples=800) 57 | 58 | # save the weights 59 | model.save_weights('driver_state_detection_small_CNN.h5') 60 | 61 | -------------------------------------------------------------------------------- /img_7574.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yifeng-He/Distracted-Driver-Detection-with-Deep-Learning/d41d679391f32cf6824f508748dde03f80bd085f/img_7574.jpg -------------------------------------------------------------------------------- /vgg16_CNN.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yifeng-He/Distracted-Driver-Detection-with-Deep-Learning/d41d679391f32cf6824f508748dde03f80bd085f/vgg16_CNN.png --------------------------------------------------------------------------------