├── .gitattributes ├── .idea ├── .gitignore ├── inspectionProfiles │ └── profiles_settings.xml ├── misc.xml ├── modules.xml ├── smoking_detection.iml └── vcs.xml ├── 3Dcnn.py ├── README.md ├── __pycache__ ├── data.cpython-37.pyc ├── extractor.cpython-37.pyc ├── models.cpython-37.pyc └── processor.cpython-37.pyc ├── data ├── 1_move_files.py ├── 2_extract_files.py ├── check │ ├── non_smoking │ │ ├── v_--0edUL8zmA.mp4 │ │ └── v_2U0vMYnC49c.mp4 │ └── smoking │ │ ├── v_Vpue9WHrcAk.mp4 │ │ └── v_ak8MVjE2p3Y.mp4 ├── data_file.csv ├── extract_files.py ├── logs │ ├── lstm-training-1585002377.8591547.log │ └── lstm-training-1585006876.5254164.log ├── move_files.py ├── sequences │ └── A_Beautiful_Mind_2_smoke_u_cm_np1_fr_goo_0-40-features.npy ├── sequences_test │ ├── -KmI1S3A9Rc-50-features.npy │ └── 0KqeKi2CBqg-50-features.npy └── test │ ├── non_smoking │ ├── ChikiMovie_ride_horse_f_cm_np1_fr_med_1.avi │ └── Compilationknifethrowing_throw_u_nm_np1_ba_med_9.avi │ └── smoking │ ├── A_Beautiful_Mind_2_smoke_u_cm_np1_fr_goo_0.avi │ ├── girl_smoking_a_cigarette_smoke_h_nm_np1_fr_med_0.avi │ └── nice_smoking_girl_smoke_h_nm_np1_le_med_0.avi ├── data_processor.py ├── download_models.sh ├── extract_features.py ├── extract_features_full.py ├── extractor.py ├── models.py ├── optical_flow.py ├── plot_trainlog.py ├── processor.py ├── requirements.txt ├── test.py ├── test.sh ├── train.py ├── train_cnn.py ├── validate_cnn.py ├── validate_model.py └── validate_rnn.py /.gitattributes: -------------------------------------------------------------------------------- 1 | *.hdf5 filter=lfs diff=lfs merge=lfs -text 2 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /workspace.xml 3 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/smoking_detection.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /3Dcnn.py: -------------------------------------------------------------------------------- 1 | from keras.preprocessing.image import ImageDataGenerator 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Dropout, Activation, Flatten 4 | from keras.layers.convolutional import Convolution3D, MaxPooling3D 5 | 6 | from keras.optimizers import SGD, RMSprop 7 | from keras.utils import np_utils, generic_utils 8 | 9 | import theano 10 | import os 11 | import matplotlib 12 | import matplotlib.pyplot as plt 13 | import numpy as np 14 | import cv2 15 | from sklearn.cross_validation import train_test_split 16 | from sklearn import cross_validation 17 | from sklearn import preprocessing 18 | 19 | 20 | X_tr_array = np.array(X_tr) # convert the frames read into array 21 | 22 | num_samples = len(X_tr_array) 23 | 24 | # Assign Label to each class 25 | 26 | label = np.ones((num_samples,), dtype=int) 27 | label[0:100] = 0 28 | label[100:199] = 1 29 | label[199:299] = 2 30 | label[299:399] = 3 31 | label[399:499] = 4 32 | label[499:] = 5 33 | 34 | train_data = [X_tr_array, label] 35 | 36 | (X_train, y_train) = (train_data[0], train_data[1]) 37 | print('X_Train shape:', X_train.shape) 38 | 39 | train_set = np.zeros((num_samples, 1, img_rows, img_cols, img_depth)) 40 | 41 | for h in xrange(num_samples): 42 | train_set[h][0][:][:][:] = X_train[h, :, :, :] 43 | 44 | patch_size = 15 # img_depth or number of frames used for each video 45 | 46 | print(train_set.shape, 'train samples') 47 | 48 | # CNN Training parameters 49 | 50 | batch_size = 2 51 | nb_classes = 6 52 | nb_epoch = 50 53 | 54 | # convert class vectors to binary class matrices 55 | Y_train = np_utils.to_categorical(y_train, nb_classes) 56 | 57 | # number of convolutional filters to use at each layer 58 | nb_filters = [32, 32] 59 | 60 | # level of pooling to perform at each layer (POOL x POOL) 61 | nb_pool = [3, 3] 62 | 63 | # level of convolution to perform at each layer (CONV x CONV) 64 | nb_conv = [5, 5] 65 | 66 | # Pre-processing 67 | 68 | train_set = train_set.astype('float32') 69 | 70 | train_set -= np.mean(train_set) 71 | 72 | train_set /= np.max(train_set) 73 | 74 | # Define model 75 | 76 | model = Sequential() 77 | model.add(Convolution3D(nb_filters[0], nb_depth=nb_conv[0], nb_row=nb_conv[0], nb_col=nb_conv[0], 78 | input_shape=(1, img_rows, img_cols, patch_size), activation='relu')) 79 | 80 | model.add(MaxPooling3D(pool_size=(nb_pool[0], nb_pool[0], nb_pool[0]))) 81 | 82 | model.add(Dropout(0.5)) 83 | 84 | model.add(Flatten()) 85 | 86 | model.add(Dense(128, init='normal', activation='relu')) 87 | 88 | model.add(Dropout(0.5)) 89 | 90 | model.add(Dense(nb_classes, init='normal')) 91 | 92 | model.add(Activation('softmax')) 93 | 94 | model.compile(loss='categorical_crossentropy', optimizer='RMSprop') 95 | 96 | # Split the data 97 | 98 | X_train_new, X_val_new, y_train_new, y_val_new = train_test_split(train_set, Y_train, test_size=0.2, random_state=4) 99 | 100 | # Train the model 101 | 102 | hist = model.fit(X_train_new, y_train_new, validation_data=(X_val_new, y_val_new), 103 | batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, shuffle=True) 104 | 105 | # hist = model.fit(train_set, Y_train, batch_size=batch_size, 106 | # nb_epoch=nb_epoch,validation_split=0.2, show_accuracy=True, 107 | # shuffle=True) 108 | 109 | 110 | # Evaluate the model 111 | score = model.evaluate(X_val_new, y_val_new, batch_size=batch_size, show_accuracy=True) 112 | print('Test score:', score[0]) 113 | print('Test accuracy:', score[1]) 114 | 115 | # Plot the results 116 | train_loss = hist.history['loss'] 117 | val_loss = hist.history['val_loss'] 118 | train_acc = hist.history['acc'] 119 | val_acc = hist.history['val_acc'] 120 | xc = range(100) 121 | 122 | plt.figure(1, figsize=(7, 5)) 123 | plt.plot(xc, train_loss) 124 | plt.plot(xc, val_loss) 125 | plt.xlabel('num of Epochs') 126 | plt.ylabel('loss') 127 | plt.title('train_loss vs val_loss') 128 | plt.grid(True) 129 | plt.legend(['train', 'val']) 130 | print 131 | plt.style.available # use bmh, classic,ggplot for big pictures 132 | plt.style.use(['classic']) 133 | 134 | plt.figure(2, figsize=(7, 5)) 135 | plt.plot(xc, train_acc) 136 | plt.plot(xc, val_acc) 137 | plt.xlabel('num of Epochs') 138 | plt.ylabel('accuracy') 139 | plt.title('train_acc vs val_acc') 140 | plt.grid(True) 141 | plt.legend(['train', 'val'], loc=4) 142 | # print plt.style.available # use bmh, classic,ggplot for big pictures 143 | plt.style.use(['classic']) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## How to run the model to test the samples 2 | 3 | ### Testing one video using standalone script: 4 | 5 | Dowwnload the script https://github.com/harshita-chaudhary/smoking_detection/blob/master/test.sh 6 | 7 | Run it: sh test.sh 8 | 9 | ### Testing one video: 10 | 11 | python test.py --video_name [video_name] 12 | 13 | Output: 14 | 15 | • timeLabel.json: JSON file containing the time in seconds vs probability of smoking action 16 | 17 | • timeLabel.jpg: Image showing action label. 0-non-smoking, 1-smoking 18 | 19 | • [video_name].avi: Video overlaid with label and probability of each frame 20 | 21 | 22 | ### Testing multiple videos: 23 | 24 | Testing multiple videos: 25 | 1. Place the videos to be tested in data/check folder under smoking and non-smoking directories. This is done to ensure that we remember which class the testing video belongs to. 26 | 27 | 2. Set the model weights in ‘validate_model.py’ file to point to the corresponding saved model in data/checkpoints directory or the downloaded model. 28 | 29 | 3. Run ‘validate_model.py’ to classify the videos using the LSTM model. This generates a plot for each video with x-axis as the frame number and y-axis as the corresponding generated label. Since the videos used for testing are clipped videos and contain smoking or non-smoking action in entirety, the graph shows a straight line at either 0 or 1. 1 denotes smoking and 0 denotes non-smoking. 30 | 31 | ## Requirements 32 | 33 | 1. Python 3 34 | 1. Keras 35 | 1. Tensorflow 36 | 1. Numpy 37 | 1. ffmpeg 38 | 1. Matplotlib 39 | 1. tqdm 40 | 1. Pillow 41 | 42 | 43 | ## Getting the data 44 | 45 | The data was taken from the HMDB51 dataset that contains clipped videos of 51 different actions. 46 | For smoking, all the videos in the smoking class were taken, and for non-smoking, videos were randomly picked from the different action action classes to create an equally distributed dataset. 47 | 48 | HMDB51 dataset link: https://serre-lab.clps.brown.edu/resource/hmdb-a-large-human-motion-database/#dataset 49 | ActivityNet200 dataset link: http://activity-net.org/download.html 50 | Kinetics700 dataset link: https://deepmind.com/research/open-source/kinetics 51 | 52 | ## Saved models 53 | 54 | The weights for models trained locally are stored in "data/chekcpoints". 55 | 56 | The latest trained model download links: 57 | 58 | CNN Model: https://drive.google.com/file/d/1DgfeKLC6t9bbkCGuLiZcpi9iLfgh7LdK/view?usp=sharing 59 | 60 | LSTM model: https://drive.google.com/file/d/1Hrn9BZ7uC9jxhHzBEgkQWakzBBr8xAnU/view?usp=sharing 61 | 62 | Other trained models: 63 | 64 | 3D spatial CNN: https://drive.google.com/file/d/18fBz26PdGsl8SXdgp3JNfhq_jWccybyy/view?usp=sharing 65 | 66 | 3D temporal CNN: https://drive.google.com/file/d/1wDToIFo55bI7dSlasIwTwIiZhOzhptL_/view?usp=sharing 67 | 68 | All Models Drive folder: https://drive.google.com/drive/folders/14yx7GTfylT7nIvt_huvvdUSRM0YCsmwy?usp=sharing 69 | 70 | ## Training models 71 | 72 | TThe CNN model is trained by running `train_cnn.py`. This trains the InceptionV3 model with initial weights taken from the model trained on ImageNet. The model is further train on the HMDB data that is present in the data/train and data/test directories. 73 | 74 | The RNN model is trained by running `train.py`. This trains the LSTM layers follwed by dense layers. 75 | The trained weights are stored in data/chekcpoints. This is used while classifying the videos. 76 | 77 | The trained weights are stored in "data/chekcpoints". This is used while extracting the features. 78 | 79 | ## Demo/Using models 80 | 81 | Demo on how to run is uploaded on Youtube. 82 | 83 | Link: https://www.youtube.com/watch?v=VDf8s8x4WLA&list=PLFrrF91jLrRZhb-3Dcq8wIwYgWP-t0pFG&index=6 84 | 85 | ## References 86 | http://jeffdonahue.com/lrcn/ 87 | 88 | https://keras.io/models/about-keras-models/ 89 | 90 | https://blog.coast.ai/five-video-classification-methods-implemented-in-keras-and-tensorflow-99cad29cc0b5 91 | 92 | https://serre-lab.clps.brown.edu/resource/hmdb-a-large-human-motion-database/#dataset 93 | 94 | http://activity-net.org/download.html 95 | 96 | https://deepmind.com/research/open-source/kinetics 97 | 98 | https://keras.io/models/about-keras-models/ 99 | 100 | https://sites.google.com/view/anxiaojiang/csce636 101 | -------------------------------------------------------------------------------- /__pycache__/data.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harshita-chaudhary/smoking_detection/41834f1f8f8ead3f9aa88074b9012be8979baf32/__pycache__/data.cpython-37.pyc -------------------------------------------------------------------------------- /__pycache__/extractor.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harshita-chaudhary/smoking_detection/41834f1f8f8ead3f9aa88074b9012be8979baf32/__pycache__/extractor.cpython-37.pyc -------------------------------------------------------------------------------- /__pycache__/models.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harshita-chaudhary/smoking_detection/41834f1f8f8ead3f9aa88074b9012be8979baf32/__pycache__/models.cpython-37.pyc -------------------------------------------------------------------------------- /__pycache__/processor.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harshita-chaudhary/smoking_detection/41834f1f8f8ead3f9aa88074b9012be8979baf32/__pycache__/processor.cpython-37.pyc -------------------------------------------------------------------------------- /data/1_move_files.py: -------------------------------------------------------------------------------- 1 | """ 2 | After extracting the RAR, we run this to move all the files into 3 | the appropriate train/test folders. 4 | 5 | Should only run this file once! 6 | """ 7 | import os 8 | import os.path 9 | 10 | def get_train_test_lists(version='01'): 11 | """ 12 | Using one of the train/test files (01, 02, or 03), get the filename 13 | breakdowns we'll later use to move everything. 14 | """ 15 | # Get our files based on version. 16 | test_file = os.path.join('ucfTrainTestlist', 'testlist' + version + '.txt') 17 | train_file = os.path.join('ucfTrainTestlist', 'trainlist' + version + '.txt') 18 | 19 | # Build the test list. 20 | with open(test_file) as fin: 21 | test_list = [row.strip() for row in list(fin)] 22 | 23 | # Build the train list. Extra step to remove the class index. 24 | with open(train_file) as fin: 25 | train_list = [row.strip() for row in list(fin)] 26 | train_list = [row.split(' ')[0] for row in train_list] 27 | 28 | # Set the groups in a dictionary. 29 | file_groups = { 30 | 'train': train_list, 31 | 'test': test_list 32 | } 33 | 34 | return file_groups 35 | 36 | def move_files(file_groups): 37 | """This assumes all of our files are currently in _this_ directory. 38 | So move them to the appropriate spot. Only needs to happen once. 39 | """ 40 | # Do each of our groups. 41 | for group, videos in file_groups.items(): 42 | 43 | # Do each of our videos. 44 | for video in videos: 45 | print(group) 46 | 47 | # Get the parts. 48 | parts = video.split("/") 49 | print(video) 50 | # parts = video.split(os.path.sep) 51 | # print(os.path.sep) 52 | # print(parts) 53 | classname = parts[0] 54 | filename = parts[1] 55 | 56 | # Check if this class exists. 57 | if not os.path.exists(os.path.join(group, classname)): 58 | print("Creating folder for %s/%s" % (group, classname)) 59 | os.makedirs(os.path.join(group, classname)) 60 | 61 | # Check if we have already moved this file, or at least that it 62 | # exists to move. 63 | # if not os.path.exists(filename): 64 | # print("Can't find %s to move. Skipping." % (filename)) 65 | # continue 66 | if not os.path.exists(video): 67 | print("Can't find %s to move. Skipping." % (video)) 68 | continue 69 | # Move it. 70 | 71 | # dest = os.path.join(group, classname, filename) 72 | # print(dest) 73 | # print("Moving %s to %s" % (filename, dest)) 74 | # os.rename(filename, dest) 75 | dest = os.path.join(group, classname, filename) 76 | 77 | src = os.path.join(classname, filename) 78 | print(dest, src) 79 | print("Moving %s to %s" % (src, dest)) 80 | os.rename(src, dest) 81 | 82 | print("Done.") 83 | 84 | def main(): 85 | """ 86 | Go through each of our train/test text files and move the videos 87 | to the right place. 88 | """ 89 | # Get the videos in groups so we can move them. 90 | group_lists = get_train_test_lists() 91 | 92 | # Move the files. 93 | move_files(group_lists) 94 | 95 | if __name__ == '__main__': 96 | main() 97 | -------------------------------------------------------------------------------- /data/2_extract_files.py: -------------------------------------------------------------------------------- 1 | """ 2 | After moving all the files using the 1_ file, we run this one to extract 3 | the images from the videos and also create a data file we can use 4 | for training and testing later. 5 | """ 6 | import csv 7 | import glob 8 | import os 9 | import os.path 10 | from subprocess import call 11 | 12 | def extract_files(): 13 | """After we have all of our videos split between train and test, and 14 | all nested within folders representing their classes, we need to 15 | make a data file that we can reference when training our RNN(s). 16 | This will let us keep track of image sequences and other parts 17 | of the training process. 18 | 19 | We'll first need to extract images from each of the videos. We'll 20 | need to record the following data in the file: 21 | 22 | [train|test], class, filename, nb frames 23 | 24 | Extracting can be done with ffmpeg: 25 | `ffmpeg -i video.mpg image-%04d.jpg` 26 | """ 27 | data_file = [] 28 | folders = ['train', 'test'] 29 | 30 | for folder in folders: 31 | class_folders = glob.glob(os.path.join(folder, '*')) 32 | 33 | for vid_class in class_folders: 34 | class_files = glob.glob(os.path.join(vid_class, '*.avi')) 35 | 36 | for video_path in class_files: 37 | # Get the parts of the file. 38 | video_parts = get_video_parts(video_path) 39 | 40 | train_or_test, classname, filename_no_ext, filename = video_parts 41 | 42 | # Only extract if we haven't done it yet. Otherwise, just get 43 | # the info. 44 | if not check_already_extracted(video_parts): 45 | # Now extract it. 46 | src = os.path.join(train_or_test, classname, filename) 47 | dest = os.path.join(train_or_test, classname, 48 | filename_no_ext + '-%04d.jpg') 49 | dirname = os.path.dirname(__file__) 50 | # src = src.replace("\\", "/") 51 | print(src) 52 | filename = os.path.join(dirname, src) 53 | # print(filename.replace("\\", "/") 54 | print(filename) 55 | print(dest) 56 | 57 | if os.path.isfile(filename): 58 | print("File exist") 59 | if os.path.isfile(dest): 60 | print("File dest exist") 61 | # call(["ffmpeg", "-i", "r\"" + src + "\"", "r\"" + dest + "\""], shell=True) 62 | call(["ffmpeg", "-i", src, dest], shell=True) 63 | # Now get how many frames it is. 64 | nb_frames = get_nb_frames_for_video(video_parts) 65 | 66 | data_file.append([train_or_test, classname, filename_no_ext, nb_frames]) 67 | 68 | print("Generated %d frames for %s" % (nb_frames, filename_no_ext)) 69 | 70 | with open('data_file.csv', 'w') as fout: 71 | writer = csv.writer(fout) 72 | writer.writerows(data_file) 73 | 74 | print("Extracted and wrote %d video files." % (len(data_file))) 75 | 76 | def get_nb_frames_for_video(video_parts): 77 | """Given video parts of an (assumed) already extracted video, return 78 | the number of frames that were extracted.""" 79 | train_or_test, classname, filename_no_ext, _ = video_parts 80 | generated_files = glob.glob(os.path.join(train_or_test, classname, 81 | filename_no_ext + '*.jpg')) 82 | return len(generated_files) 83 | 84 | def get_video_parts(video_path): 85 | """Given a full path to a video, return its parts.""" 86 | parts = video_path.split(os.path.sep) 87 | filename = parts[2] 88 | filename_no_ext = filename.split('.')[0] 89 | classname = parts[1] 90 | train_or_test = parts[0] 91 | 92 | return train_or_test, classname, filename_no_ext, filename 93 | 94 | def check_already_extracted(video_parts): 95 | """Check to see if we created the -0001 frame of this file.""" 96 | train_or_test, classname, filename_no_ext, _ = video_parts 97 | return bool(os.path.exists(os.path.join(train_or_test, classname, 98 | filename_no_ext + '-0001.jpg'))) 99 | 100 | def main(): 101 | """ 102 | Extract images from videos and build a new file that we 103 | can use as our data input file. It can have format: 104 | 105 | [train|test], class, filename, nb frames 106 | """ 107 | extract_files() 108 | 109 | if __name__ == '__main__': 110 | main() 111 | -------------------------------------------------------------------------------- /data/check/non_smoking/v_--0edUL8zmA.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harshita-chaudhary/smoking_detection/41834f1f8f8ead3f9aa88074b9012be8979baf32/data/check/non_smoking/v_--0edUL8zmA.mp4 -------------------------------------------------------------------------------- /data/check/non_smoking/v_2U0vMYnC49c.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harshita-chaudhary/smoking_detection/41834f1f8f8ead3f9aa88074b9012be8979baf32/data/check/non_smoking/v_2U0vMYnC49c.mp4 -------------------------------------------------------------------------------- /data/check/smoking/v_Vpue9WHrcAk.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harshita-chaudhary/smoking_detection/41834f1f8f8ead3f9aa88074b9012be8979baf32/data/check/smoking/v_Vpue9WHrcAk.mp4 -------------------------------------------------------------------------------- /data/check/smoking/v_ak8MVjE2p3Y.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harshita-chaudhary/smoking_detection/41834f1f8f8ead3f9aa88074b9012be8979baf32/data/check/smoking/v_ak8MVjE2p3Y.mp4 -------------------------------------------------------------------------------- /data/data_file.csv: -------------------------------------------------------------------------------- 1 | test,non_smoking,ChikiMovie_ride_horse_f_cm_np1_fr_med_1,290 2 | test,non_smoking,Compilationknifethrowing_throw_u_nm_np1_ba_med_9,80 3 | test,smoking,A_Beautiful_Mind_2_smoke_u_cm_np1_fr_goo_0,80 4 | test,smoking,girl_smoking_a_cigarette_smoke_h_nm_np1_fr_med_0,115 5 | test,smoking,nice_smoking_girl_smoke_h_nm_np1_le_med_0,155 6 | -------------------------------------------------------------------------------- /data/extract_files.py: -------------------------------------------------------------------------------- 1 | """ 2 | After moving all the files using the 1_ file, we run this one to extract 3 | the images from the videos and also create a data file we can use 4 | for training and testing later. 5 | """ 6 | import csv 7 | import glob 8 | import os 9 | import os.path 10 | from subprocess import call 11 | 12 | def extract_files(folders=None): 13 | """After we have all of our videos split between train and test, and 14 | all nested within folders representing their classes, we need to 15 | make a data file that we can reference when training our RNN(s). 16 | This will let us keep track of image sequences and other parts 17 | of the training process. 18 | 19 | We'll first need to extract images from each of the videos. We'll 20 | need to record the following data in the file: 21 | 22 | [train|test], class, filename, nb frames 23 | 24 | Extracting can be done with ffmpeg: 25 | `ffmpeg -i video.mpg image-%04d.jpg` 26 | """ 27 | data_file = [] 28 | if folders is None: 29 | folders = ['train', 'test'] 30 | 31 | for folder in folders: 32 | class_folders = glob.glob(os.path.join('data',folder, '*')) 33 | 34 | for vid_class in class_folders: 35 | class_files = glob.glob(os.path.join(vid_class, '*.avi')) 36 | class_files.extend(glob.glob(os.path.join(vid_class, '*.mp4'))) 37 | print(class_files) 38 | for video_path in class_files: 39 | # Get the parts of the file. 40 | video_parts = get_video_parts(video_path) 41 | 42 | train_or_test, classname, filename_no_ext, filename, filename_ext = video_parts 43 | 44 | # Only extract if we haven't done it yet. Otherwise, just get 45 | # the info. 46 | if not check_already_extracted(video_parts): 47 | # Now extract it. 48 | src = os.path.join('data',train_or_test, classname, filename) 49 | dest = os.path.join('data',train_or_test, classname, 50 | filename_no_ext + '-%04d.jpg') 51 | dirname = os.path.dirname(__file__) 52 | # src = src.replace("\\", "/") 53 | print(src) 54 | filename = os.path.join(dirname, src) 55 | # print(filename.replace("\\", "/") 56 | print(filename) 57 | print(dest) 58 | 59 | if os.path.isfile(filename): 60 | print("File exist") 61 | if os.path.isfile(dest): 62 | print("File dest exist") 63 | # call(["ffmpeg", "-i", "r\"" + src + "\"", "r\"" + dest + "\""], shell=True) 64 | call(["ffmpeg", "-i", src, dest], shell=True) 65 | # Now get how many frames it is. 66 | nb_frames = get_nb_frames_for_video(video_parts) 67 | 68 | data_file.append([train_or_test, classname, filename_no_ext, nb_frames, filename_ext]) 69 | 70 | print("Generated %d frames for %s" % (nb_frames, filename_no_ext)) 71 | 72 | write_file = 'data_file.csv' if folders is None else 'data/data_test_file.csv' 73 | 74 | with open(write_file, 'w') as fout: 75 | writer = csv.writer(fout) 76 | writer.writerows(data_file) 77 | 78 | print("Extracted and wrote %d video files." % (len(data_file))) 79 | 80 | def get_nb_frames_for_video(video_parts): 81 | """Given video parts of an (assumed) already extracted video, return 82 | the number of frames that were extracted.""" 83 | train_or_test, classname, filename_no_ext, _, _ = video_parts 84 | generated_files = glob.glob(os.path.join('data',train_or_test, classname, 85 | filename_no_ext + '*.jpg')) 86 | return len(generated_files) 87 | 88 | def get_video_parts(video_path): 89 | """Given a full path to a video, return its parts.""" 90 | parts = video_path.split(os.path.sep) 91 | filename = parts[3] 92 | filename_no_ext = filename.split('.')[0] 93 | classname = parts[2] 94 | train_or_test = parts[1] 95 | filename_ext = filename.split('.')[1] 96 | return train_or_test, classname, filename_no_ext, filename, filename_ext 97 | 98 | def check_already_extracted(video_parts): 99 | """Check to see if we created the -0001 frame of this file.""" 100 | train_or_test, classname, filename_no_ext, _, _ = video_parts 101 | return bool(os.path.exists(os.path.join(train_or_test, classname, 102 | filename_no_ext + '-0001.jpg'))) 103 | 104 | def main(): 105 | """ 106 | Extract images from videos and build a new file that we 107 | can use as our data input file. It can have format: 108 | 109 | [train|test], class, filename, nb frames 110 | """ 111 | extract_files() 112 | 113 | if __name__ == '__main__': 114 | main() 115 | -------------------------------------------------------------------------------- /data/logs/lstm-training-1585002377.8591547.log: -------------------------------------------------------------------------------- 1 | epoch,accuracy,loss,val_accuracy,val_loss 2 | 0,0.42857143,0.7958271803712487,0.4761904776096344,0.699584668590909 3 | 1,0.5488722,0.722956527444653,0.6190476417541504,0.6677232498214358 4 | 2,0.62406015,0.6242258907260751,0.6666666865348816,0.6384629465284801 5 | 3,0.57894737,0.667347780743936,0.761904776096344,0.6066708616794102 6 | 4,0.7067669,0.6149162322955024,0.7460317611694336,0.577295643469644 7 | 5,0.6616541,0.5790995435607165,0.7936508059501648,0.5523021646908352 8 | 6,0.7368421,0.5612340470902005,0.7936508059501648,0.5307696745509193 9 | 7,0.7894737,0.5371864650930677,0.8253968358039856,0.5110870916692037 10 | 8,0.80451125,0.5236193474970365,0.841269850730896,0.49322336722934057 11 | 9,0.7894737,0.49572645497501344,0.841269850730896,0.4785890077787732 12 | 10,0.84962404,0.48420707108382893,0.8095238208770752,0.47708723280164933 13 | 11,0.7819549,0.4724430177444802,0.8095238208770752,0.4757151598968203 14 | 12,0.8120301,0.4410141751282197,0.841269850730896,0.44545379990623113 15 | 13,0.84962404,0.42756392229768564,0.8730158805847168,0.4261011292064001 16 | 14,0.86466163,0.4109718073579602,0.8571428656578064,0.4172018544068412 17 | 15,0.87969923,0.41170141571446467,0.8730158805847168,0.4094493469548604 18 | 16,0.87218046,0.394564064597725,0.841269850730896,0.4188499528737295 19 | 17,0.87218046,0.358885389521606,0.841269850730896,0.42158426959363243 20 | 18,0.86466163,0.3380894489530334,0.841269850730896,0.40293554985334 21 | 19,0.88721806,0.32926239151703685,0.8888888955116272,0.3907801267646608 22 | 20,0.93233085,0.2915762584460409,0.8730158805847168,0.3843358774033804 23 | 21,0.88721806,0.3036008385339178,0.8730158805847168,0.3848939741414691 24 | 22,0.9097744,0.31464350145114095,0.8888888955116272,0.3884841530095963 25 | 23,0.9097744,0.30546209675476965,0.8888888955116272,0.3911096751689911 26 | 24,0.91729325,0.2778091644658182,0.8571428656578064,0.40479522281222874 27 | 25,0.93233085,0.24490920986448014,0.841269850730896,0.4113678059407643 28 | -------------------------------------------------------------------------------- /data/logs/lstm-training-1585006876.5254164.log: -------------------------------------------------------------------------------- 1 | epoch,accuracy,loss,val_accuracy,val_loss 2 | 0,0.5413534,0.714905922126053,0.6507936716079712,0.6503184447212825 3 | 1,0.6165413,0.6547514222618332,0.7777777910232544,0.6099521688052586 4 | 2,0.6165413,0.6465868837851331,0.7460317611694336,0.5805835903636993 5 | 3,0.6315789,0.6562247486939108,0.7777777910232544,0.5482622924305144 6 | 4,0.7067669,0.5618461661769035,0.7777777910232544,0.5218555146739596 7 | 5,0.7593985,0.5326783482293437,0.8571428656578064,0.49513877573467435 8 | 6,0.7819549,0.533280058462817,0.8253968358039856,0.4708064619510893 9 | 7,0.7969925,0.5175594808463764,0.8571428656578064,0.4489482934512789 10 | 8,0.7969925,0.4803439656594642,0.8571428656578064,0.4326216592675164 11 | 9,0.8120301,0.4618645554646513,0.8571428656578064,0.42345073961076285 12 | 10,0.80451125,0.42998544076331574,0.8730158805847168,0.4153602785534329 13 | 11,0.87218046,0.3746000954083034,0.8571428656578064,0.40066119082390317 14 | 12,0.88721806,0.37952006759500145,0.8571428656578064,0.38927724957466125 15 | 13,0.87969923,0.3789210429317073,0.8571428656578064,0.37986502051353455 16 | 14,0.90225565,0.3367891278033866,0.8571428656578064,0.3746222054201459 17 | 15,0.87969923,0.3512900907518272,0.8571428656578064,0.36949517566060264 18 | 16,0.87218046,0.3311726022931866,0.8571428656578064,0.36632243934131803 19 | 17,0.8947368,0.3102036005348191,0.8571428656578064,0.36715405706375365 20 | 18,0.91729325,0.2918048890909754,0.8571428656578064,0.3670268654823303 21 | 19,0.93233085,0.29629718473083094,0.8571428656578064,0.3657246132691701 22 | 20,0.93233085,0.2838467820022339,0.8571428656578064,0.3663598180763305 23 | 21,0.8947368,0.2592245725760783,0.8571428656578064,0.3714765201485346 24 | 22,0.924812,0.2640138820822078,0.8571428656578064,0.38020946203716216 25 | 23,0.87969923,0.2908601034852795,0.8571428656578064,0.3905587967426058 26 | 24,0.91729325,0.2398957157493534,0.8571428656578064,0.4004892941032137 27 | -------------------------------------------------------------------------------- /data/move_files.py: -------------------------------------------------------------------------------- 1 | """ 2 | After extracting the RAR, we run this to move all the files into 3 | the appropriate train/test folders. 4 | 5 | Should only run this file once! 6 | """ 7 | import os 8 | import os.path 9 | 10 | def get_train_test_lists(version='01'): 11 | """ 12 | Using one of the train/test files (01, 02, or 03), get the filename 13 | breakdowns we'll later use to move everything. 14 | """ 15 | # Get our files based on version. 16 | test_file = os.path.join('ucfTrainTestlist', 'testlist' + version + '.txt') 17 | train_file = os.path.join('ucfTrainTestlist', 'trainlist' + version + '.txt') 18 | 19 | # Build the test list. 20 | with open(test_file) as fin: 21 | test_list = [row.strip() for row in list(fin)] 22 | 23 | # Build the train list. Extra step to remove the class index. 24 | with open(train_file) as fin: 25 | train_list = [row.strip() for row in list(fin)] 26 | train_list = [row.split(' ')[0] for row in train_list] 27 | 28 | # Set the groups in a dictionary. 29 | file_groups = { 30 | 'train': train_list, 31 | 'test': test_list 32 | } 33 | 34 | return file_groups 35 | 36 | def move_files(file_groups): 37 | """This assumes all of our files are currently in _this_ directory. 38 | So move them to the appropriate spot. Only needs to happen once. 39 | """ 40 | # Do each of our groups. 41 | for group, videos in file_groups.items(): 42 | 43 | # Do each of our videos. 44 | for video in videos: 45 | print(group) 46 | 47 | # Get the parts. 48 | parts = video.split("/") 49 | print(video) 50 | # parts = video.split(os.path.sep) 51 | # print(os.path.sep) 52 | # print(parts) 53 | classname = parts[0] 54 | filename = parts[1] 55 | 56 | # Check if this class exists. 57 | if not os.path.exists(os.path.join(group, classname)): 58 | print("Creating folder for %s/%s" % (group, classname)) 59 | os.makedirs(os.path.join(group, classname)) 60 | 61 | # Check if we have already moved this file, or at least that it 62 | # exists to move. 63 | # if not os.path.exists(filename): 64 | # print("Can't find %s to move. Skipping." % (filename)) 65 | # continue 66 | if not os.path.exists(video): 67 | print("Can't find %s to move. Skipping." % (video)) 68 | continue 69 | # Move it. 70 | 71 | # dest = os.path.join(group, classname, filename) 72 | # print(dest) 73 | # print("Moving %s to %s" % (filename, dest)) 74 | # os.rename(filename, dest) 75 | dest = os.path.join(group, classname, filename) 76 | 77 | src = os.path.join(classname, filename) 78 | print(dest, src) 79 | print("Moving %s to %s" % (src, dest)) 80 | os.rename(src, dest) 81 | 82 | print("Done.") 83 | 84 | def main(): 85 | """ 86 | Go through each of our train/test text files and move the videos 87 | to the right place. 88 | """ 89 | # Get the videos in groups so we can move them. 90 | group_lists = get_train_test_lists() 91 | 92 | # Move the files. 93 | move_files(group_lists) 94 | 95 | if __name__ == '__main__': 96 | main() 97 | -------------------------------------------------------------------------------- /data/sequences/A_Beautiful_Mind_2_smoke_u_cm_np1_fr_goo_0-40-features.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harshita-chaudhary/smoking_detection/41834f1f8f8ead3f9aa88074b9012be8979baf32/data/sequences/A_Beautiful_Mind_2_smoke_u_cm_np1_fr_goo_0-40-features.npy -------------------------------------------------------------------------------- /data/sequences_test/-KmI1S3A9Rc-50-features.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harshita-chaudhary/smoking_detection/41834f1f8f8ead3f9aa88074b9012be8979baf32/data/sequences_test/-KmI1S3A9Rc-50-features.npy -------------------------------------------------------------------------------- /data/sequences_test/0KqeKi2CBqg-50-features.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harshita-chaudhary/smoking_detection/41834f1f8f8ead3f9aa88074b9012be8979baf32/data/sequences_test/0KqeKi2CBqg-50-features.npy -------------------------------------------------------------------------------- /data/test/non_smoking/ChikiMovie_ride_horse_f_cm_np1_fr_med_1.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harshita-chaudhary/smoking_detection/41834f1f8f8ead3f9aa88074b9012be8979baf32/data/test/non_smoking/ChikiMovie_ride_horse_f_cm_np1_fr_med_1.avi -------------------------------------------------------------------------------- /data/test/non_smoking/Compilationknifethrowing_throw_u_nm_np1_ba_med_9.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harshita-chaudhary/smoking_detection/41834f1f8f8ead3f9aa88074b9012be8979baf32/data/test/non_smoking/Compilationknifethrowing_throw_u_nm_np1_ba_med_9.avi -------------------------------------------------------------------------------- /data/test/smoking/A_Beautiful_Mind_2_smoke_u_cm_np1_fr_goo_0.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harshita-chaudhary/smoking_detection/41834f1f8f8ead3f9aa88074b9012be8979baf32/data/test/smoking/A_Beautiful_Mind_2_smoke_u_cm_np1_fr_goo_0.avi -------------------------------------------------------------------------------- /data/test/smoking/girl_smoking_a_cigarette_smoke_h_nm_np1_fr_med_0.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harshita-chaudhary/smoking_detection/41834f1f8f8ead3f9aa88074b9012be8979baf32/data/test/smoking/girl_smoking_a_cigarette_smoke_h_nm_np1_fr_med_0.avi -------------------------------------------------------------------------------- /data/test/smoking/nice_smoking_girl_smoke_h_nm_np1_le_med_0.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harshita-chaudhary/smoking_detection/41834f1f8f8ead3f9aa88074b9012be8979baf32/data/test/smoking/nice_smoking_girl_smoke_h_nm_np1_le_med_0.avi -------------------------------------------------------------------------------- /data_processor.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import numpy as np 3 | import random 4 | import glob 5 | import os.path 6 | import sys 7 | import operator 8 | import threading 9 | from processor import process_image, process_flow 10 | from keras.utils import to_categorical 11 | 12 | class threadsafe_iterator: 13 | def __init__(self, iterator): 14 | self.iterator = iterator 15 | self.lock = threading.Lock() 16 | 17 | def __iter__(self): 18 | return self 19 | 20 | def __next__(self): 21 | with self.lock: 22 | return next(self.iterator) 23 | 24 | def threadsafe_generator(func): 25 | def gen(*a, **kw): 26 | return threadsafe_iterator(func(*a, **kw)) 27 | return gen 28 | 29 | class DataSet(): 30 | 31 | def __init__(self, seq_length=40, class_limit=None, image_shape=(224, 224, 3), check_dir=None): 32 | self.seq_length = seq_length 33 | self.class_limit = class_limit 34 | if check_dir: 35 | self.sequence_path = os.path.join('data', 'sequences_test') 36 | else: 37 | self.sequence_path = os.path.join('data', 'sequences') 38 | self.max_frames = 30000 # max number of frames a video can have for us to use it 39 | 40 | # Get the data. 41 | if check_dir: 42 | self.data = self.get_test_data() 43 | else: 44 | self.data = self.get_data() 45 | # Get the classes. 46 | self.classes = self.get_classes() 47 | 48 | # Now do some minor data cleaning. 49 | self.data = self.clean_data() 50 | 51 | self.image_shape = image_shape 52 | 53 | @staticmethod 54 | def get_data(): 55 | """Load our data from file.""" 56 | with open(os.path.join('data', 'data_file.csv'), 'r') as fin: 57 | reader = csv.reader(fin) 58 | data = list(reader) 59 | 60 | return data 61 | 62 | @staticmethod 63 | def get_test_data(): 64 | """Load our data from file.""" 65 | with open(os.path.join('data', 'data_test_file.csv'), 'r') as fin: 66 | reader = csv.reader(fin) 67 | data = list(reader) 68 | 69 | return data 70 | 71 | def clean_data(self): 72 | """Limit samples to greater than the sequence length and fewer 73 | than N frames. Also limit it to classes we want to use.""" 74 | data_clean = [] 75 | for item in self.data: 76 | if len(item) == 0: 77 | continue 78 | # print(item) 79 | if int(item[3]) >= self.seq_length and int(item[3]) <= self.max_frames \ 80 | and item[1] in self.classes: 81 | data_clean.append(item) 82 | 83 | return data_clean 84 | 85 | def get_classes(self): 86 | """Extract the classes from our data. If we want to limit them, 87 | only return the classes we need.""" 88 | classes = [] 89 | for item in self.data: 90 | if len(item) == 0: 91 | continue 92 | if item[1] not in classes: 93 | classes.append(item[1]) 94 | 95 | # Sort them. 96 | classes = sorted(classes) 97 | 98 | # Return. 99 | if self.class_limit is not None: 100 | return classes[:self.class_limit] 101 | else: 102 | return classes 103 | 104 | def get_class_one_hot(self, class_str): 105 | """Given a class as a string, return its number in the classes 106 | list. This lets us encode and one-hot it for training.""" 107 | # Encode it first. 108 | label_encoded = self.classes.index(class_str) 109 | 110 | # Now one-hot it. 111 | label_hot = to_categorical(label_encoded, len(self.classes)) 112 | 113 | assert len(label_hot) == len(self.classes) 114 | 115 | return label_hot 116 | 117 | def split_train_test(self): 118 | """Split the data into train and test groups.""" 119 | train = [] 120 | test = [] 121 | for item in self.data: 122 | if item[0] == 'train': 123 | train.append(item) 124 | else: 125 | test.append(item) 126 | return train, test 127 | 128 | def get_all_sequences_in_memory(self, train_test, data_type): 129 | """ 130 | This is a mirror of our generator, but attempts to load everything into 131 | memory so we can train way faster. 132 | """ 133 | # Get the right dataset. 134 | train, test = self.split_train_test() 135 | data = train if train_test == 'train' else test 136 | 137 | print("Loading %d samples into memory for %sing." % (len(data), train_test)) 138 | 139 | X, y = [], [] 140 | for row in data: 141 | 142 | if data_type == 'images': 143 | frames = self.get_frames_for_sample(row) 144 | frames = self.rescale_list(frames, self.seq_length) 145 | 146 | # Build the image sequence 147 | sequence = self.build_image_sequence(frames) 148 | 149 | else: 150 | sequence = self.get_extracted_sequence(data_type, row) 151 | 152 | if sequence is None: 153 | print("Can't find sequence. Did you generate them?") 154 | raise 155 | 156 | X.append(sequence) 157 | y.append(self.get_class_one_hot(row[1])) 158 | 159 | return np.array(X), np.array(y) 160 | 161 | @threadsafe_generator 162 | def frame_generator(self, batch_size, train_test, data_type): 163 | """Return a generator that we can use to train on. There are 164 | a couple different things we can return: 165 | 166 | data_type: 'features', 'images' 167 | """ 168 | # Get the right dataset for the generator. 169 | train, test = self.split_train_test() 170 | data = train if train_test == 'train' else test 171 | data = [sample for sample in data if len(self.get_frames_for_sample(sample))>self.seq_length] 172 | print("Creating %s generator with %d samples." % (train_test, len(data))) 173 | 174 | while 1: 175 | X, y = [], [] 176 | 177 | # Generate batch_size samples. 178 | for _ in range(batch_size): 179 | # Reset to be safe. 180 | sequence = None 181 | 182 | # Get a random sample. 183 | sample = random.choice(data) 184 | 185 | # Check to see if we've already saved this sequence. 186 | if data_type is "images" or "flow": 187 | # Get and resample frames. 188 | frames = self.get_frames_for_sample(sample) 189 | if data_type is "flow": 190 | # print(str(len(frames)), ' ', str(self.seq_length + 1)) 191 | frames = self.rescale_list(frames, self.seq_length + 1) 192 | sequence = self.build_flow_sequence(frames) 193 | 194 | else: 195 | frames = self.rescale_list(frames, self.seq_length) 196 | # Build the image sequence 197 | sequence = self.build_image_sequence(frames) 198 | else: 199 | # Get the sequence from disk. 200 | sequence = self.get_extracted_sequence(data_type, sample) 201 | if sequence is None: 202 | raise ValueError("Can't find sequence. Did you generate them?") 203 | 204 | # print("Shape of the sequence", str(np.array(sequence).shape)) 205 | X.append(sequence) 206 | y.append(self.get_class_one_hot(sample[1])) 207 | 208 | yield np.array(X), np.array(y) 209 | 210 | def get_data_train_test(self, data_type, train_test): 211 | """Return a generator that we can use to train on. There are 212 | a couple different things we can return: 213 | 214 | data_type: 'features', 'images' 215 | """ 216 | # Get the right dataset for the generator. 217 | train, test = self.split_train_test() 218 | data = train if train_test == 'train' else test 219 | data = [sample for sample in data if len(self.get_frames_for_sample(sample))>self.seq_length] 220 | 221 | print("Total number of samples: ", len(data)) 222 | 223 | X, y = [], [] 224 | 225 | sequence = None 226 | 227 | # Get a random sample. 228 | for sample in data: 229 | # Check to see if we've already saved this sequence. 230 | 231 | if data_type in ["images", "flow"]: 232 | # Get and resample frames. 233 | frames = self.get_frames_for_sample(sample) 234 | if data_type is "flow": 235 | frames = self.rescale_list(frames, self.seq_length + 1) 236 | sequence = self.build_flow_sequence(frames) 237 | 238 | else: 239 | frames = self.rescale_list(frames, self.seq_length) 240 | # Build the image sequence 241 | sequence = self.build_image_sequence(frames) 242 | 243 | # if data_type is "images": 244 | # # Get and resample frames. 245 | # frames = self.get_frames_for_sample(sample) 246 | # frames = self.rescale_list(frames, self.seq_length) 247 | # 248 | # # Build the image sequence 249 | # sequence = self.build_image_sequence(frames) 250 | else: 251 | # Get the sequence from disk. 252 | sequence = self.get_extracted_sequence(data_type, sample) 253 | if sequence is None: 254 | raise ValueError("Can't find sequence. Did you generate them?") 255 | 256 | X.append(sequence) 257 | y.append(self.classes.index(sample[1])) 258 | # y.append(self.get_class_one_hot(sample[1])) 259 | 260 | return np.array(X), np.array(y) 261 | 262 | def build_image_sequence(self, frames): 263 | """Given a set of frames (filenames), build our sequence.""" 264 | return [process_image(x, self.image_shape) for x in frames] 265 | 266 | def build_flow_sequence(self, frames): 267 | """Given a set of frames (filenames), build our sequence.""" 268 | 269 | flow = [] 270 | for i in range(1, len(frames)): 271 | # if len(flow) == 0: 272 | # flow = process_flow(frames[i-1], frames[i], self.image_shape) 273 | # else: 274 | # flow = np.dstack((flow, process_flow(frames[i-1], frames[i], self.image_shape))) 275 | flow.append(process_flow(frames[i-1], frames[i], self.image_shape)) 276 | # print("Flow shape: ", np.array(flow).shape) 277 | return flow 278 | 279 | def get_extracted_sequence(self, data_type, sample): 280 | """Get the saved extracted features.""" 281 | filename = sample[2] 282 | path = os.path.join(self.sequence_path, filename + '-' + str(self.seq_length) + \ 283 | '-' + data_type + '.npy') 284 | if os.path.isfile(path): 285 | return np.load(path) 286 | else: 287 | return None 288 | 289 | def get_frames_by_filename(self, filename, data_type): 290 | """Given a filename for one of our samples, return the data 291 | the model needs to make predictions.""" 292 | # First, find the sample row. 293 | sample = None 294 | for row in self.data: 295 | if row[2] == filename: 296 | sample = row 297 | break 298 | if sample is None: 299 | raise ValueError("Couldn't find sample: %s" % filename) 300 | 301 | if data_type == "images": 302 | # Get and resample frames. 303 | frames = self.get_frames_for_sample(sample) 304 | frames = self.rescale_list(frames, self.seq_length) 305 | # Build the image sequence 306 | sequence = self.build_image_sequence(frames) 307 | else: 308 | # Get the sequence from disk. 309 | sequence = self.get_extracted_sequence(data_type, sample) 310 | 311 | if sequence is None: 312 | raise ValueError("Can't find sequence. Did you generate them?") 313 | 314 | return sequence 315 | 316 | @staticmethod 317 | def get_frames_for_sample(sample): 318 | """Given a sample row from the data file, get all the corresponding frame 319 | filenames.""" 320 | path = os.path.join('data', sample[0], sample[1]) 321 | filename = sample[2] 322 | images = sorted(glob.glob(os.path.join(path, filename + '*jpg'))) 323 | return images 324 | 325 | @staticmethod 326 | def get_filename_from_image(filename): 327 | parts = filename.split(os.path.sep) 328 | return parts[-1].replace('.jpg', '') 329 | 330 | @staticmethod 331 | def rescale_list(input_list, size): 332 | """Given a list and a size, return a rescaled/samples list. For example, 333 | if we want a list of size 5 and we have a list of size 25, return a new 334 | list of size five which is every 5th element of the origina list.""" 335 | assert len(input_list) >= size 336 | 337 | # Get the number to skip between iterations. 338 | skip = len(input_list) // size 339 | 340 | # Build our new output. 341 | output = [input_list[i] for i in range(0, len(input_list), skip)] 342 | 343 | # Cut off the last one if needed. 344 | return output[:size] -------------------------------------------------------------------------------- /download_models.sh: -------------------------------------------------------------------------------- 1 | wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1GhKYJs3Xb2AuvXWHQIImVlE2akhgtXID' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1GhKYJs3Xb2AuvXWHQIImVlE2akhgtXID" -O cnn.hdf5 && rm -rf /tmp/cookies.txt 2 | wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1KU4YVRRK_llPXWbiHHWLtCMxOYNmF06i' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1KU4YVRRK_llPXWbiHHWLtCMxOYNmF06i" -O lstm.hdf5 && rm -rf /tmp/cookies.txt 3 | -------------------------------------------------------------------------------- /extract_features.py: -------------------------------------------------------------------------------- 1 | """ 2 | This script generates extracted features for each video, which other 3 | models make use of. 4 | """ 5 | import numpy as np 6 | import os.path 7 | from data_processor import DataSet 8 | from extractor import Extractor 9 | from tqdm import tqdm 10 | 11 | 12 | # from keras.backend.tensorflow_backend import set_session 13 | # import tensorflow as tf 14 | # config = tf.ConfigProto() 15 | # config.gpu_options.allow_growth = True # dynamically grow the memory used on the GPU 16 | # config.log_device_placement = True # to log device placement (on which device the operation ran) 17 | # sess = tf.Session(config=config) 18 | # set_session(sess) 19 | import tensorflow as tf 20 | config = tf.compat.v1.ConfigProto() 21 | config.gpu_options.allow_growth = True 22 | session = tf.compat.v1.Session(config=config) 23 | 24 | # Set defaults. 25 | seq_length = 50 26 | class_limit = None # Number of classes to extract. Can be 1-101 or None for all. 27 | 28 | # Get the dataset. 29 | data = DataSet(seq_length=seq_length, class_limit=class_limit) 30 | 31 | model = Extractor(weights="data/checkpoints/inception.hdf5") 32 | 33 | # get the model. 34 | 35 | # model = Extractor(weights="data/checkpoints/inception.035-0.17.hdf5") 36 | 37 | # model = Extractor(weights="data/checkpoints/inception.009-0.29.hdf5") 38 | 39 | 40 | # Loop through data. 41 | # print(data.data) 42 | pbar = tqdm(total=len(data.data)) 43 | for video in data.data: 44 | 45 | # Get the path to the sequence for this video. 46 | path = os.path.join('data', 'sequences', video[2] + '-' + str(seq_length) + \ 47 | '-features') # numpy will auto-append .npy 48 | 49 | # Check if we already have it. 50 | if os.path.isfile(path + '.npy'): 51 | pbar.update(1) 52 | continue 53 | 54 | # Get the frames for this video. 55 | frames = data.get_frames_for_sample(video) 56 | 57 | # Now downsample to just the ones we need. 58 | frames = data.rescale_list(frames, seq_length) 59 | 60 | # Now loop through and extract features to build the sequence. 61 | sequence = [] 62 | for image in frames: 63 | features = model.extract(image) 64 | sequence.append(features) 65 | 66 | dir = os.path.join('data', 'sequences') 67 | if not (os.path.exists(dir)): 68 | os.mkdir(dir) 69 | #Save the sequence. 70 | np.save(path, sequence) 71 | 72 | pbar.update(1) 73 | 74 | pbar.close() 75 | -------------------------------------------------------------------------------- /extract_features_full.py: -------------------------------------------------------------------------------- 1 | """ 2 | This script generates extracted features for each video, which other 3 | models make use of. 4 | 5 | You can change you sequence length and limit to a set number of classes 6 | below. 7 | 8 | class_limit is an integer that denotes the first N classes you want to 9 | extract features from. This is useful is you don't want to wait to 10 | extract all 101 classes. For instance, set class_limit = 8 to just 11 | extract features for the first 8 (alphabetical) classes in the dataset. 12 | Then set the same number when training models. 13 | """ 14 | import numpy as np 15 | import os.path 16 | from data_processor import DataSet 17 | from extractor import Extractor 18 | from tqdm import tqdm 19 | 20 | 21 | # from keras.backend.tensorflow_backend import set_session 22 | # import tensorflow as tf 23 | # config = tf.ConfigProto() 24 | # config.gpu_options.allow_growth = True # dynamically grow the memory used on the GPU 25 | # config.log_device_placement = True # to log device placement (on which device the operation ran) 26 | # sess = tf.Session(config=config) 27 | # set_session(sess) 28 | import tensorflow as tf 29 | config = tf.compat.v1.ConfigProto() 30 | config.gpu_options.allow_growth = True 31 | session = tf.compat.v1.Session(config=config) 32 | 33 | def extract_full_features(weights, seq_length = 40): 34 | # Set defaults. 35 | 36 | class_limit = None # Number of classes to extract. Can be 1-101 or None for all. 37 | 38 | # Get the dataset. 39 | data = DataSet(seq_length=seq_length, class_limit=class_limit, check_dir='data/check') 40 | 41 | # get the model. 42 | # model = Extractor() 43 | # model = Extractor(weights="data/checkpoints/inception.009-0.29.hdf5") 44 | model = Extractor(weights) 45 | 46 | # Loop through data. 47 | print(data.data) 48 | pbar = tqdm(total=len(data.data)) 49 | for video in data.data: 50 | 51 | # Get the path to the sequence for this video. 52 | path = os.path.join('data', 'sequences_test', video[2] + '-' + str(seq_length) + \ 53 | '-features') # numpy will auto-append .npy 54 | 55 | # Check if we already have it. 56 | if os.path.isfile(path + '.npy'): 57 | pbar.update(1) 58 | continue 59 | 60 | # Get the frames for this video. 61 | frames = data.get_frames_for_sample(video) 62 | 63 | # Now downsample to just the ones we need. 64 | # frames = data.rescale_list(frames, seq_length) 65 | 66 | # Now loop through and extract features to build the sequence. 67 | sequence = [] 68 | for image in frames: 69 | features = model.extract(image) 70 | sequence.append(features) 71 | # print(path) 72 | output_dir = os.path.join('data', 'sequences_test') 73 | if not (os.path.exists(output_dir)): 74 | # create the directory you want to save to 75 | os.mkdir(output_dir) 76 | # Save the sequence. 77 | np.save(path, sequence) 78 | 79 | pbar.update(1) 80 | 81 | pbar.close() 82 | 83 | -------------------------------------------------------------------------------- /extractor.py: -------------------------------------------------------------------------------- 1 | from keras.preprocessing import image 2 | from keras.applications.inception_v3 import InceptionV3, preprocess_input 3 | from keras.models import Model, load_model 4 | from keras.layers import Input 5 | import numpy as np 6 | 7 | class Extractor(): 8 | def __init__(self, weights=None): 9 | """Either load pretrained from imagenet, or load our saved 10 | weights from our own training.""" 11 | 12 | self.weights = weights 13 | 14 | if weights is None: 15 | # Get model with pretrained weights. 16 | base_model = InceptionV3( 17 | weights='imagenet', 18 | include_top=True 19 | ) 20 | 21 | # We'll extract features at the final pool layer. 22 | self.model = Model( 23 | inputs=base_model.input, 24 | outputs=base_model.get_layer('avg_pool').output 25 | ) 26 | 27 | else: 28 | # Load the model first. 29 | self.model = load_model(weights) 30 | 31 | # Then remove the top so we get features not predictions. 32 | self.model.layers.pop() 33 | self.model.layers.pop() # two pops to get to pool layer 34 | self.model.outputs = [self.model.layers[-1].output] 35 | self.model.output_layers = [self.model.layers[-1]] 36 | self.model.layers[-1].outbound_nodes = [] 37 | 38 | def extract(self, image_path): 39 | img = image.load_img(image_path, target_size=(299, 299)) 40 | x = image.img_to_array(img) 41 | x = np.expand_dims(x, axis=0) 42 | x = preprocess_input(x) 43 | 44 | # Get the prediction. 45 | features = self.model.predict(x) 46 | 47 | if self.weights is None: 48 | # For imagenet/default network: 49 | features = features[0] 50 | else: 51 | # For loaded network: 52 | features = features[0] 53 | 54 | return features 55 | -------------------------------------------------------------------------------- /models.py: -------------------------------------------------------------------------------- 1 | """ 2 | A collection of models we'll use to attempt to classify videos. 3 | """ 4 | from keras.layers import Dense, Flatten, Dropout, ZeroPadding3D 5 | from keras.layers.recurrent import LSTM 6 | from keras.models import Sequential, load_model 7 | from keras.optimizers import Adam, RMSprop 8 | from keras.layers.wrappers import TimeDistributed 9 | from keras.layers.convolutional import (Conv2D, MaxPooling3D, Conv3D, 10 | MaxPooling2D) 11 | from collections import deque 12 | import sys 13 | from keras.layers import BatchNormalization,Activation 14 | from keras.regularizers import l2 as L2_reg 15 | 16 | class ResearchModels(): 17 | def __init__(self, nb_classes, model, seq_length, 18 | saved_model=None, features_length=2048): 19 | """ 20 | `model` = one of: 21 | lstm 22 | lrcn 23 | mlp 24 | conv_3d 25 | c3d 26 | `seq_length` = the length of our video sequences 27 | `saved_model` = the path to a saved Keras model to load 28 | """ 29 | 30 | # Set defaults. 31 | self.seq_length = seq_length 32 | self.load_model = load_model 33 | self.saved_model = saved_model 34 | self.nb_classes = nb_classes 35 | self.feature_queue = deque() 36 | 37 | metrics = ['accuracy'] 38 | 39 | # Get the appropriate model. 40 | if self.saved_model is not None: 41 | print("Loading model %s" % self.saved_model) 42 | self.model = load_model(self.saved_model) 43 | elif model == 'lstm': 44 | print("Loading LSTM model.") 45 | self.input_shape = (seq_length, features_length) 46 | self.model = self.lstm() 47 | elif model == 'lrcn': 48 | print("Loading CNN-LSTM model.") 49 | self.input_shape = (seq_length, 80, 80, 3) 50 | self.model = self.lrcn() 51 | elif model == 'mlp': 52 | print("Loading simple MLP.") 53 | self.input_shape = (seq_length, features_length) 54 | self.model = self.mlp() 55 | elif model == 'conv_3d': 56 | print("Loading Conv3D") 57 | self.input_shape = (seq_length, 80, 80, 3) 58 | self.model = self.conv_3d() 59 | elif model == 'conv_flow_3d': 60 | print("Loading Flow Conv3D") 61 | self.input_shape = (seq_length, 80, 80, 2) 62 | self.model = self.conv_flow_3d() 63 | elif model == 'c3d': 64 | print("Loading C3D") 65 | self.input_shape = (seq_length, 80, 80, 3) 66 | self.model = self.c3d() 67 | else: 68 | print("Unknown network.") 69 | sys.exit() 70 | 71 | # Now compile the network. 72 | # optimizer = Adam(lr=1e-5, decay=1e-6) 73 | optimizer = Adam(lr=1e-4, decay=1e-6) 74 | 75 | # self.model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=metrics) 76 | self.model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=metrics) 77 | print(self.model.summary()) 78 | 79 | def lstm(self): 80 | """Build a simple LSTM network. We pass the extracted features from 81 | our CNN to this model predomenently.""" 82 | # Model. 83 | model = Sequential() 84 | model.add(LSTM(2048, return_sequences=False, 85 | input_shape=self.input_shape, 86 | dropout=0.5)) 87 | model.add(Dense(512, activation='relu')) 88 | model.add(Dropout(0.5)) 89 | model.add(Dense(self.nb_classes, activation='softmax')) 90 | 91 | return model 92 | 93 | def lrcn(self): 94 | """Build a CNN into RNN. 95 | Starting version from: 96 | https://github.com/udacity/self-driving-car/blob/master/ 97 | steering-models/community-models/chauffeur/models.py 98 | 99 | Heavily influenced by VGG-16: 100 | https://arxiv.org/abs/1409.1556 101 | 102 | Also known as an LRCN: 103 | https://arxiv.org/pdf/1411.4389.pdf 104 | """ 105 | def add_default_block(model, kernel_filters, init, reg_lambda): 106 | 107 | # conv 108 | model.add(TimeDistributed(Conv2D(kernel_filters, (3, 3), padding='same', 109 | kernel_initializer=init, kernel_regularizer=L2_reg(l=reg_lambda)))) 110 | model.add(TimeDistributed(BatchNormalization())) 111 | model.add(TimeDistributed(Activation('relu'))) 112 | # conv 113 | model.add(TimeDistributed(Conv2D(kernel_filters, (3, 3), padding='same', 114 | kernel_initializer=init, kernel_regularizer=L2_reg(l=reg_lambda)))) 115 | model.add(TimeDistributed(BatchNormalization())) 116 | model.add(TimeDistributed(Activation('relu'))) 117 | # max pool 118 | model.add(TimeDistributed(MaxPooling2D((2, 2), strides=(2, 2)))) 119 | 120 | return model 121 | 122 | initialiser = 'glorot_uniform' 123 | reg_lambda = 0.001 124 | 125 | model = Sequential() 126 | 127 | # first (non-default) block 128 | model.add(TimeDistributed(Conv2D(32, (7, 7), strides=(2, 2), padding='same', 129 | kernel_initializer=initialiser, kernel_regularizer=L2_reg(l=reg_lambda)), 130 | input_shape=self.input_shape)) 131 | model.add(TimeDistributed(BatchNormalization())) 132 | model.add(TimeDistributed(Activation('relu'))) 133 | model.add(TimeDistributed(Conv2D(32, (3,3), kernel_initializer=initialiser, kernel_regularizer=L2_reg(l=reg_lambda)))) 134 | model.add(TimeDistributed(BatchNormalization())) 135 | model.add(TimeDistributed(Activation('relu'))) 136 | model.add(TimeDistributed(MaxPooling2D((2, 2), strides=(2, 2)))) 137 | 138 | # 2nd-5th (default) blocks 139 | model = add_default_block(model, 64, init=initialiser, reg_lambda=reg_lambda) 140 | model = add_default_block(model, 128, init=initialiser, reg_lambda=reg_lambda) 141 | model = add_default_block(model, 256, init=initialiser, reg_lambda=reg_lambda) 142 | model = add_default_block(model, 512, init=initialiser, reg_lambda=reg_lambda) 143 | 144 | # LSTM output head 145 | model.add(TimeDistributed(Flatten())) 146 | model.add(LSTM(256, return_sequences=False, dropout=0.5)) 147 | model.add(Dense(self.nb_classes, activation='softmax')) 148 | 149 | return model 150 | 151 | def mlp(self): 152 | """Build a simple MLP. It uses extracted features as the input 153 | because of the otherwise too-high dimensionality.""" 154 | # Model. 155 | model = Sequential() 156 | model.add(Flatten(input_shape=self.input_shape)) 157 | model.add(Dense(512)) 158 | model.add(Dropout(0.5)) 159 | model.add(Dense(512)) 160 | model.add(Dropout(0.5)) 161 | model.add(Dense(self.nb_classes, activation='softmax')) 162 | 163 | return model 164 | 165 | def conv_3d(self): 166 | """ 167 | Build a 3D convolutional network, based loosely on C3D. 168 | https://arxiv.org/pdf/1412.0767.pdf 169 | """ 170 | # Model. 171 | model = Sequential() 172 | model.add(Conv3D( 173 | 32, (3,3,3), activation='relu', input_shape=self.input_shape 174 | )) 175 | model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2))) 176 | model.add(Conv3D(64, (3,3,3), activation='relu')) 177 | model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2))) 178 | model.add(Conv3D(128, (3,3,3), activation='relu')) 179 | model.add(Conv3D(128, (3,3,3), activation='relu')) 180 | model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2))) 181 | model.add(Conv3D(256, (2,2,2), activation='relu')) 182 | model.add(Conv3D(256, (2,2,2), activation='relu')) 183 | model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2))) 184 | 185 | model.add(Flatten()) 186 | model.add(Dense(1024)) 187 | model.add(Dropout(0.5)) 188 | model.add(Dense(1024)) 189 | model.add(Dropout(0.5)) 190 | model.add(Dense(self.nb_classes, activation='softmax')) 191 | 192 | return model 193 | 194 | def conv_flow_3d(self): 195 | model = Sequential() 196 | model.add(Conv3D( 197 | 32, (3,3,3), activation='relu', input_shape=self.input_shape 198 | )) 199 | model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2))) 200 | model.add(Conv3D(64, (3,3,3), activation='relu')) 201 | model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2))) 202 | model.add(Conv3D(128, (3,3,3), activation='relu')) 203 | model.add(Conv3D(128, (3,3,3), activation='relu')) 204 | model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2))) 205 | model.add(Conv3D(256, (2,2,2), activation='relu')) 206 | model.add(Conv3D(256, (2,2,2), activation='relu')) 207 | model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2))) 208 | 209 | model.add(Flatten()) 210 | model.add(Dense(1024)) 211 | model.add(Dropout(0.5)) 212 | model.add(Dense(1024)) 213 | model.add(Dropout(0.5)) 214 | model.add(Dense(self.nb_classes, activation='softmax')) 215 | 216 | return model 217 | 218 | def c3d(self): 219 | """ 220 | Build a 3D convolutional network, aka C3D. 221 | https://arxiv.org/pdf/1412.0767.pdf 222 | 223 | With thanks: 224 | https://gist.github.com/albertomontesg/d8b21a179c1e6cca0480ebdf292c34d2 225 | """ 226 | model = Sequential() 227 | # 1st layer group 228 | model.add(Conv3D(64, 3, 3, 3, activation='relu', 229 | border_mode='same', name='conv1', 230 | subsample=(1, 1, 1), 231 | input_shape=self.input_shape)) 232 | model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2), 233 | border_mode='valid', name='pool1')) 234 | # 2nd layer group 235 | model.add(Conv3D(128, 3, 3, 3, activation='relu', 236 | border_mode='same', name='conv2', 237 | subsample=(1, 1, 1))) 238 | model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), 239 | border_mode='valid', name='pool2')) 240 | # 3rd layer group 241 | model.add(Conv3D(256, 3, 3, 3, activation='relu', 242 | border_mode='same', name='conv3a', 243 | subsample=(1, 1, 1))) 244 | model.add(Conv3D(256, 3, 3, 3, activation='relu', 245 | border_mode='same', name='conv3b', 246 | subsample=(1, 1, 1))) 247 | model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), 248 | border_mode='valid', name='pool3')) 249 | # 4th layer group 250 | model.add(Conv3D(512, 3, 3, 3, activation='relu', 251 | border_mode='same', name='conv4a', 252 | subsample=(1, 1, 1))) 253 | model.add(Conv3D(512, 3, 3, 3, activation='relu', 254 | border_mode='same', name='conv4b', 255 | subsample=(1, 1, 1))) 256 | model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), 257 | border_mode='valid', name='pool4')) 258 | 259 | # 5th layer group 260 | model.add(Conv3D(512, 3, 3, 3, activation='relu', 261 | border_mode='same', name='conv5a', 262 | subsample=(1, 1, 1))) 263 | model.add(Conv3D(512, 3, 3, 3, activation='relu', 264 | border_mode='same', name='conv5b', 265 | subsample=(1, 1, 1))) 266 | model.add(ZeroPadding3D(padding=(0, 1, 1))) 267 | model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), 268 | border_mode='valid', name='pool5')) 269 | model.add(Flatten()) 270 | 271 | # FC layers group 272 | model.add(Dense(4096, activation='relu', name='fc6')) 273 | model.add(Dropout(0.5)) 274 | model.add(Dense(4096, activation='relu', name='fc7')) 275 | model.add(Dropout(0.5)) 276 | model.add(Dense(self.nb_classes, activation='softmax')) 277 | 278 | return model 279 | -------------------------------------------------------------------------------- /optical_flow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | 4 | def getlkOpticalFlowcap(video): 5 | cap = cv2.VideoCapture(video) 6 | 7 | # params for ShiTomasi corner detection 8 | feature_params = dict( maxCorners = 100, 9 | qualityLevel = 0.3, 10 | minDistance = 7, 11 | blockSize = 7 ) 12 | 13 | # Parameters for lucas kanade optical flow 14 | lk_params = dict( winSize = (15,15), 15 | maxLevel = 2, 16 | criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03)) 17 | 18 | # Create some random colors 19 | color = np.random.randint(0,255,(100,3)) 20 | 21 | # Take first frame and find corners in it 22 | ret, old_frame = cap.read() 23 | old_gray = cv2.cvtColor(old_frame, cv2.COLOR_BGR2GRAY) 24 | p0 = cv2.goodFeaturesToTrack(old_gray, mask = None, **feature_params) 25 | 26 | # Create a mask image for drawing purposes 27 | mask = np.zeros_like(old_frame) 28 | 29 | while(1): 30 | ret,frame = cap.read() 31 | frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) 32 | 33 | # calculate optical flow 34 | p1, st, err = cv2.calcOpticalFlowPyrLK(old_gray, frame_gray, p0, None, **lk_params) 35 | 36 | # Select good points 37 | good_new = p1[st==1] 38 | good_old = p0[st==1] 39 | 40 | # draw the tracks 41 | for i,(new,old) in enumerate(zip(good_new,good_old)): 42 | a,b = new.ravel() 43 | c,d = old.ravel() 44 | mask = cv2.line(mask, (a,b),(c,d), color[i].tolist(), 2) 45 | frame = cv2.circle(frame,(a,b),5,color[i].tolist(),-1) 46 | img = cv2.add(frame,mask) 47 | 48 | cv2.imshow('frame',img) 49 | k = cv2.waitKey(30) & 0xff 50 | if k == 27: 51 | break 52 | 53 | # Now update the previous frame and previous points 54 | old_gray = frame_gray.copy() 55 | p0 = good_new.reshape(-1,1,2) 56 | 57 | cv2.destroyAllWindows() 58 | cap.release() 59 | 60 | 61 | def getDenseOpticalFlow(video): 62 | cap = cv2.VideoCapture(video) 63 | ret, frame1 = cap.read() 64 | frame_num = 1 65 | prvs = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY) 66 | hsv = np.zeros_like(frame1) 67 | hsv[..., 1] = 255 68 | 69 | while (1): 70 | ret, frame2 = cap.read() 71 | if frame2 is None: 72 | break 73 | next = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY) 74 | 75 | flow = cv2.calcOpticalFlowFarneback(prvs, next, None, 0.5, 3, 15, 3, 5, 1.2, 0) 76 | 77 | mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1]) 78 | print(flow.shape) 79 | hsv[..., 0] = ang * 180 / np.pi / 2 80 | hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX) 81 | rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR) 82 | 83 | cv2.imshow('frame2', rgb) 84 | cv2.waitKey(30) 85 | # k = cv2.waitKey(30) & 0xff 86 | # if k == 27: 87 | # break 88 | # elif k == ord('s'): 89 | # cv2.imwrite('opticalfb.png', frame2) 90 | # cv2.imwrite('opticalhsv.png', rgb) 91 | # cv2.imwrite('out/opticalfb.png', frame2) 92 | cv2.imwrite('out/opticalhsv_' + str(frame_num) + '.png', rgb) 93 | prvs = next 94 | frame_num += 1 95 | 96 | cap.release() 97 | cv2.destroyAllWindows() 98 | 99 | def main(): 100 | # video = "data/check/smoking/0KqeKi2CBqg.mp4" 101 | video = "data/train/smoking/zigarette_rauchen_smoke_h_nm_np1_fr_med_1.avi" 102 | 103 | 104 | getDenseOpticalFlow(video) 105 | # getlkOpticalFlowcap(video) 106 | 107 | if __name__ == '__main__': 108 | main() -------------------------------------------------------------------------------- /plot_trainlog.py: -------------------------------------------------------------------------------- 1 | """ 2 | Given a training log file, plot something. 3 | """ 4 | import csv 5 | import matplotlib.pyplot as plt 6 | 7 | def main(training_log): 8 | with open(training_log) as fin: 9 | reader = csv.reader(fin) 10 | next(reader, None) # skip the header 11 | val_accuracies = [] 12 | accuracies = [] 13 | val_losses = [] 14 | losses = [] 15 | cnn_benchmark = [] # this is ridiculous 16 | 17 | for epoch,acc,loss,val_accuracy,val_loss in reader: 18 | val_accuracies.append(float(val_accuracy)) 19 | accuracies.append(float(acc)) 20 | # val_losses.append(float(val_loss)) 21 | # losses.append(float(loss)) 22 | 23 | 24 | plt.plot(accuracies) 25 | plt.plot(val_accuracies) 26 | # plt.plot(losses) 27 | # plt.plot(val_losses) 28 | plt.title('model accuracy and loss') 29 | # plt.ylabel('accuracy') 30 | plt.xlabel('epoch') 31 | plt.legend(['train_acc', 'test_acc'], loc='upper left') 32 | # plt.plot(cnn_benchmark) 33 | plt.show() 34 | 35 | if __name__ == '__main__': 36 | training_log = 'data/logs/cnn-training-1587866460.33326.log' 37 | # training_log ="data/logs/conv_3d-training-1586839810.2295735.log" 38 | # main(training_log, 'cnn') 39 | # training_log = 'data/logs/lstm-training-1588036406.0306637.log' 40 | main(training_log) 41 | 42 | -------------------------------------------------------------------------------- /processor.py: -------------------------------------------------------------------------------- 1 | """ 2 | Process an image that we can pass to our networks. 3 | """ 4 | import cv2 5 | from keras.preprocessing.image import img_to_array, load_img 6 | import numpy as np 7 | 8 | def process_image(image, target_shape): 9 | """Given an image, process it and return the array.""" 10 | # Load the image. 11 | h, w, _ = target_shape 12 | image = load_img(image, target_size=(h, w)) 13 | 14 | # Turn it into numpy, normalize and return. 15 | img_arr = img_to_array(image) 16 | x = (img_arr / 255.).astype(np.float32) 17 | 18 | return x 19 | 20 | def process_flow(image1, image2, target_shape): 21 | """Given an image, process it and return the array.""" 22 | # Load the image. 23 | h, w, _ = target_shape 24 | image1 = load_img(image1, target_size=(h, w)) 25 | image2 = load_img(image2, target_size=(h, w)) 26 | img_arr = img_to_array(image1) 27 | x1 = (img_arr / 255.).astype(np.float32) 28 | img_arr = img_to_array(image2) 29 | x2 = (img_arr / 255.).astype(np.float32) 30 | image1 = cv2.cvtColor(x1, cv2.COLOR_BGR2GRAY) 31 | image2 = cv2.cvtColor(x2, cv2.COLOR_BGR2GRAY) 32 | flow = cv2.calcOpticalFlowFarneback(image1, image2, None, 0.5, 3, 15, 3, 5, 1.2, 0) 33 | return flow 34 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Keras 2 | numpy 3 | tqdm 4 | matplotlib 5 | Pillow 6 | h5py 7 | opencv-python 8 | tensorflow==1.15 9 | ffmpeg 10 | youtube_dl 11 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import shutil 3 | from subprocess import call 4 | import cv2 5 | from tqdm import tqdm 6 | from data_processor import DataSet as data 7 | from extractor import Extractor 8 | import numpy as np 9 | import glob 10 | import os.path 11 | import os 12 | import sys 13 | from keras.models import load_model 14 | from matplotlib import pyplot as plt 15 | import json 16 | import math 17 | import tensorflow as tf 18 | sys.path.insert(1, 'data') 19 | 20 | config = tf.compat.v1.ConfigProto() 21 | config.gpu_options.allow_growth = True 22 | session = tf.compat.v1.Session(config=config) 23 | 24 | def main(args): 25 | video_name = args.video_name 26 | seq_length = 50 27 | # weights_path = get_file('lstm-features.hdf5', 'https://1drv.ms/u/s!AjwTYpyMoMlUll4oFEpdBU9dlppN?e=WVXhgu') 28 | # url = 'https://1drv.ms/u/s!AjwTYpyMoMlUll4oFEpdBU9dlppN?e=WVXhgu' 29 | # urllib.request.urlretrieve(url, 'data/checkpoints/lstm-features.hdf5') 30 | # model = load_model('data/checkpoints/lstm-features.009-0.454.hdf5') 31 | # model = load_model('data/checkpoints/lstm-features.004-0.614.hdf5') 32 | # model = load_model('data/checkpoints/lstm-features.017-0.849.hdf5') 33 | 34 | cnn_model = 'cnn.hdf5' 35 | lstm_model = 'lstm.hdf5' 36 | 37 | model = load_model(lstm_model) 38 | extractor = Extractor(weights=cnn_model) 39 | 40 | filename_no_ext = video_name.split('.')[0] 41 | if not os.path.exists('tmp'): 42 | os.makedirs('tmp') 43 | dest = os.path.join('tmp', filename_no_ext + '-%04d.jpg') 44 | call(["ffmpeg", "-i", video_name, "-filter:v", "fps=fps=30", dest], shell=False) 45 | generated_frames = sorted(glob.glob(os.path.join('tmp', filename_no_ext + '*.jpg'))) 46 | nb_frames = len(generated_frames) 47 | print("Generated %d frames for %s" % (nb_frames, filename_no_ext)) 48 | pbar = tqdm(total=len(generated_frames)) 49 | print("\nExtracting features ...") 50 | # Now loop through and extract features to build the sequence. 51 | sequence = [] 52 | for image in generated_frames: 53 | features = extractor.extract(image) 54 | sequence.append(features) 55 | pbar.update(1) 56 | pbar.close() 57 | sequence = np.asarray(sequence) 58 | 59 | shutil.rmtree('tmp') 60 | classes = [] 61 | classes.append('smoking') 62 | classes.append('non_smoking') 63 | classes = sorted(classes) 64 | 65 | output_json = {"smoking": []} 66 | 67 | total = sequence.shape[0] 68 | frames = np.arange(total) 69 | frame_pred = np.ones(total) 70 | frame_pred_sum = np.zeros(total) 71 | frame_pred_count = np.zeros(total) 72 | # print("Size : " + str(total)) 73 | frame_pred_prob = np.empty(total) 74 | frame_pred_prob[:] = np.nan 75 | skip_clips = 15 #100 76 | skip_frames = 2 #6 77 | start_frame = 0 78 | end_frame = skip_frames*seq_length 79 | # end_frame = 250 80 | print("Number of frames: ", total ) 81 | label_predictions = {} 82 | if end_frame > sequence.shape[0]: 83 | sequences = data.rescale_list(sequence, seq_length) 84 | X = [] 85 | X.append(sequences) 86 | predictions = model.predict(np.array(X), batch_size=1) 87 | label_predictions = {} 88 | 89 | for i, label in enumerate(classes): 90 | label_predictions[label] = predictions[0][i] 91 | # print(label_predictions) 92 | # if label_predictions["smoking"] <= 0.5: 93 | # frame_pred[start_frame:total] = 0 94 | for i in range(start_frame, total): 95 | frame_pred_sum[i] += label_predictions["smoking"] 96 | frame_pred_count[i] += 1 97 | # else: 98 | # frame_pred[start_frame:total] = -1 99 | 100 | # frame_pred_prob[start_frame:total] = str(label_predictions["smoking"]) 101 | 102 | else: 103 | while end_frame <= sequence.shape[0]: 104 | 105 | X = [] 106 | x = [] 107 | for i in range(start_frame, end_frame, skip_frames): 108 | x.append(sequence[i,:]) 109 | X.append(x) 110 | # print("video: " + video[2] + " start frame: " + str(start_frame) + " end frame: " + str(end_frame)) 111 | # X.append(sequences[start_frame: end_frame,:]) 112 | # sequence = sequence.reshape(1, 3, 3) 113 | predictions = model.predict(np.array(X), batch_size=1) 114 | label_predictions = {} 115 | for i, label in enumerate(classes): 116 | # print(predictions) 117 | label_predictions[label] = predictions[0][i] 118 | # print(label_predictions) 119 | # if label_predictions["smoking"] <= 0.5: 120 | # frame_pred[start_frame:end_frame] = 0 121 | for i in range(start_frame, end_frame): 122 | frame_pred_sum[i] += label_predictions["smoking"] 123 | frame_pred_count[i] += 1 124 | # else: 125 | # frame_pred[start_frame:end_frame] = 0 126 | 127 | # frame_pred_prob[start_frame:end_frame] = str(label_predictions["smoking"]) 128 | 129 | start_frame += skip_clips 130 | end_frame += skip_clips 131 | 132 | for i in range(start_frame, min(sequence.shape[0], end_frame-1)): 133 | frame_pred_sum[i] += label_predictions["smoking"] 134 | frame_pred_count[i] += 1 135 | # 136 | # for i in range(start_frame, min(sequences.shape[0], end_frame-1)): 137 | # # frame_pred_prob.append(str(label_predictions["smoking"])) 138 | # frame_pred_prob[i] = str(label_predictions["smoking"]) 139 | # if label_predictions["smoking"] <= 0.5: 140 | # frame_pred[i] = 0 141 | # print(frame_pred) 142 | for i in range(0,total): 143 | frame_pred_prob[i] = frame_pred_sum[i]/frame_pred_count[i] 144 | if frame_pred_prob[i] < 0.5: 145 | frame_pred[i] = 0 146 | 147 | plt.title("Smoking action detection") 148 | plt.xlabel("Time (in seconds)") 149 | plt.ylabel("Smoking label") 150 | plt.plot(frames, frame_pred) 151 | plt.yticks(np.arange(2)) 152 | xlabel_count = 10 153 | ticker_len = 1 154 | if len(frames)/30 > xlabel_count: 155 | ticker_len = math.ceil(len(frames)/(30*xlabel_count)) 156 | plt.xticks(np.arange(min(frames), max(frames) + ticker_len, (30 * ticker_len)), 157 | np.arange(int(min(frames) / 30), int(max(frames) / 30) + 30, ticker_len)) 158 | 159 | output_path = "timeLabel.jpg" 160 | print("Saving output labels to: ", output_path) 161 | 162 | plt.savefig(output_path) 163 | plt.close() 164 | # plt.show() 165 | # plt.figure() 166 | frame_time = [frame/30 for frame in frames] 167 | output_json["smoking"] = list(zip(frame_time, frame_pred_prob)) 168 | y = json.dumps(output_json) 169 | # with open('frameLabel.json', 'w') as outfile: 170 | # json.dump(y, outfile) 171 | output_path = "timeLabel.json" 172 | print(y) 173 | with open(output_path, 'w') as outfile: 174 | json.dump(output_json, outfile) 175 | print('Output JSON saved under {}'.format(output_path)) 176 | label_video(video_name, frame_pred, frame_pred_prob) 177 | 178 | def label_video(video_name, labels, label_prob): 179 | filename_no_ext = video_name.split('.')[0] 180 | print('Starting: {}'.format(filename_no_ext)) 181 | start_frame = 0 182 | end_frame = None 183 | reader = cv2.VideoCapture(video_name) 184 | video_fn = filename_no_ext+'_output.avi' 185 | fourcc = cv2.VideoWriter_fourcc(*'MJPG') 186 | fps = reader.get(cv2.CAP_PROP_FPS) 187 | num_frames = int(reader.get(cv2.CAP_PROP_FRAME_COUNT)) 188 | print("Total number of frames ", str(num_frames)) 189 | writer = None 190 | # Text variables 191 | font_face = cv2.FONT_HERSHEY_SIMPLEX 192 | thickness = 2 193 | font_scale = 0.35 194 | 195 | # Frame numbers and length of output video 196 | frame_num = 0 197 | assert start_frame < num_frames - 1 198 | end_frame = end_frame if end_frame else num_frames 199 | pbar = tqdm(total=end_frame-start_frame) 200 | 201 | while reader.isOpened(): 202 | _, image = reader.read() 203 | if image is None: 204 | break 205 | frame_num += 1 206 | 207 | if frame_num < start_frame or frame_num-1 >= len(labels): 208 | continue 209 | pbar.update(1) 210 | 211 | # Image size 212 | height, width = image.shape[:2] 213 | 214 | # Init output writer 215 | if writer is None: 216 | writer = cv2.VideoWriter( video_fn, fourcc, fps, 217 | (height, width)[::-1]) 218 | # print("frame num " + str(frame_num)) 219 | label = 'smoking' if labels[frame_num-1] == 1 else 'non_smoking' 220 | color = (0, 255, 0) if labels[frame_num-1] == 0 else (0, 0, 255) 221 | # cv2.putText(image, 'Label =>' + str(label), (x, y + h + 30), 222 | # font_face, font_scale, 223 | # color, thickness, 2) 224 | cv2.putText(image, 'Frame: ' + str(frame_num) + ', Label: ' + str(label) + 225 | ', Prob =>' + str(label_prob[frame_num-1]), (10, 20), 226 | font_face, font_scale, 227 | color, thickness, 2) 228 | # cv2.putText(image, 'Frame num: ' + str(frame_num) + ', Label =>' + str(label) + 229 | # ', Prob =>' + label_prob[frame_num-1], (10, 20), 230 | # font_face, font_scale, 231 | # color, thickness, 2) 232 | if frame_num >= end_frame: 233 | break 234 | cv2.imshow('test', image) 235 | cv2.waitKey(1) # About 30 fps 236 | writer.write(image) 237 | pbar.close() 238 | if writer is not None: 239 | writer.release() 240 | print('Finished! Output saved under {}'.format(video_fn)) 241 | else: 242 | print('Input video file was empty') 243 | 244 | if __name__ == "__main__": 245 | parser = argparse.ArgumentParser("Run the smoking detection model.") 246 | parser.add_argument("-v", "--video_name", required=True, help="Path to testing video") 247 | parsed = parser.parse_args() 248 | main(parsed) 249 | 250 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #Replace the variables with your github repo url, repo name, test video name, json named by your UIN 3 | GIT_REPO_URL="https://github.com/harshita-chaudhary/smoking_detection.git" 4 | REPO="smoking_detection" 5 | VIDEO="test_video1.mp4" 6 | UIN_JSON="529005682.json" 7 | UIN_JPG="529005682.jpg" 8 | git clone $GIT_REPO_URL 9 | cd $REPO 10 | # cp ../$VIDEO . 11 | #Replace this line with commands for running your test python file. 12 | pip install -r requirements.txt 13 | youtube-dl -f best -f mp4 "https://www.youtube.com/watch?v=OCT3Y3BhrLo" -o $VIDEO 14 | # Install ffmpeg 15 | sudo apt install ffmpeg 16 | echo $VIDEO 17 | chmod +x download_models.sh 18 | ./download_models.sh 19 | python test.py --video_name $VIDEO 20 | #rename the generated timeLabel.json and figure with your UIN. 21 | cp timeLabel.json ../$UIN_JSON 22 | cp timeLabel.jpg ../$UIN_JPG 23 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | """ 2 | Train our RNN on extracted features or images. 3 | """ 4 | from keras.callbacks import TensorBoard, ModelCheckpoint, EarlyStopping, CSVLogger 5 | from models import ResearchModels 6 | from data_processor import DataSet 7 | import time 8 | import os.path 9 | 10 | def train(data_type, seq_length, model, saved_model=None, 11 | class_limit=None, image_shape=None, 12 | load_to_memory=False, batch_size=32, nb_epoch=100): 13 | # Helper: Save the model. 14 | checkpointer = ModelCheckpoint( 15 | filepath=os.path.join('data', 'checkpoints', model + '-' + data_type + \ 16 | '.{epoch:03d}-{val_accuracy:.3f}.hdf5'), 17 | monitor='val_accuracy', 18 | verbose=1, 19 | save_best_only=True) 20 | 21 | # Helper: TensorBoard 22 | tb = TensorBoard(log_dir=os.path.join('data', 'logs', model)) 23 | 24 | # Helper: Stop when we stop learning. 25 | early_stopper = EarlyStopping(patience=50, monitor='val_accuracy') 26 | 27 | # Helper: Save results. 28 | timestamp = time.time() 29 | csv_logger = CSVLogger(os.path.join('data', 'logs', model + '-' + 'training-' + \ 30 | str(timestamp) + '.log')) 31 | 32 | # Get the data and process it. 33 | if image_shape is None: 34 | data = DataSet( 35 | seq_length=seq_length, 36 | class_limit=class_limit 37 | ) 38 | else: 39 | data = DataSet( 40 | seq_length=seq_length, 41 | class_limit=class_limit, 42 | image_shape=image_shape 43 | ) 44 | 45 | # Get samples per epoch. 46 | # Multiply by 0.7 to attempt to guess how much of data.data is the train set. 47 | steps_per_epoch = (len(data.data) * 0.7) // batch_size 48 | 49 | if load_to_memory: 50 | # Get data. 51 | X, y = data.get_all_sequences_in_memory('train', data_type) 52 | X_test, y_test = data.get_all_sequences_in_memory('test', data_type) 53 | else: 54 | # Get generators. 55 | generator = data.frame_generator(batch_size, 'train', data_type) 56 | val_generator = data.frame_generator(batch_size, 'test', data_type) 57 | 58 | # Get the model. 59 | rm = ResearchModels(len(data.classes), model, seq_length, saved_model) 60 | 61 | # Fit! 62 | if load_to_memory: 63 | # Use standard fit. 64 | rm.model.fit( 65 | X, 66 | y, 67 | batch_size=batch_size, 68 | validation_data=(X_test, y_test), 69 | verbose=1, 70 | # callbacks=[tb, early_stopper, csv_logger], 71 | callbacks=[early_stopper, csv_logger, checkpointer], 72 | epochs=nb_epoch) 73 | else: 74 | # Use fit generator. 75 | rm.model.fit_generator( 76 | generator=generator, 77 | steps_per_epoch=steps_per_epoch, 78 | epochs=nb_epoch, 79 | verbose=1, 80 | # callbacks=[tb, early_stopper, csv_logger, checkpointer], 81 | callbacks=[ early_stopper, checkpointer], 82 | validation_data=val_generator, 83 | validation_steps=40, 84 | workers=4) 85 | 86 | def main(): 87 | """These are the main training settings. Set each before running 88 | this file.""" 89 | # model can be one of lstm, lrcn, mlp, conv_3d, c3d 90 | model = 'lstm' 91 | # model = 'conv_flow_3d' 92 | saved_model = None # None or weights file 93 | class_limit = None # int, can be 1-101 or None 94 | seq_length = 50 95 | load_to_memory = True # pre-load the sequences into memory 96 | # load_to_memory = False # pre-load the sequences into memory 97 | 98 | # batch_size = 32 99 | batch_size = 16 100 | nb_epoch = 1000 101 | 102 | # Chose images or features and image shape based on network. 103 | if model in ['conv_3d', 'c3d', 'lrcn', 'conv_flow_3d']: 104 | data_type = 'images' 105 | image_shape = (80, 80, 3) 106 | elif model in ['lstm', 'mlp']: 107 | data_type = 'features' 108 | image_shape = None 109 | else: 110 | raise ValueError("Invalid model. See train.py for options.") 111 | 112 | # data_type = "flow" 113 | 114 | train(data_type, seq_length, model, saved_model=saved_model, 115 | class_limit=class_limit, image_shape=image_shape, 116 | load_to_memory=load_to_memory, batch_size=batch_size, nb_epoch=nb_epoch) 117 | 118 | if __name__ == '__main__': 119 | main() 120 | -------------------------------------------------------------------------------- /train_cnn.py: -------------------------------------------------------------------------------- 1 | """ 2 | Train on images split into directories. This assumes we've split 3 | our videos into frames and moved them to their respective folders. 4 | 5 | Based on: 6 | https://keras.io/preprocessing/image/ 7 | and 8 | https://keras.io/applications/ 9 | """ 10 | import time 11 | 12 | from keras.applications.inception_v3 import InceptionV3 13 | from keras.optimizers import SGD 14 | from keras.preprocessing.image import ImageDataGenerator 15 | from keras.models import Model 16 | from keras.layers import Dense, GlobalAveragePooling2D 17 | from keras.callbacks import ModelCheckpoint, EarlyStopping, CSVLogger 18 | from data_processor import DataSet 19 | import os.path 20 | 21 | data = DataSet() 22 | 23 | # Helper: Save the model. 24 | checkpointer = ModelCheckpoint( 25 | # filepath=os.path.join('data', 'checkpoints', 'inception.{epoch:03d}-{val_accuracy:.2f}.hdf5'), 26 | filepath=os.path.join('data', 'checkpoints', 'inception.hdf5'), 27 | verbose=1, 28 | monitor='val_accuracy', 29 | save_best_only=True) 30 | 31 | # Helper: Stop when we stop learning. 32 | early_stopper = EarlyStopping(patience=15, monitor='val_accuracy') 33 | # early_stopper = EarlyStopping(patience=10) 34 | 35 | # Helper: TensorBoard 36 | # tensorboard = TensorBoard(log_dir=os.path.join('data', 'logs')) 37 | 38 | timestamp = time.time() 39 | csv_logger = CSVLogger(os.path.join('data', 'logs', 'cnn' + '-' + 'training-' + \ 40 | str(timestamp) + '.log')) 41 | 42 | 43 | def get_generators(): 44 | train_datagen = ImageDataGenerator( 45 | rescale=1. / 255, 46 | shear_range=0.2, 47 | horizontal_flip=True, 48 | rotation_range=10., 49 | width_shift_range=0.2, 50 | height_shift_range=0.2) 51 | 52 | test_datagen = ImageDataGenerator(rescale=1./255) 53 | 54 | train_generator = train_datagen.flow_from_directory( 55 | os.path.join('data', 'train'), 56 | target_size=(299, 299), 57 | batch_size=32, 58 | classes=data.classes, 59 | class_mode='categorical') 60 | 61 | validation_generator = test_datagen.flow_from_directory( 62 | os.path.join('data', 'test'), 63 | target_size=(299, 299), 64 | batch_size=32, 65 | classes=data.classes, 66 | class_mode='categorical') 67 | 68 | return train_generator, validation_generator 69 | 70 | 71 | def get_model(weights='imagenet'): 72 | # create the base pre-trained model 73 | base_model = InceptionV3(weights=weights, include_top=False) 74 | 75 | # add a global spatial average pooling layer 76 | x = base_model.output 77 | x = GlobalAveragePooling2D()(x) 78 | # let's add a fully-connected layer 79 | x = Dense(1024, activation='relu')(x) 80 | # and a logistic layer 81 | predictions = Dense(len(data.classes), activation='softmax')(x) 82 | 83 | # this is the model we will train 84 | model = Model(inputs=base_model.input, outputs=predictions) 85 | model.summary() 86 | return model 87 | 88 | 89 | def freeze_all_but_top(model): 90 | """Used to train just the top layers of the model.""" 91 | # first: train only the top layers (which were randomly initialized) 92 | # i.e. freeze all convolutional InceptionV3 layers 93 | for layer in model.layers[:-2]: 94 | layer.trainable = False 95 | 96 | # compile the model (should be done *after* setting layers to non-trainable) 97 | model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy']) 98 | 99 | return model 100 | 101 | 102 | def freeze_all_but_mid_and_top(model): 103 | """After we fine-tune the dense layers, train deeper.""" 104 | # we chose to train the top 2 inception blocks, i.e. we will freeze 105 | # the first 172 layers and unfreeze the rest: 106 | for layer in model.layers[:172]: 107 | layer.trainable = False 108 | for layer in model.layers[172:]: 109 | layer.trainable = True 110 | 111 | # we need to recompile the model for these modifications to take effect 112 | # we use SGD with a low learning rate 113 | model.compile( 114 | optimizer=SGD(lr=0.0001, momentum=0.9), 115 | loss='binary_crossentropy', 116 | metrics=['accuracy']) 117 | 118 | return model 119 | 120 | def train_model(model, nb_epoch, generators, callbacks=[]): 121 | train_generator, validation_generator = generators 122 | model.fit_generator( 123 | train_generator, 124 | steps_per_epoch=100, 125 | validation_data=validation_generator, 126 | validation_steps=10, epochs=nb_epoch, callbacks=callbacks) 127 | # epochs=nb_epoch, 128 | # callbacks=callbacks) 129 | return model 130 | 131 | 132 | def main(weights_file): 133 | model = get_model() 134 | generators = get_generators() 135 | 136 | if weights_file is None: 137 | print("Loading network from ImageNet weights.") 138 | # Get and train the top layers. 139 | model = freeze_all_but_top(model) 140 | # model = train_model(model, 10, generators) 141 | model = train_model(model, 10, generators) 142 | 143 | else: 144 | print("Loading saved model: %s." % weights_file) 145 | model.load_weights(weights_file) 146 | 147 | # Get and train the mid layers. 148 | model = freeze_all_but_mid_and_top(model) 149 | # model = train_model(model, 1000, generators,[checkpointer, early_stopper, tensorboard, csv_logger]) 150 | model = train_model(model, 1000, generators, [checkpointer, early_stopper, csv_logger]) 151 | 152 | # [checkpointer, early_stopper, tensorboard]) 153 | 154 | if __name__ == '__main__': 155 | weights_file = None 156 | main(weights_file) 157 | -------------------------------------------------------------------------------- /validate_cnn.py: -------------------------------------------------------------------------------- 1 | """ 2 | Classify a few images through our CNN. 3 | """ 4 | import numpy as np 5 | import operator 6 | import random 7 | import glob 8 | import os.path 9 | from data_processor import DataSet 10 | from processor import process_image 11 | from keras.models import load_model 12 | 13 | def main(nb_images=5): 14 | """Spot-check `nb_images` images.""" 15 | data = DataSet() 16 | model = load_model('data/checkpoints/inception.035-0.17.hdf5') 17 | # Get all our test images. 18 | images = glob.glob(os.path.join('data', 'test', '**', '*.jpg')) 19 | nb_images = 500 20 | for _ in range(nb_images): 21 | print('-'*80) 22 | # Get a random row. 23 | sample = random.randint(0, len(images) - 1) 24 | image = images[sample] 25 | 26 | # Turn the image into an array. 27 | print(image) 28 | image_arr = process_image(image, (299, 299, 3)) 29 | image_arr = np.expand_dims(image_arr, axis=0) 30 | 31 | # Predict. 32 | predictions = model.predict(image_arr) 33 | 34 | # Show how much we think it's each one. 35 | label_predictions = {} 36 | for i, label in enumerate(data.classes): 37 | label_predictions[label] = predictions[0][i] 38 | 39 | sorted_lps = sorted(label_predictions.items(), key=operator.itemgetter(1), reverse=True) 40 | print(label_predictions) 41 | print(sorted_lps) 42 | for i, class_prediction in enumerate(sorted_lps): 43 | # Just get the top five. 44 | # if i > 4: 45 | # break 46 | print("%s: %.2f" % (class_prediction[0], class_prediction[1])) 47 | i += 1 48 | 49 | if __name__ == '__main__': 50 | main() 51 | -------------------------------------------------------------------------------- /validate_model.py: -------------------------------------------------------------------------------- 1 | """ 2 | Validate our RNN. Basically just runs a validation generator on 3 | about the same number of videos as we have in our test set. 4 | """ 5 | import urllib 6 | 7 | import cv2 8 | from keras.callbacks import TensorBoard, ModelCheckpoint, CSVLogger 9 | from tqdm import tqdm 10 | import math 11 | from models import ResearchModels 12 | import numpy as np 13 | import operator 14 | import random 15 | import glob 16 | import os.path 17 | import os 18 | from data_processor import DataSet 19 | import sys 20 | from keras.models import load_model 21 | from matplotlib import pyplot as plt 22 | import json 23 | sys.path.insert(1, 'data') 24 | from data.extract_files import extract_files 25 | from extract_features_full import extract_full_features 26 | from keras.utils.data_utils import get_file 27 | 28 | 29 | def validate(data_type, model, seq_length=40, saved_model=None, 30 | class_limit=None, image_shape=None): 31 | batch_size = 32 32 | 33 | # Get the data and process it. 34 | if image_shape is None: 35 | data = DataSet( 36 | seq_length=seq_length, 37 | class_limit=class_limit 38 | ) 39 | else: 40 | data = DataSet( 41 | seq_length=seq_length, 42 | class_limit=class_limit, 43 | image_shape=image_shape 44 | ) 45 | 46 | val_generator = data.frame_generator(batch_size, 'test', data_type) 47 | 48 | # Get the model. 49 | rm = ResearchModels(len(data.classes), model, seq_length, saved_model) 50 | 51 | # Evaluate! 52 | results = rm.model.evaluate_generator( 53 | generator=val_generator, 54 | val_samples=3200) 55 | 56 | print(results) 57 | print(rm.model.metrics_names) 58 | 59 | 60 | def main(videos=5): 61 | seq_length = 50 62 | # weights_path = get_file('lstm-features.hdf5', 'https://1drv.ms/u/s!AjwTYpyMoMlUll4oFEpdBU9dlppN?e=WVXhgu') 63 | # url = 'https://1drv.ms/u/s!AjwTYpyMoMlUll4oFEpdBU9dlppN?e=WVXhgu' 64 | # urllib.request.urlretrieve(url, 'data/checkpoints/lstm-features.hdf5') 65 | # model = load_model('data/checkpoints/lstm-features.009-0.454.hdf5') 66 | # model = load_model('data/checkpoints/lstm-features.004-0.614.hdf5') 67 | 68 | 69 | cnn_model = 'data/checkpoints/inception.hdf5' 70 | # lstm_model = 'data/checkpoints/lstm-features.086-0.895.hdf5' 71 | # lstm_model = 'data/checkpoints/lstm-features.051-0.902.hdf5' 72 | lstm_model = 'data/checkpoints/lstm-features.017-0.849.hdf5' 73 | 74 | extract_files(folders=['check']) 75 | extract_full_features(weights=cnn_model, seq_length=seq_length) 76 | model = load_model(lstm_model) 77 | 78 | output_json = {} 79 | output_json["smoking"] = [] 80 | test_dir = "check" 81 | data = DataSet(seq_length=seq_length, check_dir='check') 82 | random.shuffle(data.data) 83 | 84 | # model = load_model('data/checkpoints/inception.057-1.16.hdf5') 85 | for video in data.data: 86 | X, y = [], [] 87 | sequences = data.get_extracted_sequence('features', video) 88 | total = sequences.shape[0] 89 | frames = np.arange(total) 90 | frame_pred = np.ones(total) 91 | frame_pred_sum = np.zeros(total) 92 | frame_pred_count = np.zeros(total) 93 | # print("Size : " + str(total)) 94 | frame_pred_prob = np.empty(total) 95 | frame_pred_prob[:] = np.nan 96 | # X.append(sequence) 97 | y.append(data.get_class_one_hot(video[1])) 98 | print(y) 99 | print("video: " + video[2]) 100 | skip_clips = 50 #100 101 | skip_frames = 2 #6 102 | start_frame = 0 103 | end_frame = skip_frames*seq_length 104 | # end_frame = 250 105 | print("Number of frames: ", total ) 106 | label_predictions = {} 107 | if end_frame > sequences.shape[0]: 108 | sequences = data.rescale_list(sequences, seq_length) 109 | X = [] 110 | X.append(sequences) 111 | predictions = model.predict(np.array(X), batch_size=1) 112 | label_predictions = {} 113 | for i, label in enumerate(data.classes): 114 | # print(predictions) 115 | label_predictions[label] = predictions[0][i] 116 | # print(label_predictions) 117 | # if label_predictions["smoking"] <= 0.5: 118 | # frame_pred[start_frame:total] = 0 119 | for i in range(start_frame, total): 120 | frame_pred_sum[i] += label_predictions["smoking"] 121 | frame_pred_count[i] += 1 122 | # else: 123 | # frame_pred[start_frame:total] = -1 124 | 125 | # frame_pred_prob[start_frame:total] = str(label_predictions["smoking"]) 126 | 127 | else: 128 | while end_frame <= sequences.shape[0]: 129 | 130 | X = [] 131 | x = [] 132 | for i in range(start_frame, end_frame, skip_frames): 133 | x.append(sequences[i,:]) 134 | X.append(x) 135 | # print("video: " + video[2] + " start frame: " + str(start_frame) + " end frame: " + str(end_frame)) 136 | # X.append(sequences[start_frame: end_frame,:]) 137 | # sequence = sequence.reshape(1, 3, 3) 138 | predictions = model.predict(np.array(X), batch_size=1) 139 | label_predictions = {} 140 | for i, label in enumerate(data.classes): 141 | # print(predictions) 142 | label_predictions[label] = predictions[0][i] 143 | # print(label_predictions) 144 | # if label_predictions["smoking"] <= 0.5: 145 | # frame_pred[start_frame:end_frame] = 0 146 | for i in range(start_frame, end_frame): 147 | frame_pred_sum[i] += label_predictions["smoking"] 148 | frame_pred_count[i] += 1 149 | # else: 150 | # frame_pred[start_frame:end_frame] = 0 151 | 152 | # frame_pred_prob[start_frame:end_frame] = str(label_predictions["smoking"]) 153 | 154 | start_frame += skip_clips 155 | end_frame += skip_clips 156 | 157 | for i in range(start_frame, min(sequences.shape[0], end_frame-1)): 158 | frame_pred_sum[i] += label_predictions["smoking"] 159 | frame_pred_count[i] += 1 160 | # 161 | # for i in range(start_frame, min(sequences.shape[0], end_frame-1)): 162 | # # frame_pred_prob.append(str(label_predictions["smoking"])) 163 | # frame_pred_prob[i] = str(label_predictions["smoking"]) 164 | # if label_predictions["smoking"] <= 0.5: 165 | # frame_pred[i] = 0 166 | # print(frame_pred) 167 | for i in range(0,total): 168 | frame_pred_prob[i] = frame_pred_sum[i]/frame_pred_count[i] 169 | if frame_pred_prob[i] < 0.5: 170 | frame_pred[i] = 0 171 | 172 | plt.title("Smoking action detection") 173 | plt.xlabel("Frame") 174 | plt.ylabel("Smoking action present") 175 | plt.plot(frames, frame_pred) 176 | 177 | output_path = os.path.join('data', 'out', video[2] + '.png') 178 | print("Saving output labels to: ", output_path) 179 | 180 | plt.savefig(output_path) 181 | plt.close() 182 | # plt.show() 183 | # plt.figure() 184 | output_json["smoking"] = list(zip(frames.tolist(), frame_pred_prob)) 185 | y = json.dumps(output_json) 186 | # with open('frameLabel.json', 'w') as outfile: 187 | # json.dump(y, outfile) 188 | output_path = os.path.join('data','out',video[2] + '.json') 189 | print(y) 190 | with open(output_path, 'w') as outfile: 191 | json.dump(output_json, outfile) 192 | print('Output JSON saved under {}'.format(output_path)) 193 | label_video(video, frame_pred, frame_pred_prob) 194 | 195 | def label_video(video_path, labels, label_prob): 196 | print('Starting: {}'.format(video_path[2])) 197 | output_path = os.path.join('data', 'out') 198 | # Read and write 199 | 200 | start_frame = 0 201 | end_frame = None 202 | filepath = os.path.join('data', video_path[0], video_path[1], video_path[2] + "." + video_path[4]) 203 | reader = cv2.VideoCapture(filepath) 204 | # video_fn = video + '.avi' 205 | video_fn = video_path[2]+'.avi' 206 | os.makedirs(output_path, exist_ok=True) 207 | fourcc = cv2.VideoWriter_fourcc(*'MJPG') 208 | fps = reader.get(cv2.CAP_PROP_FPS) 209 | num_frames = int(reader.get(cv2.CAP_PROP_FRAME_COUNT)) 210 | writer = None 211 | # Text variables 212 | font_face = cv2.FONT_HERSHEY_SIMPLEX 213 | thickness = 2 214 | font_scale = 0.35 215 | print("Filepath : ", filepath) 216 | print("Total number of frames ", str(num_frames) ) 217 | # Frame numbers and length of output video 218 | frame_num = 0 219 | assert start_frame < num_frames - 1 220 | end_frame = end_frame if end_frame else num_frames 221 | pbar = tqdm(total=end_frame-start_frame) 222 | 223 | while reader.isOpened(): 224 | _, image = reader.read() 225 | if image is None: 226 | break 227 | frame_num += 1 228 | 229 | if frame_num < start_frame or frame_num-1 >= len(labels): 230 | continue 231 | pbar.update(1) 232 | 233 | # Image size 234 | height, width = image.shape[:2] 235 | 236 | # Init output writer 237 | if writer is None: 238 | writer = cv2.VideoWriter(os.path.join(output_path, video_fn), fourcc, fps, 239 | (height, width)[::-1]) 240 | # print("frame num " + str(frame_num)) 241 | label = 'smoking' if labels[frame_num-1] == 1 else 'non_smoking' 242 | color = (0, 255, 0) if labels[frame_num-1] == 0 else (0, 0, 255) 243 | # cv2.putText(image, 'Label =>' + str(label), (x, y + h + 30), 244 | # font_face, font_scale, 245 | # color, thickness, 2) 246 | cv2.putText(image, 'Frame: ' + str(frame_num) + ', Label: ' + str(label) + 247 | ', Prob =>' + str(label_prob[frame_num-1]), (10, 20), 248 | font_face, font_scale, 249 | color, thickness, 2) 250 | # cv2.putText(image, 'Frame num: ' + str(frame_num) + ', Label =>' + str(label) + 251 | # ', Prob =>' + label_prob[frame_num-1], (10, 20), 252 | # font_face, font_scale, 253 | # color, thickness, 2) 254 | if frame_num >= end_frame: 255 | break 256 | 257 | cv2.imshow('test', image) 258 | cv2.waitKey(1) # About 30 fps 259 | writer.write(image) 260 | pbar.close() 261 | if writer is not None: 262 | writer.release() 263 | print('Finished! Output saved under {}'.format(output_path)) 264 | else: 265 | print('Input video file was empty') 266 | 267 | if __name__ == '__main__': 268 | main() 269 | -------------------------------------------------------------------------------- /validate_rnn.py: -------------------------------------------------------------------------------- 1 | """ 2 | Validate our RNN. Basically just runs a validation generator on 3 | about the same number of videos as we have in our test set. 4 | """ 5 | from keras.callbacks import TensorBoard, ModelCheckpoint, CSVLogger 6 | from models import ResearchModels 7 | from data_processor import DataSet 8 | import numpy as np 9 | from sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix 10 | from sklearn import svm, linear_model 11 | from sklearn.model_selection import GridSearchCV 12 | 13 | 14 | def validate(data_type, model, seq_length=50, saved_model=None, 15 | class_limit=None, image_shape=None, train_test='test'): 16 | # batch_size = 32 17 | batch_size = 1 18 | 19 | 20 | # Get the data and process it. 21 | if image_shape is None: 22 | data = DataSet( 23 | seq_length=seq_length, 24 | class_limit=class_limit 25 | ) 26 | else: 27 | data = DataSet( 28 | seq_length=seq_length, 29 | class_limit=class_limit, 30 | image_shape=image_shape 31 | ) 32 | 33 | # _, test = data.split_train_test() 34 | # size = len(test) 35 | # val_generator = data.frame_generator(batch_size, 'test', data_type) 36 | 37 | # Get the model. 38 | rm = ResearchModels(len(data.classes), model, seq_length, saved_model) 39 | rm.model.layers.pop() 40 | rm.model.outputs = [rm.model.layers[-2].output] 41 | rm.model.output_layers = [rm.model.layers[-2]] 42 | rm.model.layers[-2].outbound_nodes = [] 43 | # X = rm.layers[-1].output 44 | # self.model.layers.pop() # two pops to get to pool layer 45 | # self.model.outputs = [self.model.layers[-1].output] 46 | 47 | X, y = data.get_data_train_test(data_type, train_test) 48 | size = len(X) 49 | 50 | # Evaluate! 51 | # results = rm.model.evaluate_generator( 52 | # generator=val_generator, 53 | # val_samples=3200) 54 | # 55 | # print(results) 56 | # print(rm.model.metrics_names) 57 | 58 | # results = rm.model.predict_generator( 59 | # generator=val_generator, 60 | # val_samples=size, 61 | # # val_samples=3200, 62 | # verbose=1) 63 | 64 | results = rm.model.predict( 65 | X, 66 | # val_samples=size, 67 | # val_samples=3200, 68 | verbose=1) 69 | 70 | print(results.shape) 71 | 72 | return (results, y) 73 | # print(results) 74 | # print(rm.model.metrics_names) 75 | 76 | def main(train_test): 77 | 78 | model = 'conv_flow_3d' 79 | saved_model = 'data/checkpoints/conv_flow_3d-flow.035-0.715.hdf5' 80 | 81 | if model == 'conv_flow_3d' or model == 'lrcn': 82 | data_type = 'flow' 83 | image_shape = (80, 80, 3) 84 | else: 85 | data_type = 'features' 86 | image_shape = None 87 | 88 | 89 | conv_flow_3d_results, y_conv_flow = validate(data_type, model, saved_model=saved_model, 90 | image_shape=image_shape, class_limit=4, train_test=train_test) 91 | 92 | model = 'conv_3d' 93 | saved_model = 'data/checkpoints/conv_3d-images.018-0.616.hdf5' 94 | 95 | if model == 'conv_3d' or model == 'lrcn': 96 | data_type = 'images' 97 | image_shape = (80, 80, 3) 98 | else: 99 | data_type = 'features' 100 | image_shape = None 101 | 102 | 103 | conv3d_results, y_conv = validate(data_type, model, saved_model=saved_model, 104 | image_shape=image_shape, class_limit=4, train_test=train_test) 105 | 106 | model = 'lstm' 107 | # saved_model = 'data/checkpoints/lstm-features.004-0.614.hdf5' 108 | saved_model = 'data/checkpoints/lstm-features.017-0.849.hdf5' 109 | 110 | if model == 'conv_3d' or model == 'lrcn': 111 | data_type = 'images' 112 | image_shape = (80, 80, 3) 113 | else: 114 | data_type = 'features' 115 | image_shape = None 116 | 117 | lstm_results, y_lstm = validate(data_type, model, saved_model=saved_model, 118 | image_shape=image_shape, class_limit=4, train_test=train_test) 119 | 120 | 121 | 122 | 123 | # results = [] 124 | combined_result = [] 125 | results = np.concatenate((lstm_results, conv_flow_3d_results, conv3d_results), axis=1) 126 | print(results.shape) 127 | # for i in range(len(lstm_results)): 128 | # results.append([lstm_results[i][0],lstm_results[i][1], conv3d_results[i][0], conv3d_results[i][1], conv_flow_3d_results[i][0], conv_flow_3d_results[i][1]]) 129 | # combined_result.append([lstm_results[i][0]+conv3d_results[i][0]+conv_flow_3d_results[i][0], 3*lstm_results[i][1]+conv3d_results[i][1]+conv_flow_3d_results[i][1]]) 130 | # combined_result = np.argmax(lstm_results[i][0]+conv3d_results[i][0], lstm_results[i][1]+conv3d_results[i][1]) 131 | # Print f1, precision, and recall scores 132 | np.save('output_' + train_test + '.npy', results) 133 | np.save('output_labels_' + train_test + '.npy' , y_lstm) 134 | 135 | # print("Conv Flow 3d") 136 | # print(precision_score(y_lstm, np.argmax(conv_flow_3d_results, axis=1), average="macro")) 137 | # print(recall_score(y_lstm, np.argmax(conv_flow_3d_results, axis=1), average="macro")) 138 | # print(f1_score(y_lstm, np.argmax(conv_flow_3d_results, axis=1), average="macro")) 139 | # 140 | # print("LSTM") 141 | # print(precision_score(y_lstm, np.argmax(lstm_results, axis=1) , average="macro")) 142 | # print(recall_score(y_lstm, np.argmax(lstm_results, axis=1) , average="macro")) 143 | # print(f1_score(y_lstm, np.argmax(lstm_results, axis=1) , average="macro")) 144 | # 145 | # print("Conv 3d") 146 | # print(precision_score(y_lstm, np.argmax(conv3d_results, axis=1), average="macro")) 147 | # print(recall_score(y_lstm, np.argmax(conv3d_results, axis=1), average="macro")) 148 | # print(f1_score(y_lstm, np.argmax(conv3d_results, axis=1), average="macro")) 149 | # 150 | # print("Combined") 151 | # print(precision_score(y_lstm, np.argmax(combined_result, axis=1), average="macro")) 152 | # print(recall_score(y_lstm, np.argmax(combined_result, axis=1), average="macro")) 153 | # print(f1_score(y_lstm, np.argmax(combined_result, axis=1), average="macro")) 154 | 155 | # results.append((lstm_results[i][0]+conv3d_results[i][0], lstm_results[i][1]+conv3d_results[i][1])) 156 | # print(str(lstm_results[i]),str(conv3d_results[i])) 157 | # print(str(y_conv[i]),str(y_lstm[i])) 158 | 159 | 160 | # y_pred = np.argmax(y_pred1, axis=1) 161 | # 162 | # # Print f1, precision, and recall scores 163 | # print(precision_score(y_test, y_pred , average="macro")) 164 | # print(recall_score(y_test, y_pred , average="macro")) 165 | # print(f1_score(y_test, y_pred , average="macro")) 166 | 167 | return results, y_lstm 168 | 169 | if __name__ == '__main__': 170 | results, labels = main('train') 171 | results_test, labels_test = main('test') 172 | # svc = svm.SVC(kernel='rbf', C=1, gamma='auto') 173 | parameters = {'kernel':('linear', 'rbf', 'poly'), 'C':[0.1, 1, 10]} 174 | svc = svm.SVC() 175 | # Create regularization penalty space 176 | # penalty = ['l1', 'l2'] 177 | 178 | # Create regularization hyperparameter space 179 | # C = np.logspace(0, 4, 10) 180 | 181 | # Create hyperparameter options 182 | # parameters = dict(C=C, penalty=penalty) 183 | # linear = linear_model.LogisticRegression() 184 | clf = GridSearchCV(svc, parameters) 185 | # clf.fit(iris.data, iris.target) 186 | clf.fit(results, labels) 187 | print("Accuracy for train: ", clf.score(results, labels)) 188 | print("Accuracy for test: ", clf.score(results_test, labels_test)) 189 | --------------------------------------------------------------------------------