├── .gitattributes ├── .gitignore ├── LICENSE ├── Models ├── CNN+GRU │ ├── GRU_FeatureVector.py │ ├── GRU_Image.py │ └── preprocessing │ │ ├── __init__.py │ │ ├── image.py │ │ ├── image_img.py │ │ └── image_oulu.py ├── CNN+Geometry │ ├── confusion_matrix_Face.txt │ ├── confusion_matrix_Geometry.txt │ ├── confusion_matrix_cnn.txt │ ├── confusion_matrix_geo.txt │ ├── predict.py │ ├── preprocessing │ │ ├── __init__.py │ │ └── image.py │ ├── vggFace_geometry.py │ └── vggface_model.py ├── CNN+LSTM │ ├── LSTM.py │ ├── LSTM_FeatureVector.py │ ├── LSTM_Image.py │ ├── SaveModelDirectory.py │ ├── feature_vector.py │ ├── preprocessing │ │ ├── __init__.py │ │ ├── image.py │ │ ├── image_img.py │ │ ├── image_img_oulu.py │ │ └── image_oulu.py │ └── pretrained_lstm.py ├── CNN │ ├── vggFace.py │ └── vggface_model.py ├── CNN_2Stream │ ├── preprocessing │ │ ├── __init__.py │ │ └── image.py │ ├── vggFace_MM.py │ └── vggface_model_MM.py ├── CNN_3D │ ├── 3dcnn.py │ ├── cnn3d_model.py │ └── preprocessing │ │ ├── __init__.py │ │ └── image.py └── _saving │ ├── ModelParameters.py │ ├── SaveModelDirectory.py │ └── __init__.py ├── README.md ├── Thesis.pdf └── _Preprocess ├── block_video_to_data.py ├── count-images.py ├── crop-to-aligned-and-geometry.py ├── egg_preprocessing.py ├── errors.txt ├── faceDetection.py ├── folder-person_to_folder-emotion.py ├── folder-person_to_geometry-face.py ├── frame_to_16frames.py ├── frames_to_dataset_3d.py ├── oulu-obtain_face.py ├── preprocess_EGG.py ├── resize16.py ├── verify_16.py └── video_to_frame.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Project Related # 2 | ################### 3 | /_Dataset/* 4 | /_Legacy/* 5 | /_Preprocess/Frontalization/* 6 | /_Preprocess/frames_per_participants/* 7 | 8 | /Models/CNN/weights/* 9 | /Models/CNN_2Stream/weights/* 10 | /Models/CNN_3D/weights/* 11 | /Models/CNN+EGG/weights/* 12 | /Models/CNN+Geometry/weights/* 13 | /Models/CNN+LSTM/weights/* 14 | /Models/CNN+GRU/weights/* 15 | 16 | /Models/CNN/_versions/* 17 | /Models/CNN_2Stream/_versions/* 18 | /Models/CNN_3D/_versions/* 19 | /Models/CNN+EGG/_versions/* 20 | /Models/CNN+Geometry/_versions/* 21 | /Models/CNN+LSTM/_versions/* 22 | /Models/CNN+GRU/_versions/* 23 | 24 | # Office files # 25 | ################### 26 | *.pptx 27 | 28 | # Compiled source # 29 | ################### 30 | *.com 31 | *.class 32 | *.dll 33 | *.exe 34 | *.o 35 | *.so 36 | *.pyc 37 | 38 | # Packages # 39 | ############ 40 | # it's better to unpack these files and commit the raw source 41 | # git has its own built in compression methods 42 | *.7z 43 | *.dmg 44 | *.gz 45 | *.iso 46 | *.jar 47 | *.rar 48 | *.tar 49 | *.zip 50 | 51 | # Logs and databases # 52 | ###################### 53 | *.log 54 | *.sql 55 | *.sqlite 56 | 57 | # OS generated files # 58 | ###################### 59 | .DS_Store 60 | .DS_Store? 61 | ._* 62 | .Spotlight-V100 63 | .Trashes 64 | ehthumbs.db 65 | Thumbs.db 66 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Daniel 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /Models/CNN+GRU/GRU_FeatureVector.py: -------------------------------------------------------------------------------- 1 | # export CUDA_VISIBLE_DEVICES=1,2 2 | import os 3 | import sys 4 | import time 5 | import os.path as osp 6 | 7 | # Add the parent module to the import path 8 | sys.path.append(osp.realpath(osp.join(osp.dirname(__file__), '../'))) 9 | 10 | from keras.optimizers import adam 11 | from keras.initializers import glorot_uniform 12 | from keras.models import Sequential, Model, load_model 13 | from keras.layers import GRU, Dense, Activation, Flatten, Input 14 | from keras.callbacks import ModelCheckpoint, CSVLogger, TensorBoard, Callback 15 | 16 | from collections import OrderedDict 17 | 18 | import warnings 19 | warnings.simplefilter("ignore", DeprecationWarning) 20 | 21 | from collections import OrderedDict 22 | from _saving.SaveModelDirectory import create_path 23 | from _saving.ModelParameters import ModelParameters 24 | 25 | __author__ = 'Daniel Garcia Zapata' 26 | 27 | # Flow From Directory 28 | def obtain_datagen(datagen, train_path, h5=True): 29 | return datagen.flow_from_directory( 30 | train_path, 31 | target_size=(img_height, img_width), 32 | batch_size=batch_size, 33 | class_mode='binary', 34 | partition=partition) 35 | 36 | # Yield for data generators 37 | def generate_data_generator_for_two_images(genX1): 38 | while True: 39 | X1i = genX1.next() 40 | yield X1i[0], X1i[1] 41 | 42 | if __name__ == '__main__': 43 | 44 | print("Starting:", time.ctime()) 45 | 46 | ########################################### 47 | # Data 48 | 49 | input_shape = 4096 50 | 51 | dataset = 'OULU-CASIA' # OULU-CASIA, SASE-FE 52 | partition = 'feature_vector_prealigned' 53 | if dataset == 'OULU-CASIA': 54 | from preprocessing.image_oulu import ImageDataGenerator 55 | 56 | train_data_dir = os.path.join('..', '..', '_Dataset', dataset, 'consecutive', 'training') 57 | validation_data_dir = os.path.join('..', '..', '_Dataset', dataset, 'consecutive', 'validation') 58 | n_output = 6 59 | else: 60 | from preprocessing.image import ImageDataGenerator 61 | 62 | train_data_dir = os.path.join('..', '..', '_Dataset', 'SASE-FE', '5frames', 'training') 63 | validation_data_dir = os.path.join('..', '..', '_Dataset', 'SASE-FE', '5frames', 'validation') 64 | n_output = 12 65 | 66 | __model__ = 'CNN-GRU-' + partition 67 | 68 | ########################################### 69 | # Parameters 70 | 71 | epochs = 20 72 | batch_size = 60 73 | nb_train_samples = 29798 / batch_size 74 | nb_validation_samples = 4428 / batch_size 75 | impl = 2 # gpu 76 | 77 | ############################################ 78 | # Model 79 | 80 | neurons = 512 81 | drop = 0.5 82 | lr = 0.0001 83 | 84 | # features_model = 'weights/Stream_1_Frontalization_epoch-09_val-accu-0.35.hdf5' 85 | # model = define_model(12, lr, drop, neurons, features_model) 86 | 87 | neurons = 512 88 | dropout = 0.5 89 | nlayers = 3 90 | 91 | activation = 'relu' 92 | activation_r = 'sigmoid' 93 | 94 | # Initialize weights 95 | weight_init = glorot_uniform(seed=3) 96 | 97 | ''' 98 | Load the output of the CNN 99 | ''' 100 | 101 | model = Sequential() 102 | if nlayers == 1: 103 | model.add(GRU(neurons, input_shape=(None, input_shape), implementation=impl, dropout=drop, 104 | activation=activation, recurrent_activation=activation_r)) 105 | else: 106 | model.add(GRU(neurons, input_shape=(None, input_shape), implementation=impl, dropout=drop, 107 | activation=activation, recurrent_activation=activation_r, return_sequences=True)) 108 | for i in range(1, nlayers-1): 109 | model.add(GRU(neurons, dropout=drop, implementation=impl, 110 | activation=activation, recurrent_activation=activation_r, return_sequences=True)) 111 | model.add(GRU(neurons, dropout=drop, implementation=impl, 112 | activation=activation, recurrent_activation=activation_r)) 113 | 114 | model.add(Dense(n_output, activation='softmax', kernel_initializer=weight_init)) 115 | 116 | model.summary() 117 | 118 | ########################################### 119 | # Data Generator 120 | 121 | datagen = ImageDataGenerator() 122 | 123 | # Training data generators 124 | train_generator = obtain_datagen(datagen, train_data_dir) 125 | validation_generator = obtain_datagen(datagen, validation_data_dir) 126 | 127 | # Yield for data generators 128 | dataset_train_gen = generate_data_generator_for_two_images(train_generator) 129 | dataset_val_gen = generate_data_generator_for_two_images(validation_generator) 130 | 131 | ############################################ 132 | # Training 133 | lr = 0.0001 134 | optimizer = adam(lr=lr) 135 | loss = 'sparse_categorical_crossentropy' 136 | model.compile( loss=loss, 137 | optimizer=optimizer, 138 | metrics=['accuracy', 'top_k_categorical_accuracy']) 139 | 140 | ''' 141 | Callbacks 142 | ''' 143 | row_dict = OrderedDict({'model': __model__, 144 | 'dataset': dataset, 145 | 'partition': partition, 146 | 'loss': loss, 147 | 'lr': lr, 148 | 'nlayers': nlayers, 149 | 'date': time.ctime()}) 150 | 151 | # Create Version folder 152 | export_path = create_path(__model__, dataset) 153 | 154 | checkpointer = ModelCheckpoint(filepath=osp.join(export_path, __model__+'_epoch-{epoch:02d}_val-accu-{val_acc:.4f}.hdf5'), verbose=1) #, save_best_only=True) 155 | csv_logger = CSVLogger(osp.join(export_path, '_logs_'+__model__+'.log'), separator=',', append=False) 156 | tensorboard = TensorBoard(log_dir=export_path, histogram_freq=0, batch_size=batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) 157 | model_parameters = ModelParameters(osp.join(export_path, '_model_'+__model__+'.log'), row_dict) 158 | 159 | model.fit_generator(dataset_train_gen, 160 | steps_per_epoch=nb_train_samples, 161 | epochs=epochs, 162 | validation_data=dataset_val_gen, 163 | validation_steps=nb_validation_samples, 164 | callbacks=[checkpointer, csv_logger]) 165 | 166 | print("Ending:", time.ctime()) 167 | -------------------------------------------------------------------------------- /Models/CNN+GRU/GRU_Image.py: -------------------------------------------------------------------------------- 1 | # export CUDA_VISIBLE_DEVICES=1,2 2 | import os 3 | import sys 4 | import time 5 | import os.path as osp 6 | 7 | # Add the parent module to the import path 8 | sys.path.append(osp.realpath(osp.join(osp.dirname(__file__), '../'))) 9 | 10 | from keras.optimizers import adam 11 | from keras.initializers import glorot_uniform 12 | from keras.models import Sequential, Model, load_model 13 | from keras.callbacks import ModelCheckpoint, CSVLogger, TensorBoard, Callback 14 | from keras.layers import Dense, Activation, Flatten, Input, GRU, TimeDistributed 15 | 16 | from collections import OrderedDict 17 | from preprocessing.image_img import ImageDataGenerator 18 | 19 | import warnings 20 | warnings.simplefilter("ignore", DeprecationWarning) 21 | 22 | from collections import OrderedDict 23 | from _saving.SaveModelDirectory import create_path 24 | from _saving.ModelParameters import ModelParameters 25 | 26 | __author__ = 'Daniel Garcia Zapata' 27 | 28 | # Flow From Directory 29 | def obtain_datagen(datagen, train_path, h5=True): 30 | return datagen.flow_from_directory( 31 | train_path, 32 | target_size=(img_width,img_height), 33 | batch_size=batch_size, 34 | class_mode='binary', 35 | partition=partition) 36 | 37 | # Yield for data generators 38 | def generate_data_generator_for_two_images(genX1): 39 | while True: 40 | X1i = genX1.next() 41 | yield X1i[0], X1i[1] 42 | 43 | if __name__ == '__main__': 44 | 45 | __model__ = 'CNN_GRU_Image' 46 | 47 | print('Starting:', time.ctime(), '\n') 48 | 49 | 50 | ########################################### 51 | # Parameters 52 | 53 | epochs = 20 54 | batch_size = 20 55 | impl = 2 # gpu 56 | 57 | ########################################### 58 | # Data 59 | 60 | img_width, img_height, channels = 224, 224, 3 # Resolution of inputs 61 | input_shape = 4096 62 | 63 | dataset = 'SASE-FE' 64 | partition = 'prealigned' 65 | if dataset == 'OULU-CASIA': 66 | train_data_dir = os.path.join('..', '..', '_Dataset', dataset, 'consecutive', 'training') 67 | validation_data_dir = os.path.join('..', '..', '_Dataset', dataset, 'consecutive', 'validation') 68 | 69 | frames = 5 70 | n_output = 6 71 | 72 | nb_train_samples = 5941 / batch_size 73 | nb_validation_samples = 2025 / batch_size 74 | 75 | else: 76 | partition = '' 77 | 78 | train_data_dir = os.path.join('..', '..', '_Dataset', dataset, '5frames', 'training') 79 | validation_data_dir = os.path.join('..', '..', '_Dataset', dataset, '5frames', 'validation') 80 | 81 | frames = 5 82 | n_output = 12 83 | 84 | nb_train_samples = 27971 / batch_size 85 | nb_validation_samples = 4173 / batch_size 86 | 87 | ############################################ 88 | # Model 89 | 90 | neurons = 1024 91 | nlayers = 2 92 | dropout = 0.5 93 | 94 | activation = 'relu' 95 | activation_r = 'sigmoid' 96 | 97 | # Initialize weights 98 | weight_init = glorot_uniform(seed=3) 99 | 100 | ''' 101 | Load the output of the CNN 102 | ''' 103 | cnn_model = load_model(os.path.join('weights', 'CNN_prealigned.hdf5')) 104 | 105 | model_input = Input(shape=(frames, img_width, img_height, channels), 106 | name='seq_input') 107 | 108 | x = TimeDistributed(cnn_model)(model_input) 109 | x = TimeDistributed(Flatten())(x) 110 | x = GRU(neurons, dropout=dropout, name='gru_1')(x) 111 | out = Dense(n_output, kernel_initializer=weight_init, name='out')(x) 112 | 113 | model = Model(inputs=[model_input], outputs=out) 114 | 115 | model.summary() 116 | 117 | ''' Freeze previous layers ''' 118 | for layer in cnn_model.layers: 119 | layer.trainable = False 120 | 121 | ########################################### 122 | # Data Generator 123 | 124 | datagen = ImageDataGenerator( 125 | rescale=1. / 224, 126 | shear_range=0.2, 127 | zoom_range=0.2, 128 | horizontal_flip=True) 129 | 130 | # Training data generators 131 | train_generator = obtain_datagen(datagen, train_data_dir) 132 | validation_generator = obtain_datagen(datagen, validation_data_dir) 133 | 134 | # Yield for data generators 135 | dataset_train_gen = generate_data_generator_for_two_images(train_generator) 136 | dataset_val_gen = generate_data_generator_for_two_images(validation_generator) 137 | 138 | ############################################ 139 | ''' Training ''' 140 | lr = 0.0001 141 | optimizer = adam(lr=lr) 142 | loss = 'sparse_categorical_crossentropy' 143 | model.compile( loss=loss, 144 | optimizer=optimizer, 145 | metrics=['accuracy', 'top_k_categorical_accuracy']) 146 | 147 | ''' 148 | Callbacks 149 | ''' 150 | row_dict = OrderedDict({'model': __model__, 151 | 'dataset': dataset, 152 | 'partition': partition, 153 | 'loss': loss, 154 | 'lr': lr, 155 | 'nlayers': nlayers, 156 | 'date': time.ctime()}) 157 | 158 | # Create Version folder 159 | export_path = create_path(__model__, dataset) 160 | 161 | checkpointer = ModelCheckpoint(filepath=osp.join(export_path, __model__+'_epoch-{epoch:02d}_val-accu-{val_acc:.4f}.hdf5'), verbose=1) #, save_best_only=True) 162 | csv_logger = CSVLogger(osp.join(export_path, '_logs_'+__model__+'.log'), separator=',', append=False) 163 | tensorboard = TensorBoard(log_dir=export_path, histogram_freq=0, batch_size=batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) 164 | model_parameters = ModelParameters(osp.join(export_path, '_model_'+__model__+'.log'), row_dict) 165 | 166 | model.fit_generator(dataset_train_gen, 167 | steps_per_epoch=nb_train_samples, 168 | epochs=epochs, 169 | validation_data=dataset_val_gen, 170 | validation_steps=nb_validation_samples) 171 | # ,callbacks=[checkpointer, csv_logger, model_parameters]) 172 | 173 | print('\nEnding:', time.ctime()) 174 | -------------------------------------------------------------------------------- /Models/CNN+GRU/preprocessing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dangz90/Deep-Learning-for-Expression-Recognition-in-Image-Sequences/66c50292cac9a66eace32a040c49267b06c45de5/Models/CNN+GRU/preprocessing/__init__.py -------------------------------------------------------------------------------- /Models/CNN+Geometry/confusion_matrix_Face.txt: -------------------------------------------------------------------------------- 1 | [[36, 0, 8, 0, 18, 5], 2 | [16, 0, 8, 8, 21, 14], 3 | [57, 0, 7, 0, 18, 3], 4 | [23, 0, 6, 9, 40, 2], 5 | [21, 0, 3, 17, 28, 20], 6 | [12, 0, 5, 2, 32, 10]] -------------------------------------------------------------------------------- /Models/CNN+Geometry/confusion_matrix_Geometry.txt: -------------------------------------------------------------------------------- 1 | [[ 0, 0, 0, 67, 0, 0], 2 | [ 0, 0, 0, 67, 0, 0], 3 | [ 0, 0, 0, 85, 0, 0], 4 | [ 0, 0, 0, 80, 0, 0], 5 | [ 0, 0, 0, 89, 0, 0], 6 | [ 0, 0, 0, 61, 0, 0]] -------------------------------------------------------------------------------- /Models/CNN+Geometry/confusion_matrix_cnn.txt: -------------------------------------------------------------------------------- 1 | [[ 27, 0, 0, 285, 11, 104], 2 | [ 0, 0, 0, 361, 8, 78], 3 | [ 0, 0, 0, 287, 0, 161], 4 | [ 0, 0, 0, 286, 0, 162], 5 | [ 0, 0, 0, 292, 1, 128], 6 | [ 0, 0, 0, 147, 0, 287]] -------------------------------------------------------------------------------- /Models/CNN+Geometry/confusion_matrix_geo.txt: -------------------------------------------------------------------------------- 1 | [[ 0, 0, 0, 427, 0, 0], 2 | [ 0, 0, 0, 447, 0, 0], 3 | [ 0, 0, 0, 448, 0, 0], 4 | [ 0, 0, 0, 448, 0, 0], 5 | [ 0, 0, 0, 421, 0, 0], 6 | [ 0, 0, 0, 434, 0, 0]] -------------------------------------------------------------------------------- /Models/CNN+Geometry/predict.py: -------------------------------------------------------------------------------- 1 | # export CUDA_VISIBLE_DEVICES=1,2 2 | import os 3 | import h5py 4 | import time 5 | import csv, six 6 | import numpy as np 7 | from keras import backend 8 | from keras.models import load_model 9 | from sklearn.metrics import confusion_matrix 10 | 11 | try: 12 | from PIL import Image as pil_image 13 | except ImportError: 14 | pil_image = None 15 | 16 | classes = {0:'Anger', 1:'Disgust', 2:'Fear', 3:'Happiness', 4:'Sadness', 5:'Surprise'} 17 | emotions = {'Anger':0, 'Disgust':1, 'Fear':2, 'Happiness':3, 'Sadness':4, 'Surprise':5} 18 | # labels = ['Anger','Disgust','Fear','Happiness','Sadness','Surprise'] 19 | labels = [0, 1, 2, 3, 4, 5] 20 | 21 | 22 | def load_img(path, grayscale=False, target_size=None, 23 | interpolation='bilinear'): 24 | if pil_image is None: 25 | raise ImportError('Could not import PIL.Image. ' 26 | 'The use of `array_to_img` requires PIL.') 27 | img = pil_image.open(path) 28 | if grayscale: 29 | if img.mode != 'L': 30 | img = img.convert('L') 31 | else: 32 | if img.mode != 'RGB': 33 | img = img.convert('RGB') 34 | if target_size is not None: 35 | width_height_tuple = (target_size[1], target_size[0]) 36 | if img.size != width_height_tuple: 37 | if interpolation not in _PIL_INTERPOLATION_METHODS: 38 | raise ValueError( 39 | 'Invalid interpolation method {} specified. Supported ' 40 | 'methods are {}'.format( 41 | interpolation, 42 | ", ".join(_PIL_INTERPOLATION_METHODS.keys()))) 43 | resample = _PIL_INTERPOLATION_METHODS[interpolation] 44 | img = img.resize(width_height_tuple, resample) 45 | return img 46 | 47 | def img_to_array(img, data_format=None): 48 | if data_format is None: 49 | data_format = backend.image_data_format() 50 | if data_format not in {'channels_first', 'channels_last'}: 51 | raise ValueError('Unknown data_format: ', data_format) 52 | # Numpy array x has format (height, width, channel) 53 | # or (channel, height, width) 54 | # but original PIL image has format (width, height, channel) 55 | x = np.asarray(img, dtype=backend.floatx()) 56 | if len(x.shape) == 3: 57 | if data_format == 'channels_first': 58 | x = x.transpose(2, 0, 1) 59 | elif len(x.shape) == 2: 60 | if data_format == 'channels_first': 61 | x = x.reshape((1, x.shape[0], x.shape[1])) 62 | else: 63 | x = x.reshape((x.shape[0], x.shape[1], 1)) 64 | else: 65 | raise ValueError('Unsupported image shape: ', x.shape) 66 | return x 67 | 68 | if __name__ == '__main__': 69 | 70 | __model__ = 'CNN_Geometry' 71 | 72 | print "Starting:", time.ctime() 73 | 74 | ########################################### 75 | # Get Dataset 76 | dataset = 'OULU-CASIA' 77 | partition = 'aligned' 78 | directory = os.path.join('..', '..', '_Dataset', dataset, partition, '_Face', 'validation') 79 | 80 | def list_files(dir, extension): 81 | from collections import defaultdict 82 | 83 | # Explore sub directories 84 | subdirs = [x[0] for x in os.walk(dir)] # if len(x[0].split(os.sep)) >= 6 85 | 86 | names_of_files = defaultdict(list) 87 | list_of_img = defaultdict(list) 88 | list_of_geo = defaultdict(list) 89 | # Cycle through sub directories 90 | for subdir in sorted(subdirs[1:]): 91 | 92 | # Obtain files from sub directories 93 | files = os.walk(subdir).next()[2] 94 | 95 | # Cycle through files 96 | for e, file in enumerate(sorted(files)): 97 | 98 | if file.endswith(extension): 99 | file_path = os.path.join(subdir, file) 100 | names_of_files[os.path.basename(subdir)].append( file ) 101 | 102 | ''' Images ''' 103 | grayscale = 'grayscale' 104 | data_format = backend.image_data_format() 105 | image_shape = (224,224,3) 106 | index_array = np.arange(1) 107 | 108 | batch_x = np.zeros( (len(index_array),) + image_shape, dtype=backend.floatx() ) 109 | img = load_img( os.path.join(file_path), 110 | grayscale=grayscale, 111 | target_size=[224,224]) 112 | x = img_to_array(img, data_format=data_format) 113 | batch_x[0] = x 114 | 115 | list_of_img[os.path.basename(subdir)].append( batch_x ) 116 | 117 | ''' Geometry ''' 118 | batch_x_2 = np.zeros( (len(index_array),) + (136,), dtype=backend.floatx() ) 119 | 120 | # f = h5py.File(os.path.join(file_path.replace('prealigned', 'pregeometry').replace(extension, '.h5')), 'r') 121 | # x = f['geometry'].value 122 | f = np.load( os.path.join(file_path.replace('_Face', '_Geometry').replace(extension, '.npy')) ) 123 | x = f 124 | 125 | # # Normalize 126 | new_origin = [112,135] 127 | new_x = x - new_origin 128 | new_x = (new_x - new_x.min())/(new_x.max() - new_x.min()) 129 | x = new_x.reshape((136,)) 130 | batch_x_2[0] = x 131 | 132 | list_of_geo[os.path.basename(subdir)].append( batch_x_2 ) 133 | 134 | return list_of_img, list_of_geo, names_of_files 135 | # return list_of_img, names_of_files 136 | 137 | extension = '.jpg' 138 | # list_of_img, list_of_geo, names_of_files = list_files(directory, extension) 139 | list_of_img, list_of_geo, names_of_files = list_files(directory, extension) 140 | 141 | ''' 142 | Load the model 143 | ''' 144 | model = load_model(os.path.join('weights', 'CNN_pregeometry.hdf5')) 145 | 146 | print('START') 147 | 148 | y_true = [] 149 | y_pred = [] 150 | accuracy = 0.0 151 | total_files = 0 152 | 153 | for (class0, files), (class1, videos), (class2, geos) in zip(names_of_files.items(), list_of_img.items(), list_of_geo.items()): 154 | 155 | for file, video, geo in zip(files, videos, geos): 156 | 157 | inputs = [video, geo] 158 | # inputs = video 159 | 160 | prediction = model.predict( inputs ) 161 | class_number = (np.argmax(prediction)) 162 | 163 | y_true.append(emotions[class1]) 164 | y_pred.append(class_number) 165 | 166 | # print(file, 'True', class1, 'Pred', classes[class_number]) 167 | 168 | if class_number == emotions[class1]: 169 | accuracy += 1.0 170 | 171 | total_files += 1 172 | 173 | filename = 'confusion_matrix.txt' 174 | with open(filename, 'w') as f: 175 | f.write(np.array2string(confusion_matrix(y_true, y_pred, labels=labels), separator=', ')) 176 | 177 | print('Accuracy', float(accuracy)/float(total_files) ) 178 | 179 | print('END') 180 | -------------------------------------------------------------------------------- /Models/CNN+Geometry/preprocessing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dangz90/Deep-Learning-for-Expression-Recognition-in-Image-Sequences/66c50292cac9a66eace32a040c49267b06c45de5/Models/CNN+Geometry/preprocessing/__init__.py -------------------------------------------------------------------------------- /Models/CNN+Geometry/vggFace_geometry.py: -------------------------------------------------------------------------------- 1 | # export CUDA_VISIBLE_DEVICES=1,2 2 | import os 3 | import sys 4 | import time 5 | import os.path as osp 6 | 7 | # Add the parent module to the import path 8 | sys.path.append(osp.realpath(osp.join(osp.dirname(__file__), '../'))) 9 | 10 | from vggface_model import VGGFace 11 | 12 | from keras.optimizers import adam 13 | from keras.models import Model, load_model 14 | from keras.layers import Dense, Activation, Flatten, Input, concatenate 15 | from keras.callbacks import ModelCheckpoint, CSVLogger, TensorBoard, Callback 16 | 17 | from collections import OrderedDict 18 | from preprocessing.image import ImageDataGenerator 19 | 20 | from collections import OrderedDict 21 | from _saving.SaveModelDirectory import create_path 22 | from _saving.ModelParameters import ModelParameters 23 | 24 | __author__ = 'Daniel Garcia Zapata' 25 | 26 | 27 | # Datagen 28 | def obtain_datagen(datagen, path, h5=False, npy=False): 29 | return datagen.flow_from_directory( 30 | path, 31 | target_size=(img_height, img_width), 32 | batch_size=batch_size, 33 | class_mode='binary') 34 | 35 | # Yield for data generators 36 | def generate_data_generator_for_two_images(genX): 37 | while True: 38 | Xi = genX.next() 39 | 40 | yield [Xi[0], Xi[1]], Xi[2] 41 | 42 | if __name__ == '__main__': 43 | 44 | __model__ = 'CNN_Geometry' 45 | 46 | print "Starting:", time.ctime() 47 | 48 | ########################################### 49 | # Parameters 50 | 51 | epochs = 20 52 | batch_size = 40 53 | 54 | ########################################### 55 | # Data 56 | 57 | img_width, img_height, channel = 224, 224, 3 # Resolution of inputs 58 | 59 | dataset = 'OULU-CASIA' 60 | partition = 'prealigned' 61 | # Dataset for Stream 1 62 | train_data_dir = os.path.join('..', '..', '_Dataset', dataset, partition, 'test') 63 | validation_data_dir = os.path.join('..', '..', '_Dataset', dataset, partition, 'validation') 64 | 65 | if dataset == 'OULU-CASIA': 66 | nb_class = 6 67 | 68 | nb_train_samples = 7754 69 | nb_validation_samples = 2625 70 | else: 71 | nb_class = 12 72 | 73 | nb_train_samples = 29798 74 | nb_validation_samples = 4428 75 | 76 | nb_train_samples = nb_train_samples / batch_size 77 | nb_validation_samples = nb_validation_samples / batch_size 78 | 79 | ''' 80 | Load the model 81 | ''' 82 | vgg_model = load_model(os.path.join('weights', 'CNN_prealigned.hdf5')) 83 | 84 | ''' 85 | Customize the model 86 | ''' 87 | # Add geometry input 88 | input_geo = Input(shape=(136,), name='input-geometry') 89 | # geo_layer = Flatten(name='flatten-geo')(input_geo) 90 | 91 | # Obtain frontalization's last layer 92 | last_layer = vgg_model.get_layer('pool5').output 93 | for layer in vgg_model.layers: 94 | layer.name = layer.name + "-model_frontalization" 95 | front_layer = Flatten(name='flatten-front')(last_layer) 96 | 97 | # Concatenate geo and front layers and customize it 98 | x = concatenate([front_layer, input_geo]) 99 | x = Dense(4096, activation='relu', name='fc6')(x) 100 | x = Dense(4096, activation='relu', name='fc7')(x) 101 | x = Dense(2622, activation='relu', name='fc8')(x) 102 | out = Dense(nb_class, activation='softmax', name='fc9')(x) 103 | 104 | model = Model( 105 | [vgg_model.input, input_geo], 106 | out) 107 | 108 | ''' 109 | Dataset Generators 110 | ''' 111 | datagen = ImageDataGenerator( 112 | rescale=1. / 224, 113 | shear_range=0.2, 114 | zoom_range=0.2, 115 | horizontal_flip=True) 116 | 117 | # Training and Validation data generators 118 | train_generator = obtain_datagen(datagen, train_data_dir, npy=True) 119 | validation_generator = obtain_datagen(datagen, validation_data_dir, npy=True) 120 | 121 | # Yield for data generators 122 | dataset_train_gen = generate_data_generator_for_two_images(train_generator) 123 | dataset_val_gen = generate_data_generator_for_two_images(validation_generator) 124 | 125 | 126 | ''' 127 | Fine-tune the model 128 | ''' 129 | # Freeze previous layers 130 | for i, layer in enumerate(vgg_model.layers): 131 | layer.trainable = False 132 | 133 | # Compile the model 134 | lr = 0.0001 135 | optimizer = adam(lr=lr) 136 | loss = 'sparse_categorical_crossentropy' 137 | model.compile( loss=loss, 138 | optimizer=optimizer, 139 | metrics=['accuracy', 'top_k_categorical_accuracy']) 140 | 141 | ''' 142 | Callbacks 143 | ''' 144 | row_dict = OrderedDict({'model': __model__, 145 | 'dataset': dataset, 146 | 'partition': partition, 147 | 'loss': loss, 148 | 'lr': lr, 149 | 'date': time.ctime()}) 150 | 151 | # Create Version folder 152 | export_path = create_path(__model__, dataset) 153 | 154 | checkpointer = ModelCheckpoint(filepath=osp.join(export_path, __model__+'_epoch-{epoch:02d}_val-accu-{val_acc:.4f}.hdf5'), verbose=1) #, save_best_only=True) 155 | csv_logger = CSVLogger(osp.join(export_path, '_logs_'+__model__+'.log'), separator=',', append=False) 156 | tensorboard = TensorBoard(log_dir=export_path, histogram_freq=0, batch_size=batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) 157 | model_parameters = ModelParameters(osp.join(export_path, '_model_'+__model__+'.log'), row_dict) 158 | 159 | ''' 160 | Train the model 161 | ''' 162 | print('*************************************\nTraining \n*************************************') 163 | model.fit_generator( 164 | dataset_train_gen, 165 | steps_per_epoch=nb_train_samples, 166 | epochs=epochs, 167 | validation_data=dataset_val_gen, 168 | validation_steps=nb_validation_samples, 169 | callbacks=[checkpointer, csv_logger, tensorboard, model_parameters]) 170 | 171 | print("Ending:", time.ctime()) 172 | -------------------------------------------------------------------------------- /Models/CNN+Geometry/vggface_model.py: -------------------------------------------------------------------------------- 1 | '''VGGFace model for Keras. 2 | # Reference: 3 | - [Deep Face Recognition](http://www.robots.ox.ac.uk/~vgg/publications/2015/Parkhi15/parkhi15.pdf) 4 | ''' 5 | from __future__ import print_function 6 | import warnings 7 | 8 | from keras.applications.imagenet_utils import _obtain_input_shape 9 | from keras.engine.topology import get_source_inputs 10 | from keras.models import Model 11 | from keras.layers import Flatten, Dense, Input, GlobalAveragePooling2D, GlobalMaxPooling2D, Activation 12 | from keras.layers import Convolution2D, MaxPooling2D 13 | from keras.utils import layer_utils 14 | from keras.utils.data_utils import get_file 15 | from keras import backend as K 16 | 17 | WEIGHTS_PATH = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_tf_v2.h5' 18 | WEIGHTS_PATH_NO_TOP = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_tf_notop_v2.h5' 19 | 20 | def VGGFace(include_top=False, weights='vggface', 21 | input_tensor=None, input_shape=None, 22 | pooling='max', 23 | classes=2622): 24 | """Instantiates the VGGFace architecture. 25 | Optionally loads weights pre-trained 26 | on VGGFace dataset. Note that when using TensorFlow, 27 | for best performance you should set 28 | `image_data_format="channels_last"` in your Keras config 29 | at ~/.keras/keras.json. 30 | The model and the weights are compatible with both 31 | TensorFlow and Theano. The data format 32 | convention used by the model is the one 33 | specified in your Keras config file. 34 | # Arguments 35 | include_top: whether to include the 3 fully-connected 36 | layers at the top of the network. 37 | weights: one of `None` (random initialization) 38 | or "imagenet" (pre-training on ImageNet). 39 | input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) 40 | to use as image input for the model. 41 | input_shape: optional shape tuple, only to be specified 42 | if `include_top` is False (otherwise the input shape 43 | has to be `(224, 224, 3)` (with `channels_last` data format) 44 | or `(3, 224, 244)` (with `channels_first` data format). 45 | It should have exactly 3 inputs channels, 46 | and width and height should be no smaller than 48. 47 | E.g. `(200, 200, 3)` would be one valid value. 48 | pooling: Optional pooling mode for feature extraction 49 | when `include_top` is `False`. 50 | - `None` means that the output of the model will be 51 | the 4D tensor output of the 52 | last convolutional layer. 53 | - `avg` means that global average pooling 54 | will be applied to the output of the 55 | last convolutional layer, and thus 56 | the output of the model will be a 2D tensor. 57 | - `max` means that global max pooling will 58 | be applied. 59 | classes: optional number of classes to classify images 60 | into, only to be specified if `include_top` is True, and 61 | if no `weights` argument is specified. 62 | # Returns 63 | A Keras model instance. 64 | # Raises 65 | ValueError: in case of invalid argument for `weights`, 66 | or invalid input shape. 67 | """ 68 | 69 | if weights not in {'vggface', None}: 70 | raise ValueError('The `weights` argument should be either ' 71 | '`None` (random initialization) or `vggface` ' 72 | '(pre-training on VGGFace Dataset).') 73 | 74 | if weights == 'vggface' and include_top and classes != 2622: 75 | raise ValueError('If using `weights` as vggface original with `include_top`' 76 | ' as true, `classes` should be 2622') 77 | # Determine proper input shape 78 | input_shape = _obtain_input_shape(input_shape, 79 | default_size=224, 80 | min_size=48, 81 | data_format=K.image_data_format(), 82 | include_top=include_top) 83 | 84 | if input_tensor is None: 85 | img_input = Input(shape=input_shape) 86 | else: 87 | if not K.is_keras_tensor(input_tensor): 88 | img_input = Input(tensor=input_tensor, shape=input_shape) 89 | else: 90 | img_input = input_tensor 91 | 92 | # Block 1 93 | x = Convolution2D(64, (3, 3), activation='relu', padding='same', name='conv1_1')(img_input) 94 | x = Convolution2D(64, (3, 3), activation='relu', padding='same', name='conv1_2')(x) 95 | x = MaxPooling2D((2, 2), strides=(2, 2), name='pool1')(x) 96 | 97 | # Block 2 98 | x = Convolution2D(128, (3, 3), activation='relu', padding='same', name='conv2_1')(x) 99 | x = Convolution2D(128, (3, 3), activation='relu', padding='same', name='conv2_2')(x) 100 | x = MaxPooling2D((2, 2), strides=(2, 2), name='pool2')(x) 101 | 102 | # Block 3 103 | x = Convolution2D(256, (3, 3), activation='relu', padding='same', name='conv3_1')(x) 104 | x = Convolution2D(256, (3, 3), activation='relu', padding='same', name='conv3_2')(x) 105 | x = Convolution2D(256, (3, 3), activation='relu', padding='same', name='conv3_3')(x) 106 | x = MaxPooling2D((2, 2), strides=(2, 2), name='pool3')(x) 107 | 108 | # Block 4 109 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv4_1')(x) 110 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv4_2')(x) 111 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv4_3')(x) 112 | x = MaxPooling2D((2, 2), strides=(2, 2), name='pool4')(x) 113 | 114 | # Block 5 115 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv5_1')(x) 116 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv5_2')(x) 117 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv5_3')(x) 118 | x = MaxPooling2D((2, 2), strides=(2, 2), name='pool5')(x) 119 | 120 | if include_top: 121 | # Classification block 122 | x = Flatten(name='flatten')(x) 123 | x = Dense(4096, name='fc6')(x) 124 | x = Activation('relu', name='fc6/relu')(x) 125 | x = Dense(4096, name='fc7')(x) 126 | x = Activation('relu', name='fc7/relu')(x) 127 | x = Dense(2622, name='fc8')(x) 128 | x = Activation('relu', name='fc8/softmax')(x) 129 | else: 130 | if pooling == 'avg': 131 | x = GlobalAveragePooling2D()(x) 132 | elif pooling == 'max': 133 | x = GlobalMaxPooling2D()(x) 134 | 135 | # Ensure that the model takes into account 136 | # any potential predecessors of `input_tensor`. 137 | if input_tensor is not None: 138 | inputs = get_source_inputs(input_tensor) 139 | else: 140 | inputs = img_input 141 | # Create model. 142 | model = Model(inputs, x, name='VGGFace') # load weights 143 | if weights == 'vggface': 144 | if include_top: 145 | weights_path = get_file('rcmalli_vggface_tf_v2.h5', 146 | WEIGHTS_PATH, 147 | cache_subdir='models') 148 | else: 149 | weights_path = get_file('rcmalli_vggface_tf_notop_v2.h5', 150 | WEIGHTS_PATH_NO_TOP, 151 | cache_subdir='models') 152 | model.load_weights(weights_path, by_name=True) 153 | if K.backend() == 'theano': 154 | layer_utils.convert_all_kernels_in_model(model) 155 | 156 | if K.image_data_format() == 'channels_first': 157 | if include_top: 158 | maxpool = model.get_layer(name='pool5') 159 | shape = maxpool.output_shape[1:] 160 | dense = model.get_layer(name='fc6') 161 | layer_utils.convert_dense_weights_data_format(dense, shape, 'channels_first') 162 | 163 | if K.backend() == 'tensorflow': 164 | warnings.warn('You are using the TensorFlow backend, yet you ' 165 | 'are using the Theano ' 166 | 'image data format convention ' 167 | '(`image_data_format="channels_first"`). ' 168 | 'For best performance, set ' 169 | '`image_data_format="channels_last"` in ' 170 | 'your Keras config ' 171 | 'at ~/.keras/keras.json.') 172 | return model -------------------------------------------------------------------------------- /Models/CNN+LSTM/LSTM.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import numpy as np 4 | 5 | from pretrained_lstm import define_model 6 | 7 | from keras.models import Sequential, Model, load_model 8 | from keras.layers import LSTM 9 | from keras.layers import Dense, Activation, Flatten, Input 10 | from keras.optimizers import RMSprop, SGD, adam 11 | from keras.callbacks import ModelCheckpoint, CSVLogger 12 | # from keras.preprocessing.image import ImageDataGenerator 13 | from preprocessing.image import ImageDataGenerator 14 | 15 | __author__ = 'Daniel Garcia Zapata' 16 | 17 | if __name__ == '__main__': 18 | 19 | __model__ = 'CNN-LSTM_1' 20 | 21 | print("Starting:", time.ctime()) 22 | 23 | ########################################### 24 | # Parameters 25 | 26 | epochs = 20 27 | batch_size = 60 28 | nb_train_samples = 29798 / batch_size 29 | nb_validation_samples = 4428 / batch_size 30 | impl = 2 # gpu 31 | 32 | ############################################ 33 | # Model 34 | 35 | neurons = 512 36 | drop = 0.5 37 | lr = 0.0001 38 | 39 | # features_model = 'weights/Stream_1_Frontalization_epoch-09_val-accu-0.35.hdf5' 40 | # model = define_model(12, lr, drop, neurons, features_model) 41 | 42 | neurons = 512 43 | drop = 0.5 44 | nlayers = 4 45 | 46 | activation = 'relu' 47 | activation_r = 'sigmoid' 48 | 49 | ''' 50 | Load the output of the CNN 51 | ''' 52 | 53 | model = Sequential() 54 | if nlayers == 1: 55 | model.add(LSTM(neurons, input_shape=(None, 4096), implementation=impl, dropout=drop, 56 | activation=activation, recurrent_activation=activation_r)) 57 | else: 58 | model.add(LSTM(neurons, input_shape=(None, 4096), implementation=impl, dropout=drop, 59 | activation=activation, recurrent_activation=activation_r, return_sequences=True)) 60 | for i in range(1, nlayers-1): 61 | model.add(LSTM(neurons, dropout=drop, implementation=impl, 62 | activation=activation, recurrent_activation=activation_r, return_sequences=True)) 63 | model.add(LSTM(neurons, dropout=drop, implementation=impl, 64 | activation=activation, recurrent_activation=activation_r)) 65 | model.add(Dense(1)) 66 | 67 | print('Neurons: ', neurons, 'Layers: ', nlayers, activation, activation_r) 68 | print('') 69 | 70 | ########################################### 71 | # Data 72 | 73 | input_shape = 4096 74 | train_data_dir = os.path.join('..', '..', '_Dataset', 'dataset16_consecutive_3d', 'training') 75 | validation_data_dir = os.path.join('..', '..', '_Dataset', 'dataset16_consecutive_3d', 'validation') 76 | 77 | # Data Generator 78 | train_datagen = ImageDataGenerator() 79 | test_datagen = ImageDataGenerator() 80 | 81 | # Flow From Directory 82 | def obtain_datagen(datagen, train_path, seed_number=7, h5=True): 83 | return datagen.flow_from_directory( 84 | train_path, 85 | target_size=(224,224), 86 | # batch_size=batch_size, 87 | seed=seed_number, 88 | class_mode='binary', 89 | h5=h5) 90 | 91 | # Training data generators 92 | train_generator = obtain_datagen(train_datagen, train_data_dir) 93 | validation_generator = obtain_datagen(test_datagen, validation_data_dir) 94 | 95 | # Yield for data generators 96 | def generate_data_generator_for_two_images(genX1): 97 | while True: 98 | X1i = genX1.next() 99 | yield X1i[0], X1i[1] 100 | 101 | # Yield for data generators 102 | dataset_train_gen = generate_data_generator_for_two_images(train_generator) 103 | dataset_val_gen = generate_data_generator_for_two_images(validation_generator) 104 | 105 | ############################################ 106 | # Training 107 | 108 | optimizer = adam(lr=0.0001) 109 | model.compile(optimizer=optimizer, loss='mean_squared_error', metrics=['accuracy']) 110 | 111 | ''' 112 | Callbacks 113 | ''' 114 | filepath = 'weights' 115 | checkpointer = ModelCheckpoint(filepath= os.path.join(filepath, '_epoch-{epoch:02d}_val-accu-{val_acc:.2f}.hdf5'), verbose=1) #, save_best_only=True) 116 | csv_logger = CSVLogger('logs/'+__model__+'.log', separator=',', append=False) 117 | 118 | model.fit_generator(dataset_train_gen, 119 | steps_per_epoch=nb_train_samples, 120 | epochs=epochs, 121 | validation_data=dataset_val_gen, 122 | validation_steps=nb_validation_samples, 123 | callbacks=[checkpointer, csv_logger]) 124 | 125 | print("Ending:", time.ctime()) -------------------------------------------------------------------------------- /Models/CNN+LSTM/LSTM_FeatureVector.py: -------------------------------------------------------------------------------- 1 | # export CUDA_VISIBLE_DEVICES=1,2 2 | import os 3 | import sys 4 | import time 5 | import os.path as osp 6 | 7 | # Add the parent module to the import path 8 | sys.path.append(osp.realpath(osp.join(osp.dirname(__file__), '../'))) 9 | 10 | import keras.backend as K 11 | 12 | from keras.initializers import glorot_uniform 13 | from keras.optimizers import RMSprop, SGD, adam 14 | from keras.metrics import top_k_categorical_accuracy 15 | from keras.models import Sequential, Model, load_model 16 | from keras.layers import LSTM, Dense, Activation, Flatten, Input 17 | from keras.callbacks import ModelCheckpoint, CSVLogger, TensorBoard, Callback 18 | 19 | from collections import OrderedDict 20 | from preprocessing.image import ImageDataGenerator 21 | 22 | from _saving.SaveModelDirectory import create_path 23 | from _saving.ModelParameters import ModelParameters 24 | 25 | # import warnings 26 | # warnings.simplefilter("ignore", DeprecationWarning) 27 | 28 | 29 | __author__ = 'Daniel Garcia Zapata' 30 | 31 | # Flow From Directory 32 | def obtain_datagen(datagen, train_path, h5=True): 33 | return datagen.flow_from_directory( 34 | train_path, 35 | # target_size=(224,224), 36 | batch_size=batch_size, 37 | class_mode='binary', 38 | h5=h5) 39 | 40 | # Yield for data generators 41 | def generate_data_generator_for_two_images(genX1): 42 | while True: 43 | X1i = genX1.next() 44 | yield X1i[0], X1i[1] 45 | 46 | if __name__ == '__main__': 47 | 48 | __model__ = 'CNN-LSTM_Features' 49 | 50 | print("Starting:", time.ctime()) 51 | 52 | ########################################### 53 | # Data 54 | 55 | input_shape = 4096 56 | 57 | dataset = 'SASE-FE' 58 | partition = 'prealigned' 59 | if dataset == 'OULU-CASIA': # OULU-CASIA 60 | train_data_dir = os.path.join('..', '..', '_Dataset', dataset, 'consecutive', 'training') 61 | validation_data_dir = os.path.join('..', '..', '_Dataset', dataset, 'consecutive', 'validation') 62 | n_output = 6 63 | else: 64 | train_data_dir = os.path.join('..', '..', '_Dataset', dataset, '5frames', 'training') 65 | validation_data_dir = os.path.join('..', '..', '_Dataset', dataset, '5frames', 'validation') 66 | n_output = 12 67 | 68 | ########################################### 69 | # Parameters 70 | 71 | epochs = 20 72 | batch_size = 60 73 | nb_train_samples = 29798 / batch_size 74 | nb_validation_samples = 4428 / batch_size 75 | impl = 2 # gpu 76 | 77 | ############################################ 78 | # Model 79 | 80 | neurons = 2622 # 512 81 | drop = 0.5 82 | lr = 0.0001 83 | nlayers = 3 84 | 85 | activation = 'relu' 86 | activation_r = 'sigmoid' 87 | 88 | # Initialize weights 89 | weight_init = glorot_uniform(seed=3) 90 | 91 | ''' 92 | Load the output of the CNN 93 | ''' 94 | 95 | model = Sequential() 96 | if nlayers == 1: 97 | model.add(LSTM(neurons, input_shape=(None, input_shape), implementation=impl, dropout=drop, 98 | activation=activation, recurrent_activation=activation_r)) 99 | else: 100 | model.add(LSTM(neurons, input_shape=(None, input_shape), implementation=impl, dropout=drop, 101 | activation=activation, recurrent_activation=activation_r, return_sequences=True)) 102 | for i in range(1, nlayers-1): 103 | model.add(LSTM(neurons, dropout=drop, implementation=impl, 104 | activation=activation, recurrent_activation=activation_r, return_sequences=True)) 105 | model.add(LSTM(neurons, dropout=drop, implementation=impl, 106 | activation=activation, recurrent_activation=activation_r)) 107 | 108 | model.add(Dense(n_output, activation='softmax', kernel_initializer=weight_init)) 109 | 110 | model.summary() 111 | 112 | ########################################### 113 | # Data Generator 114 | 115 | datagen = ImageDataGenerator() 116 | 117 | # Training data generators 118 | train_generator = obtain_datagen(datagen, train_data_dir) 119 | validation_generator = obtain_datagen(datagen, validation_data_dir) 120 | 121 | # Yield for data generators 122 | dataset_train_gen = generate_data_generator_for_two_images(train_generator) 123 | dataset_val_gen = generate_data_generator_for_two_images(validation_generator) 124 | 125 | ########################################### 126 | ''' Training ''' 127 | 128 | optimizer = adam(lr=lr) 129 | loss = 'sparse_categorical_crossentropy' 130 | model.compile( loss=loss, 131 | optimizer=optimizer, 132 | metrics=['accuracy', 'top_k_categorical_accuracy']) 133 | 134 | ''' 135 | Callbacks 136 | ''' 137 | row_dict = OrderedDict({'model': __model__, 138 | 'dataset': dataset, 139 | 'partition': partition, 140 | 'loss': loss, 141 | 'lr': lr, 142 | 'nlayers': nlayers, 143 | 'date': time.ctime()}) 144 | 145 | # Create Version folder 146 | export_path = create_path(__model__, dataset) 147 | 148 | checkpointer = ModelCheckpoint(filepath=osp.join(export_path, __model__+'_epoch-{epoch:02d}_val-accu-{val_acc:.4f}.hdf5'), verbose=1) #, save_best_only=True) 149 | csv_logger = CSVLogger(osp.join(export_path, '_logs_'+__model__+'.log'), separator=',', append=False) 150 | tensorboard = TensorBoard(log_dir=export_path, histogram_freq=0, batch_size=batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) 151 | model_parameters = ModelParameters(osp.join(export_path, '_model_'+__model__+'.log'), row_dict) 152 | 153 | ''' 154 | Fit Model 155 | ''' 156 | model.fit_generator(dataset_train_gen, 157 | steps_per_epoch=nb_train_samples, 158 | epochs=epochs, 159 | validation_data=dataset_val_gen, 160 | validation_steps=nb_validation_samples, 161 | callbacks=[checkpointer, csv_logger, tensorboard, model_parameters]) 162 | 163 | print("Ending:", time.ctime()) 164 | -------------------------------------------------------------------------------- /Models/CNN+LSTM/LSTM_Image.py: -------------------------------------------------------------------------------- 1 | # export CUDA_VISIBLE_DEVICES=1,2 2 | import os 3 | import sys 4 | import time 5 | import os.path as osp 6 | 7 | # Add the parent module to the import path 8 | sys.path.append(osp.realpath(osp.join(osp.dirname(__file__), '../'))) 9 | 10 | from keras.optimizers import adam 11 | from keras.initializers import glorot_uniform 12 | from keras.models import Sequential, Model, load_model 13 | from keras.callbacks import ModelCheckpoint, CSVLogger, TensorBoard, Callback 14 | from keras.layers import Dense, Activation, Flatten, Input, LSTM, TimeDistributed 15 | 16 | from collections import OrderedDict 17 | from SaveModelDirectory import create_path 18 | 19 | from _saving.SaveModelDirectory import create_path 20 | from _saving.ModelParameters import ModelParameters 21 | 22 | import warnings 23 | warnings.simplefilter("ignore", DeprecationWarning) 24 | 25 | 26 | __author__ = 'Daniel Garcia Zapata' 27 | 28 | # Flow From Directory 29 | def obtain_datagen(datagen, train_path, h5=True): 30 | return datagen.flow_from_directory( 31 | train_path, 32 | target_size=(img_width,img_height), 33 | batch_size=batch_size, 34 | class_mode='binary', 35 | partition=partition) 36 | 37 | # Yield for data generators 38 | def generate_data_generator_for_two_images(genX1): 39 | while True: 40 | X1i = genX1.next() 41 | yield X1i[0], X1i[1] 42 | 43 | if __name__ == '__main__': 44 | 45 | __model__ = 'CNN_LSTM_Image' 46 | 47 | print('Starting:', time.ctime(), '\n') 48 | 49 | 50 | ########################################### 51 | # Parameters 52 | 53 | epochs = 20 54 | batch_size = 30 55 | impl = 2 # gpu 56 | 57 | ########################################### 58 | # Data 59 | 60 | img_width, img_height, channels = 224, 224, 3 # Resolution of inputs 61 | input_shape = 4096 62 | 63 | dataset = 'OULU-CASIA' 64 | partition = 'prealigned' 65 | if dataset == 'OULU-CASIA': 66 | from preprocessing.image_img import ImageDataGenerator 67 | 68 | train_data_dir = os.path.join('..', '..', '_Dataset', dataset, 'consecutive', 'training') 69 | validation_data_dir = os.path.join('..', '..', '_Dataset', dataset, 'consecutive', 'validation') 70 | 71 | frames = 5 72 | n_output = 6 73 | 74 | nb_train_samples = 6019 / batch_size 75 | nb_validation_samples = 1947 / batch_size 76 | 77 | else: 78 | from preprocessing.image_img import ImageDataGenerator 79 | 80 | train_data_dir = os.path.join('..', '..', '_Dataset', dataset, '5frames', 'training') 81 | validation_data_dir = os.path.join('..', '..', '_Dataset', dataset, '5frames', 'validation') 82 | 83 | frames = 5 84 | n_output = 12 85 | 86 | nb_train_samples = 27971 / batch_size 87 | nb_validation_samples = 4173 / batch_size 88 | 89 | ############################################ 90 | # Model 91 | 92 | neurons = 512 93 | nlayers = 2 94 | dropout = 0.5 95 | 96 | lr = 0.0001 97 | 98 | activation = 'relu' 99 | activation_r = 'sigmoid' 100 | 101 | # Initialize weights 102 | weight_init = glorot_uniform(seed=3) 103 | 104 | ''' 105 | Load the output of the CNN 106 | ''' 107 | cnn_model = load_model(os.path.join('weights', 'CNN_prealigned_epoch-14_val-accu-0.2636.hdf5')) 108 | 109 | model_input = Input(shape=(frames, img_width, img_height, channels), 110 | name='seq_input') 111 | 112 | x = TimeDistributed(cnn_model)(model_input) 113 | x = TimeDistributed(Flatten())(x) 114 | x = LSTM(neurons, dropout=dropout, name='lstm')(x) 115 | out = Dense(n_output, kernel_initializer=weight_init, name='out')(x) 116 | 117 | model = Model(inputs=[model_input], outputs=out) 118 | 119 | model.summary() 120 | 121 | ''' Freeze previous layers ''' 122 | for layer in cnn_model.layers: 123 | layer.trainable = False 124 | 125 | ########################################### 126 | # Data Generator 127 | 128 | datagen = ImageDataGenerator( 129 | rescale=1. / 224, 130 | shear_range=0.2, 131 | zoom_range=0.2, 132 | horizontal_flip=True) 133 | 134 | # Training data generators 135 | train_generator = obtain_datagen(datagen, train_data_dir) 136 | validation_generator = obtain_datagen(datagen, validation_data_dir) 137 | 138 | # Yield for data generators 139 | dataset_train_gen = generate_data_generator_for_two_images(train_generator) 140 | dataset_val_gen = generate_data_generator_for_two_images(validation_generator) 141 | 142 | ############################################ 143 | ''' Training ''' 144 | 145 | optimizer = adam(lr=lr) 146 | loss = 'sparse_categorical_crossentropy' 147 | model.compile( loss=loss, 148 | optimizer=optimizer, 149 | metrics=['accuracy', 'top_k_categorical_accuracy']) 150 | 151 | ''' 152 | Callbacks 153 | ''' 154 | row_dict = OrderedDict({'model': __model__, 155 | 'dataset': dataset, 156 | 'partition': partition, 157 | 'loss': loss, 158 | 'lr': lr, 159 | 'nlayers': nlayers, 160 | 'date': time.ctime()}) 161 | 162 | # Create Version folder 163 | export_path = create_path(__model__, dataset) 164 | 165 | checkpointer = ModelCheckpoint(filepath=osp.join(export_path, __model__+'_epoch-{epoch:02d}_val-accu-{val_acc:.4f}.hdf5'), verbose=1) #, save_best_only=True) 166 | csv_logger = CSVLogger(osp.join(export_path, '_logs_'+__model__+'.log'), separator=',', append=False) 167 | tensorboard = TensorBoard(log_dir=export_path, histogram_freq=0, batch_size=batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) 168 | model_parameters = ModelParameters(osp.join(export_path, '_model_'+__model__+'.log'), row_dict) 169 | 170 | model.fit_generator(dataset_train_gen, 171 | steps_per_epoch=nb_train_samples, 172 | epochs=epochs, 173 | validation_data=dataset_val_gen, 174 | validation_steps=nb_validation_samples, 175 | callbacks=[checkpointer, csv_logger, tensorboard, model_parameters]) 176 | 177 | print('\nEnding:', time.ctime()) 178 | -------------------------------------------------------------------------------- /Models/CNN+LSTM/SaveModelDirectory.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import tensorflow as tf 4 | 5 | def create_path(__model__, dataset): 6 | 7 | ''' 8 | Directory base folder 9 | ''' 10 | export_path_base = os.path.dirname(sys.argv[0]) 11 | 12 | ''' 13 | Define model version 14 | ''' 15 | 16 | # Obtain most recent version 17 | export_base_path = os.path.join(export_path_base, '_versions', dataset, __model__) 18 | 19 | # Verify a folder already exists and assign version number 20 | if os.path.isdir(export_base_path): 21 | all_subdirs = [int(d) for d in os.listdir(export_base_path) if os.path.isdir(export_base_path)] 22 | __version__ = int(max(all_subdirs)) + 1 23 | else: 24 | __version__ = 1 25 | 26 | # Assign FLAG version number 27 | tf.app.flags.DEFINE_integer('model_version', __version__, 'Version number of the model.') 28 | FLAGS = tf.app.flags.FLAGS 29 | 30 | ''' 31 | Create version folder 32 | ''' 33 | export_path = os.path.join( export_base_path, 34 | str(FLAGS.model_version)) 35 | builder = tf.saved_model.builder.SavedModelBuilder(export_path) 36 | 37 | return export_path 38 | 39 | if __name__ == '__main__': 40 | create_path() -------------------------------------------------------------------------------- /Models/CNN+LSTM/feature_vector.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import h5py 4 | import numpy as np 5 | 6 | from keras.preprocessing import image 7 | from keras.models import Model, load_model 8 | 9 | __author__ = 'Daniel Garcia Zapata' 10 | __model__ = 'Feature_Vector' 11 | 12 | def create_feature_vector(dir, features, processed): 13 | img_width, img_height = 224, 224 14 | subdirs = [x[0] for x in os.walk(dir)] 15 | 16 | for subdir in sorted(subdirs): 17 | files = next(os.walk(subdir))[2] # os.walk(subdir).__next__()[2] 18 | 19 | for file in sorted(files): 20 | if file.endswith('.jpeg'): 21 | 22 | # Check if destination directory exists if not then create it 23 | destination = subdir.replace(processed, 'feature_vector_'+processed) 24 | if not os.path.exists(destination): 25 | os.makedirs(destination) 26 | 27 | # Load image and transform it into feature vector 28 | image_path = os.path.join(subdir, file) 29 | img = image.load_img(image_path, target_size=(img_width, img_height)) 30 | x = image.img_to_array(img) 31 | x = np.expand_dims(x, axis=0) 32 | 33 | images = np.vstack([x]) 34 | 35 | feature_vec = features.predict(images) 36 | 37 | # Save feature vector in h5 format 38 | feature_vector_path = os.path.join(destination, file.replace('.jpeg', '.h5')) 39 | 40 | print('Saving', feature_vector_path) 41 | hf = h5py.File(feature_vector_path, 'w') 42 | hf.create_dataset('data', data=feature_vec) 43 | hf.close() 44 | 45 | if __name__ == '__main__': 46 | 47 | print("Starting:", time.ctime()) 48 | 49 | ########################################### 50 | # Parameters 51 | 52 | dataset = 'OULU-CASIA' 53 | processed = 'prealigned' 54 | train_data_dir = os.path.join('..', '..', '_Dataset', dataset, processed, 'training') 55 | validation_data_dir = os.path.join('..', '..', '_Dataset', dataset, processed, 'validation') 56 | data_dir = [train_data_dir, validation_data_dir] 57 | 58 | ############################################ 59 | # Model 60 | 61 | ''' Load the output of the CNN ''' 62 | weights = 'CNN_patch_epoch-06_val-accu-0.34.hdf5' 63 | cnn_model = load_model(os.path.join('weights', weights)) 64 | cnn_output = cnn_model.get_layer('fc7').output 65 | 66 | ''' Feature Vector ''' 67 | features = Model(cnn_model.input, cnn_output) 68 | for dirs in data_dir: 69 | create_feature_vector(dirs, features, processed) 70 | 71 | print("Ending:", time.ctime()) -------------------------------------------------------------------------------- /Models/CNN+LSTM/preprocessing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dangz90/Deep-Learning-for-Expression-Recognition-in-Image-Sequences/66c50292cac9a66eace32a040c49267b06c45de5/Models/CNN+LSTM/preprocessing/__init__.py -------------------------------------------------------------------------------- /Models/CNN+LSTM/preprocessing/image_img_oulu.py: -------------------------------------------------------------------------------- 1 | """Fairly basic set of tools for real-time data augmentation on image data. 2 | Can easily be extended to include new transformations, 3 | new preprocessing methods, etc... 4 | """ 5 | from __future__ import absolute_import 6 | from __future__ import print_function 7 | 8 | import numpy as np 9 | import re 10 | import h5py 11 | from scipy import linalg 12 | import scipy.ndimage as ndi 13 | from six.moves import range 14 | import os 15 | import threading 16 | import warnings 17 | import multiprocessing.pool 18 | from functools import partial 19 | from pathlib import Path 20 | 21 | from keras import backend 22 | from keras.utils.data_utils import Sequence 23 | 24 | try: 25 | from PIL import Image as pil_image 26 | except ImportError: 27 | pil_image = None 28 | 29 | 30 | if pil_image is not None: 31 | _PIL_INTERPOLATION_METHODS = { 32 | 'nearest': pil_image.NEAREST, 33 | 'bilinear': pil_image.BILINEAR, 34 | 'bicubic': pil_image.BICUBIC, 35 | } 36 | # These methods were only introduced in version 3.4.0 (2016). 37 | if hasattr(pil_image, 'HAMMING'): 38 | _PIL_INTERPOLATION_METHODS['hamming'] = pil_image.HAMMING 39 | if hasattr(pil_image, 'BOX'): 40 | _PIL_INTERPOLATION_METHODS['box'] = pil_image.BOX 41 | # This method is new in version 1.1.3 (2013). 42 | if hasattr(pil_image, 'LANCZOS'): 43 | _PIL_INTERPOLATION_METHODS['lanczos'] = pil_image.LANCZOS 44 | 45 | 46 | def random_rotation(x, rg, row_axis=1, col_axis=2, channel_axis=0, 47 | fill_mode='nearest', cval=0.): 48 | """Performs a random rotation of a Numpy image tensor. 49 | 50 | # Arguments 51 | x: Input tensor. Must be 3D. 52 | rg: Rotation range, in degrees. 53 | row_axis: Index of axis for rows in the input tensor. 54 | col_axis: Index of axis for columns in the input tensor. 55 | channel_axis: Index of axis for channels in the input tensor. 56 | fill_mode: Points outside the boundaries of the input 57 | are filled according to the given mode 58 | (one of `{'constant', 'nearest', 'reflect', 'wrap'}`). 59 | cval: Value used for points outside the boundaries 60 | of the input if `mode='constant'`. 61 | 62 | # Returns 63 | Rotated Numpy image tensor. 64 | """ 65 | theta = np.pi / 180 * np.random.uniform(-rg, rg) 66 | rotation_matrix = np.array([[np.cos(theta), -np.sin(theta), 0], 67 | [np.sin(theta), np.cos(theta), 0], 68 | [0, 0, 1]]) 69 | 70 | h, w = x.shape[row_axis], x.shape[col_axis] 71 | transform_matrix = transform_matrix_offset_center(rotation_matrix, h, w) 72 | x = apply_transform(x, transform_matrix, channel_axis, fill_mode, cval) 73 | return x 74 | 75 | 76 | def random_shift(x, wrg, hrg, row_axis=1, col_axis=2, channel_axis=0, 77 | fill_mode='nearest', cval=0.): 78 | """Performs a random spatial shift of a Numpy image tensor. 79 | 80 | # Arguments 81 | x: Input tensor. Must be 3D. 82 | wrg: Width shift range, as a float fraction of the width. 83 | hrg: Height shift range, as a float fraction of the height. 84 | row_axis: Index of axis for rows in the input tensor. 85 | col_axis: Index of axis for columns in the input tensor. 86 | channel_axis: Index of axis for channels in the input tensor. 87 | fill_mode: Points outside the boundaries of the input 88 | are filled according to the given mode 89 | (one of `{'constant', 'nearest', 'reflect', 'wrap'}`). 90 | cval: Value used for points outside the boundaries 91 | of the input if `mode='constant'`. 92 | 93 | # Returns 94 | Shifted Numpy image tensor. 95 | """ 96 | h, w = x.shape[row_axis], x.shape[col_axis] 97 | tx = np.random.uniform(-hrg, hrg) * h 98 | ty = np.random.uniform(-wrg, wrg) * w 99 | translation_matrix = np.array([[1, 0, tx], 100 | [0, 1, ty], 101 | [0, 0, 1]]) 102 | 103 | transform_matrix = translation_matrix # no need to do offset 104 | x = apply_transform(x, transform_matrix, channel_axis, fill_mode, cval) 105 | return x 106 | 107 | 108 | def random_shear(x, intensity, row_axis=1, col_axis=2, channel_axis=0, 109 | fill_mode='nearest', cval=0.): 110 | """Performs a random spatial shear of a Numpy image tensor. 111 | 112 | # Arguments 113 | x: Input tensor. Must be 3D. 114 | intensity: Transformation intensity. 115 | row_axis: Index of axis for rows in the input tensor. 116 | col_axis: Index of axis for columns in the input tensor. 117 | channel_axis: Index of axis for channels in the input tensor. 118 | fill_mode: Points outside the boundaries of the input 119 | are filled according to the given mode 120 | (one of `{'constant', 'nearest', 'reflect', 'wrap'}`). 121 | cval: Value used for points outside the boundaries 122 | of the input if `mode='constant'`. 123 | 124 | # Returns 125 | Sheared Numpy image tensor. 126 | """ 127 | shear = np.random.uniform(-intensity, intensity) 128 | shear_matrix = np.array([[1, -np.sin(shear), 0], 129 | [0, np.cos(shear), 0], 130 | [0, 0, 1]]) 131 | 132 | h, w = x.shape[row_axis], x.shape[col_axis] 133 | transform_matrix = transform_matrix_offset_center(shear_matrix, h, w) 134 | x = apply_transform(x, transform_matrix, channel_axis, fill_mode, cval) 135 | return x 136 | 137 | 138 | def random_zoom(x, zoom_range, row_axis=1, col_axis=2, channel_axis=0, 139 | fill_mode='nearest', cval=0.): 140 | """Performs a random spatial zoom of a Numpy image tensor. 141 | 142 | # Arguments 143 | x: Input tensor. Must be 3D. 144 | zoom_range: Tuple of floats; zoom range for width and height. 145 | row_axis: Index of axis for rows in the input tensor. 146 | col_axis: Index of axis for columns in the input tensor. 147 | channel_axis: Index of axis for channels in the input tensor. 148 | fill_mode: Points outside the boundaries of the input 149 | are filled according to the given mode 150 | (one of `{'constant', 'nearest', 'reflect', 'wrap'}`). 151 | cval: Value used for points outside the boundaries 152 | of the input if `mode='constant'`. 153 | 154 | # Returns 155 | Zoomed Numpy image tensor. 156 | 157 | # Raises 158 | ValueError: if `zoom_range` isn't a tuple. 159 | """ 160 | if len(zoom_range) != 2: 161 | raise ValueError('`zoom_range` should be a tuple or list of two floats. ' 162 | 'Received arg: ', zoom_range) 163 | 164 | if zoom_range[0] == 1 and zoom_range[1] == 1: 165 | zx, zy = 1, 1 166 | else: 167 | zx, zy = np.random.uniform(zoom_range[0], zoom_range[1], 2) 168 | zoom_matrix = np.array([[zx, 0, 0], 169 | [0, zy, 0], 170 | [0, 0, 1]]) 171 | 172 | h, w = x.shape[row_axis], x.shape[col_axis] 173 | transform_matrix = transform_matrix_offset_center(zoom_matrix, h, w) 174 | x = apply_transform(x, transform_matrix, channel_axis, fill_mode, cval) 175 | return x 176 | 177 | 178 | def random_channel_shift(x, intensity, channel_axis=0): 179 | x = np.rollaxis(x, channel_axis, 0) 180 | min_x, max_x = np.min(x), np.max(x) 181 | channel_images = [np.clip(x_channel + np.random.uniform(-intensity, intensity), min_x, max_x) 182 | for x_channel in x] 183 | x = np.stack(channel_images, axis=0) 184 | x = np.rollaxis(x, 0, channel_axis + 1) 185 | return x 186 | 187 | 188 | def transform_matrix_offset_center(matrix, x, y): 189 | o_x = float(x) / 2 + 0.5 190 | o_y = float(y) / 2 + 0.5 191 | offset_matrix = np.array([[1, 0, o_x], [0, 1, o_y], [0, 0, 1]]) 192 | reset_matrix = np.array([[1, 0, -o_x], [0, 1, -o_y], [0, 0, 1]]) 193 | transform_matrix = np.dot(np.dot(offset_matrix, matrix), reset_matrix) 194 | return transform_matrix 195 | 196 | 197 | def apply_transform(x, 198 | transform_matrix, 199 | channel_axis=0, 200 | fill_mode='nearest', 201 | cval=0.): 202 | """Apply the image transformation specified by a matrix. 203 | 204 | # Arguments 205 | x: 2D numpy array, single image. 206 | transform_matrix: Numpy array specifying the geometric transformation. 207 | channel_axis: Index of axis for channels in the input tensor. 208 | fill_mode: Points outside the boundaries of the input 209 | are filled according to the given mode 210 | (one of `{'constant', 'nearest', 'reflect', 'wrap'}`). 211 | cval: Value used for points outside the boundaries 212 | of the input if `mode='constant'`. 213 | 214 | # Returns 215 | The transformed version of the input. 216 | """ 217 | x = np.rollaxis(x, channel_axis, 0) 218 | final_affine_matrix = transform_matrix[:2, :2] 219 | final_offset = transform_matrix[:2, 2] 220 | channel_images = [ndi.interpolation.affine_transform( 221 | x_channel, 222 | final_affine_matrix, 223 | final_offset, 224 | order=0, 225 | mode=fill_mode, 226 | cval=cval) for x_channel in x] 227 | x = np.stack(channel_images, axis=0) 228 | x = np.rollaxis(x, 0, channel_axis + 1) 229 | return x 230 | 231 | 232 | def flip_axis(x, axis): 233 | x = np.asarray(x).swapaxes(axis, 0) 234 | x = x[::-1, ...] 235 | x = x.swapaxes(0, axis) 236 | return x 237 | 238 | 239 | def array_to_img(x, data_format=None, scale=True): 240 | """Converts a 3D Numpy array to a PIL Image instance. 241 | 242 | # Arguments 243 | x: Input Numpy array. 244 | data_format: Image data format. 245 | scale: Whether to rescale image values 246 | to be within [0, 255]. 247 | 248 | # Returns 249 | A PIL Image instance. 250 | 251 | # Raises 252 | ImportError: if PIL is not available. 253 | ValueError: if invalid `x` or `data_format` is passed. 254 | """ 255 | if pil_image is None: 256 | raise ImportError('Could not import PIL.Image. ' 257 | 'The use of `array_to_img` requires PIL.') 258 | x = np.asarray(x, dtype=backend.floatx()) 259 | if x.ndim != 3: 260 | raise ValueError('Expected image array to have rank 3 (single image). ' 261 | 'Got array with shape:', x.shape) 262 | 263 | if data_format is None: 264 | data_format = backend.image_data_format() 265 | if data_format not in {'channels_first', 'channels_last'}: 266 | raise ValueError('Invalid data_format:', data_format) 267 | 268 | # Original Numpy array x has format (height, width, channel) 269 | # or (channel, height, width) 270 | # but target PIL image has format (width, height, channel) 271 | if data_format == 'channels_first': 272 | x = x.transpose(1, 2, 0) 273 | if scale: 274 | x = x + max(-np.min(x), 0) 275 | x_max = np.max(x) 276 | if x_max != 0: 277 | x /= x_max 278 | x *= 255 279 | if x.shape[2] == 3: 280 | # RGB 281 | return pil_image.fromarray(x.astype('uint8'), 'RGB') 282 | elif x.shape[2] == 1: 283 | # grayscale 284 | return pil_image.fromarray(x[:, :, 0].astype('uint8'), 'L') 285 | else: 286 | raise ValueError('Unsupported channel number: ', x.shape[2]) 287 | 288 | 289 | def img_to_array(img, data_format=None): 290 | """Converts a PIL Image instance to a Numpy array. 291 | 292 | # Arguments 293 | img: PIL Image instance. 294 | data_format: Image data format. 295 | 296 | # Returns 297 | A 3D Numpy array. 298 | 299 | # Raises 300 | ValueError: if invalid `img` or `data_format` is passed. 301 | """ 302 | if data_format is None: 303 | data_format = backend.image_data_format() 304 | if data_format not in {'channels_first', 'channels_last'}: 305 | raise ValueError('Unknown data_format: ', data_format) 306 | # Numpy array x has format (height, width, channel) 307 | # or (channel, height, width) 308 | # but original PIL image has format (width, height, channel) 309 | x = np.asarray(img, dtype=backend.floatx()) 310 | if len(x.shape) == 3: 311 | if data_format == 'channels_first': 312 | x = x.transpose(2, 0, 1) 313 | elif len(x.shape) == 2: 314 | if data_format == 'channels_first': 315 | x = x.reshape((1, x.shape[0], x.shape[1])) 316 | else: 317 | x = x.reshape((x.shape[0], x.shape[1], 1)) 318 | else: 319 | raise ValueError('Unsupported image shape: ', x.shape) 320 | return x 321 | 322 | 323 | def load_img(path, grayscale=False, target_size=None, 324 | interpolation='bilinear'): 325 | """Loads an image into PIL format. 326 | 327 | # Arguments 328 | path: Path to image file 329 | grayscale: Boolean, whether to load the image as grayscale. 330 | target_size: Either `None` (default to original size) 331 | or tuple of ints `(img_height, img_width)`. 332 | interpolation: Interpolation method used to resample the image if the 333 | target size is different from that of the loaded image. 334 | Supported methods are "nearest", "bilinear", and "bicubic". 335 | If PIL version 1.1.3 or newer is installed, "lanczos" is also 336 | supported. If PIL version 3.4.0 or newer is installed, "box" and 337 | "hamming" are also supported. By default, "bilinear" is used. 338 | 339 | # Returns 340 | A PIL Image instance. 341 | 342 | # Raises 343 | ImportError: if PIL is not available. 344 | ValueError: if interpolation method is not supported. 345 | """ 346 | if pil_image is None: 347 | raise ImportError('Could not import PIL.Image. ' 348 | 'The use of `array_to_img` requires PIL.') 349 | img = pil_image.open(path) 350 | if grayscale: 351 | if img.mode != 'L': 352 | img = img.convert('L') 353 | else: 354 | if img.mode != 'RGB': 355 | img = img.convert('RGB') 356 | if target_size is not None: 357 | width_height_tuple = (target_size[1], target_size[0]) 358 | if img.size != width_height_tuple: 359 | if interpolation not in _PIL_INTERPOLATION_METHODS: 360 | raise ValueError( 361 | 'Invalid interpolation method {} specified. Supported ' 362 | 'methods are {}'.format( 363 | interpolation, 364 | ", ".join(_PIL_INTERPOLATION_METHODS.keys()))) 365 | resample = _PIL_INTERPOLATION_METHODS[interpolation] 366 | img = img.resize(width_height_tuple, resample) 367 | return img 368 | 369 | 370 | def list_pictures(directory, ext='jpg|jpeg|bmp|png|ppm'): 371 | return [os.path.join(root, f) 372 | for root, _, files in os.walk(directory) for f in files 373 | if re.match(r'([\w]+\.(?:' + ext + '))', f)] 374 | 375 | 376 | class ImageDataGenerator(object): 377 | """Generate minibatches of image data with real-time data augmentation. 378 | 379 | # Arguments 380 | featurewise_center: set input mean to 0 over the dataset. 381 | samplewise_center: set each sample mean to 0. 382 | featurewise_std_normalization: divide inputs by std of the dataset. 383 | samplewise_std_normalization: divide each input by its std. 384 | zca_whitening: apply ZCA whitening. 385 | zca_epsilon: epsilon for ZCA whitening. Default is 1e-6. 386 | rotation_range: degrees (0 to 180). 387 | width_shift_range: fraction of total width. 388 | height_shift_range: fraction of total height. 389 | shear_range: shear intensity (shear angle in radians). 390 | zoom_range: amount of zoom. if scalar z, zoom will be randomly picked 391 | in the range [1-z, 1+z]. A sequence of two can be passed instead 392 | to select this range. 393 | channel_shift_range: shift range for each channel. 394 | fill_mode: points outside the boundaries are filled according to the 395 | given mode ('constant', 'nearest', 'reflect' or 'wrap'). Default 396 | is 'nearest'. 397 | cval: value used for points outside the boundaries when fill_mode is 398 | 'constant'. Default is 0. 399 | horizontal_flip: whether to randomly flip images horizontally. 400 | vertical_flip: whether to randomly flip images vertically. 401 | rescale: rescaling factor. If None or 0, no rescaling is applied, 402 | otherwise we multiply the data by the value provided. This is 403 | applied after the `preprocessing_function` (if any provided) 404 | but before any other transformation. 405 | preprocessing_function: function that will be implied on each input. 406 | The function will run before any other modification on it. 407 | The function should take one argument: 408 | one image (Numpy tensor with rank 3), 409 | and should output a Numpy tensor with the same shape. 410 | data_format: 'channels_first' or 'channels_last'. In 'channels_first' mode, the channels dimension 411 | (the depth) is at index 1, in 'channels_last' mode it is at index 3. 412 | It defaults to the `image_data_format` value found in your 413 | Keras config file at `~/.keras/keras.json`. 414 | If you never set it, then it will be "channels_last". 415 | """ 416 | 417 | def __init__(self, 418 | featurewise_center=False, 419 | samplewise_center=False, 420 | featurewise_std_normalization=False, 421 | samplewise_std_normalization=False, 422 | zca_whitening=False, 423 | zca_epsilon=1e-6, 424 | rotation_range=0., 425 | width_shift_range=0., 426 | height_shift_range=0., 427 | shear_range=0., 428 | zoom_range=0., 429 | channel_shift_range=0., 430 | fill_mode='nearest', 431 | cval=0., 432 | horizontal_flip=False, 433 | vertical_flip=False, 434 | rescale=None, 435 | preprocessing_function=None, 436 | data_format=None): 437 | if data_format is None: 438 | data_format = backend.image_data_format() 439 | self.featurewise_center = featurewise_center 440 | self.samplewise_center = samplewise_center 441 | self.featurewise_std_normalization = featurewise_std_normalization 442 | self.samplewise_std_normalization = samplewise_std_normalization 443 | self.zca_whitening = zca_whitening 444 | self.zca_epsilon = zca_epsilon 445 | self.rotation_range = rotation_range 446 | self.width_shift_range = width_shift_range 447 | self.height_shift_range = height_shift_range 448 | self.shear_range = shear_range 449 | self.zoom_range = zoom_range 450 | self.channel_shift_range = channel_shift_range 451 | self.fill_mode = fill_mode 452 | self.cval = cval 453 | self.horizontal_flip = horizontal_flip 454 | self.vertical_flip = vertical_flip 455 | self.rescale = rescale 456 | self.preprocessing_function = preprocessing_function 457 | 458 | if data_format not in {'channels_last', 'channels_first'}: 459 | raise ValueError('`data_format` should be `"channels_last"` (channel after row and ' 460 | 'column) or `"channels_first"` (channel before row and column). ' 461 | 'Received arg: ', data_format) 462 | self.data_format = data_format 463 | if data_format == 'channels_first': 464 | self.channel_axis = 1 465 | self.row_axis = 2 466 | self.col_axis = 3 467 | if data_format == 'channels_last': 468 | self.channel_axis = 3 469 | self.row_axis = 1 470 | self.col_axis = 2 471 | 472 | self.mean = None 473 | self.std = None 474 | self.principal_components = None 475 | 476 | if np.isscalar(zoom_range): 477 | self.zoom_range = [1 - zoom_range, 1 + zoom_range] 478 | elif len(zoom_range) == 2: 479 | self.zoom_range = [zoom_range[0], zoom_range[1]] 480 | else: 481 | raise ValueError('`zoom_range` should be a float or ' 482 | 'a tuple or list of two floats. ' 483 | 'Received arg: ', zoom_range) 484 | 485 | def flow(self, x, y=None, batch_size=32, shuffle=True, seed=None, 486 | save_to_dir=None, save_prefix='', save_format='png'): 487 | return NumpyArrayIterator( 488 | x, y, self, 489 | batch_size=batch_size, 490 | shuffle=shuffle, 491 | seed=seed, 492 | data_format=self.data_format, 493 | save_to_dir=save_to_dir, 494 | save_prefix=save_prefix, 495 | save_format=save_format) 496 | 497 | def flow_from_directory(self, directory, 498 | target_size=(256, 256), color_mode='rgb', 499 | classes=None, class_mode='categorical', 500 | batch_size=32, shuffle=True, seed=None, 501 | save_to_dir=None, 502 | save_prefix='', 503 | save_format='png', 504 | follow_links=False, 505 | h5=False): 506 | return DirectoryIterator( 507 | directory, self, 508 | target_size=target_size, color_mode=color_mode, 509 | classes=classes, class_mode=class_mode, 510 | data_format=self.data_format, 511 | batch_size=batch_size, shuffle=shuffle, seed=seed, 512 | save_to_dir=save_to_dir, 513 | save_prefix=save_prefix, 514 | save_format=save_format, 515 | follow_links=follow_links, 516 | h5=h5) 517 | 518 | def standardize(self, x): 519 | """Apply the normalization configuration to a batch of inputs. 520 | 521 | # Arguments 522 | x: batch of inputs to be normalized. 523 | 524 | # Returns 525 | The inputs, normalized. 526 | """ 527 | if self.preprocessing_function: 528 | x = self.preprocessing_function(x) 529 | if self.rescale: 530 | x *= self.rescale 531 | # x is a single image, so it doesn't have image number at index 0 532 | img_channel_axis = self.channel_axis - 1 533 | if self.samplewise_center: 534 | x -= np.mean(x, axis=img_channel_axis, keepdims=True) 535 | if self.samplewise_std_normalization: 536 | x /= (np.std(x, axis=img_channel_axis, keepdims=True) + 1e-7) 537 | 538 | if self.featurewise_center: 539 | if self.mean is not None: 540 | x -= self.mean 541 | else: 542 | warnings.warn('This ImageDataGenerator specifies ' 543 | '`featurewise_center`, but it hasn\'t' 544 | 'been fit on any training data. Fit it ' 545 | 'first by calling `.fit(numpy_data)`.') 546 | if self.featurewise_std_normalization: 547 | if self.std is not None: 548 | x /= (self.std + 1e-7) 549 | else: 550 | warnings.warn('This ImageDataGenerator specifies ' 551 | '`featurewise_std_normalization`, but it hasn\'t' 552 | 'been fit on any training data. Fit it ' 553 | 'first by calling `.fit(numpy_data)`.') 554 | if self.zca_whitening: 555 | if self.principal_components is not None: 556 | flatx = np.reshape(x, (-1, np.prod(x.shape[-3:]))) 557 | whitex = np.dot(flatx, self.principal_components) 558 | x = np.reshape(whitex, x.shape) 559 | else: 560 | warnings.warn('This ImageDataGenerator specifies ' 561 | '`zca_whitening`, but it hasn\'t' 562 | 'been fit on any training data. Fit it ' 563 | 'first by calling `.fit(numpy_data)`.') 564 | return x 565 | 566 | def random_transform(self, x, seed=None): 567 | """Randomly augment a single image tensor. 568 | 569 | # Arguments 570 | x: 3D tensor, single image. 571 | seed: random seed. 572 | 573 | # Returns 574 | A randomly transformed version of the input (same shape). 575 | """ 576 | # x is a single image, so it doesn't have image number at index 0 577 | img_row_axis = self.row_axis - 1 578 | img_col_axis = self.col_axis - 1 579 | img_channel_axis = self.channel_axis - 1 580 | 581 | if seed is not None: 582 | np.random.seed(seed) 583 | 584 | # use composition of homographies 585 | # to generate final transform that needs to be applied 586 | if self.rotation_range: 587 | theta = np.pi / 180 * np.random.uniform(-self.rotation_range, self.rotation_range) 588 | else: 589 | theta = 0 590 | 591 | if self.height_shift_range: 592 | tx = np.random.uniform(-self.height_shift_range, self.height_shift_range) * x.shape[img_row_axis] 593 | else: 594 | tx = 0 595 | 596 | if self.width_shift_range: 597 | ty = np.random.uniform(-self.width_shift_range, self.width_shift_range) * x.shape[img_col_axis] 598 | else: 599 | ty = 0 600 | 601 | if self.shear_range: 602 | shear = np.random.uniform(-self.shear_range, self.shear_range) 603 | else: 604 | shear = 0 605 | 606 | if self.zoom_range[0] == 1 and self.zoom_range[1] == 1: 607 | zx, zy = 1, 1 608 | else: 609 | zx, zy = np.random.uniform(self.zoom_range[0], self.zoom_range[1], 2) 610 | 611 | transform_matrix = None 612 | if theta != 0: 613 | rotation_matrix = np.array([[np.cos(theta), -np.sin(theta), 0], 614 | [np.sin(theta), np.cos(theta), 0], 615 | [0, 0, 1]]) 616 | transform_matrix = rotation_matrix 617 | 618 | if tx != 0 or ty != 0: 619 | shift_matrix = np.array([[1, 0, tx], 620 | [0, 1, ty], 621 | [0, 0, 1]]) 622 | transform_matrix = shift_matrix if transform_matrix is None else np.dot(transform_matrix, shift_matrix) 623 | 624 | if shear != 0: 625 | shear_matrix = np.array([[1, -np.sin(shear), 0], 626 | [0, np.cos(shear), 0], 627 | [0, 0, 1]]) 628 | transform_matrix = shear_matrix if transform_matrix is None else np.dot(transform_matrix, shear_matrix) 629 | 630 | if zx != 1 or zy != 1: 631 | zoom_matrix = np.array([[zx, 0, 0], 632 | [0, zy, 0], 633 | [0, 0, 1]]) 634 | transform_matrix = zoom_matrix if transform_matrix is None else np.dot(transform_matrix, zoom_matrix) 635 | 636 | if transform_matrix is not None: 637 | h, w = x.shape[img_row_axis], x.shape[img_col_axis] 638 | transform_matrix = transform_matrix_offset_center(transform_matrix, h, w) 639 | x = apply_transform(x, transform_matrix, img_channel_axis, 640 | fill_mode=self.fill_mode, cval=self.cval) 641 | 642 | if self.channel_shift_range != 0: 643 | x = random_channel_shift(x, 644 | self.channel_shift_range, 645 | img_channel_axis) 646 | if self.horizontal_flip: 647 | if np.random.random() < 0.5: 648 | x = flip_axis(x, img_col_axis) 649 | 650 | if self.vertical_flip: 651 | if np.random.random() < 0.5: 652 | x = flip_axis(x, img_row_axis) 653 | 654 | return x 655 | 656 | def fit(self, x, 657 | augment=False, 658 | rounds=1, 659 | seed=None): 660 | """Fits internal statistics to some sample data. 661 | 662 | Required for featurewise_center, featurewise_std_normalization 663 | and zca_whitening. 664 | 665 | # Arguments 666 | x: Numpy array, the data to fit on. Should have rank 4. 667 | In case of grayscale data, 668 | the channels axis should have value 1, and in case 669 | of RGB data, it should have value 3. 670 | augment: Whether to fit on randomly augmented samples 671 | rounds: If `augment`, 672 | how many augmentation passes to do over the data 673 | seed: random seed. 674 | 675 | # Raises 676 | ValueError: in case of invalid input `x`. 677 | """ 678 | x = np.asarray(x, dtype=backend.floatx()) 679 | if x.ndim != 4: 680 | raise ValueError('Input to `.fit()` should have rank 4. ' 681 | 'Got array with shape: ' + str(x.shape)) 682 | if x.shape[self.channel_axis] not in {1, 3, 4}: 683 | warnings.warn( 684 | 'Expected input to be images (as Numpy array) ' 685 | 'following the data format convention "' + self.data_format + '" ' 686 | '(channels on axis ' + str(self.channel_axis) + '), i.e. expected ' 687 | 'either 1, 3 or 4 channels on axis ' + str(self.channel_axis) + '. ' 688 | 'However, it was passed an array with shape ' + str(x.shape) + 689 | ' (' + str(x.shape[self.channel_axis]) + ' channels).') 690 | 691 | if seed is not None: 692 | np.random.seed(seed) 693 | 694 | x = np.copy(x) 695 | if augment: 696 | ax = np.zeros(tuple([rounds * x.shape[0]] + list(x.shape)[1:]), dtype=backend.floatx()) 697 | for r in range(rounds): 698 | for i in range(x.shape[0]): 699 | ax[i + r * x.shape[0]] = self.random_transform(x[i]) 700 | x = ax 701 | 702 | if self.featurewise_center: 703 | self.mean = np.mean(x, axis=(0, self.row_axis, self.col_axis)) 704 | broadcast_shape = [1, 1, 1] 705 | broadcast_shape[self.channel_axis - 1] = x.shape[self.channel_axis] 706 | self.mean = np.reshape(self.mean, broadcast_shape) 707 | x -= self.mean 708 | 709 | if self.featurewise_std_normalization: 710 | self.std = np.std(x, axis=(0, self.row_axis, self.col_axis)) 711 | broadcast_shape = [1, 1, 1] 712 | broadcast_shape[self.channel_axis - 1] = x.shape[self.channel_axis] 713 | self.std = np.reshape(self.std, broadcast_shape) 714 | x /= (self.std + backend.epsilon()) 715 | 716 | if self.zca_whitening: 717 | flat_x = np.reshape(x, (x.shape[0], x.shape[1] * x.shape[2] * x.shape[3])) 718 | sigma = np.dot(flat_x.T, flat_x) / flat_x.shape[0] 719 | u, s, _ = linalg.svd(sigma) 720 | self.principal_components = np.dot(np.dot(u, np.diag(1. / np.sqrt(s + self.zca_epsilon))), u.T) 721 | 722 | 723 | class Iterator(Sequence): 724 | """Base class for image data iterators. 725 | 726 | Every `Iterator` must implement the `_get_batches_of_transformed_samples` 727 | method. 728 | 729 | # Arguments 730 | n: Integer, total number of samples in the dataset to loop over. 731 | batch_size: Integer, size of a batch. 732 | shuffle: Boolean, whether to shuffle the data between epochs. 733 | seed: Random seeding for data shuffling. 734 | """ 735 | 736 | def __init__(self, n, batch_size, shuffle, seed): 737 | self.n = n 738 | self.batch_size = batch_size 739 | self.seed = seed 740 | self.shuffle = shuffle 741 | self.batch_index = 0 742 | self.total_batches_seen = 0 743 | self.lock = threading.Lock() 744 | self.index_array = None 745 | self.index_generator = self._flow_index() 746 | 747 | def _set_index_array(self): 748 | self.index_array = np.arange(self.n) 749 | if self.shuffle: 750 | self.index_array = np.random.permutation(self.n) 751 | 752 | def __getitem__(self, idx): 753 | if idx >= len(self): 754 | raise ValueError('Asked to retrieve element {idx}, ' 755 | 'but the Sequence ' 756 | 'has length {length}'.format(idx=idx, 757 | length=len(self))) 758 | if self.seed is not None: 759 | np.random.seed(self.seed + self.total_batches_seen) 760 | self.total_batches_seen += 1 761 | if self.index_array is None: 762 | self._set_index_array() 763 | index_array = self.index_array[self.batch_size * idx: 764 | self.batch_size * (idx + 1)] 765 | return self._get_batches_of_transformed_samples(index_array) 766 | 767 | def __len__(self): 768 | return int(np.ceil(self.n / float(self.batch_size))) 769 | 770 | def on_epoch_end(self): 771 | self._set_index_array() 772 | 773 | def reset(self): 774 | self.batch_index = 0 775 | 776 | def _flow_index(self): 777 | # Ensure self.batch_index is 0. 778 | self.reset() 779 | while 1: 780 | if self.seed is not None: 781 | np.random.seed(self.seed + self.total_batches_seen) 782 | if self.batch_index == 0: 783 | self._set_index_array() 784 | 785 | current_index = (self.batch_index * self.batch_size) % self.n 786 | if self.n > current_index + self.batch_size: 787 | self.batch_index += 1 788 | else: 789 | self.batch_index = 0 790 | self.total_batches_seen += 1 791 | yield self.index_array[current_index: 792 | current_index + self.batch_size] 793 | 794 | def __iter__(self): 795 | # Needed if we want to do something like: 796 | # for x, y in data_gen.flow(...): 797 | return self 798 | 799 | def __next__(self, *args, **kwargs): 800 | return self.next(*args, **kwargs) 801 | 802 | def _get_batches_of_transformed_samples(self, index_array): 803 | """Gets a batch of transformed samples. 804 | 805 | # Arguments 806 | index_array: array of sample indices to include in batch. 807 | 808 | # Returns 809 | A batch of transformed samples. 810 | """ 811 | raise NotImplementedError 812 | 813 | 814 | class NumpyArrayIterator(Iterator): 815 | """Iterator yielding data from a Numpy array. 816 | 817 | # Arguments 818 | x: Numpy array of input data. 819 | y: Numpy array of targets data. 820 | image_data_generator: Instance of `ImageDataGenerator` 821 | to use for random transformations and normalization. 822 | batch_size: Integer, size of a batch. 823 | shuffle: Boolean, whether to shuffle the data between epochs. 824 | seed: Random seed for data shuffling. 825 | data_format: String, one of `channels_first`, `channels_last`. 826 | save_to_dir: Optional directory where to save the pictures 827 | being yielded, in a viewable format. This is useful 828 | for visualizing the random transformations being 829 | applied, for debugging purposes. 830 | save_prefix: String prefix to use for saving sample 831 | images (if `save_to_dir` is set). 832 | save_format: Format to use for saving sample images 833 | (if `save_to_dir` is set). 834 | """ 835 | 836 | def __init__(self, x, y, image_data_generator, 837 | batch_size=32, shuffle=False, seed=None, 838 | data_format=None, 839 | save_to_dir=None, save_prefix='', save_format='png'): 840 | if y is not None and len(x) != len(y): 841 | raise ValueError('X (images tensor) and y (labels) ' 842 | 'should have the same length. ' 843 | 'Found: X.shape = %s, y.shape = %s' % 844 | (np.asarray(x).shape, np.asarray(y).shape)) 845 | 846 | if data_format is None: 847 | data_format = backend.image_data_format() 848 | self.x = np.asarray(x, dtype=backend.floatx()) 849 | 850 | if self.x.ndim != 4: 851 | raise ValueError('Input data in `NumpyArrayIterator` ' 852 | 'should have rank 4. You passed an array ' 853 | 'with shape', self.x.shape) 854 | channels_axis = 3 if data_format == 'channels_last' else 1 855 | if self.x.shape[channels_axis] not in {1, 3, 4}: 856 | warnings.warn('NumpyArrayIterator is set to use the ' 857 | 'data format convention "' + data_format + '" ' 858 | '(channels on axis ' + str(channels_axis) + '), i.e. expected ' 859 | 'either 1, 3 or 4 channels on axis ' + str(channels_axis) + '. ' 860 | 'However, it was passed an array with shape ' + str(self.x.shape) + 861 | ' (' + str(self.x.shape[channels_axis]) + ' channels).') 862 | if y is not None: 863 | self.y = np.asarray(y) 864 | else: 865 | self.y = None 866 | self.image_data_generator = image_data_generator 867 | self.data_format = data_format 868 | self.save_to_dir = save_to_dir 869 | self.save_prefix = save_prefix 870 | self.save_format = save_format 871 | super(NumpyArrayIterator, self).__init__(x.shape[0], batch_size, shuffle, seed) 872 | 873 | def _get_batches_of_transformed_samples(self, index_array): 874 | batch_x = np.zeros(tuple([len(index_array)] + list(self.x.shape)[1:]), 875 | dtype=backend.floatx()) 876 | for i, j in enumerate(index_array): 877 | x = self.x[j] 878 | x = self.image_data_generator.random_transform(x.astype(backend.floatx())) 879 | x = self.image_data_generator.standardize(x) 880 | batch_x[i] = x 881 | if self.save_to_dir: 882 | for i, j in enumerate(index_array): 883 | img = array_to_img(batch_x[i], self.data_format, scale=True) 884 | fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix, 885 | index=j, 886 | hash=np.random.randint(1e4), 887 | format=self.save_format) 888 | img.save(os.path.join(self.save_to_dir, fname)) 889 | if self.y is None: 890 | return batch_x 891 | batch_y = self.y[index_array] 892 | return batch_x, batch_y 893 | 894 | def next(self): 895 | """For python 2.x. 896 | 897 | # Returns 898 | The next batch. 899 | """ 900 | # Keeps under lock only the mechanism which advances 901 | # the indexing of each batch. 902 | with self.lock: 903 | index_array = next(self.index_generator) 904 | # The transformation of images is not under thread lock 905 | # so it can be done in parallel 906 | return self._get_batches_of_transformed_samples(index_array) 907 | 908 | 909 | def _count_valid_files_in_directory(directory, white_list_formats, follow_links): 910 | """Count files with extension in `white_list_formats` contained in a directory. 911 | 912 | # Arguments 913 | directory: absolute path to the directory containing files to be counted 914 | white_list_formats: set of strings containing allowed extensions for 915 | the files to be counted. 916 | 917 | # Returns 918 | the count of files with extension in `white_list_formats` contained in 919 | the directory. 920 | """ 921 | def _recursive_list(subpath): 922 | return sorted(os.walk(subpath, followlinks=follow_links), key=lambda tpl: tpl[0]) 923 | 924 | samples = 0 925 | for root, _, files in _recursive_list(directory): 926 | for fname in files: 927 | is_valid = False 928 | for extension in white_list_formats: 929 | if fname.lower().endswith('.' + extension): 930 | is_valid = True 931 | break 932 | if is_valid: 933 | samples += 1 934 | return samples 935 | 936 | 937 | def _list_valid_filenames_in_directory(directory, white_list_formats, 938 | class_indices, follow_links): 939 | """List paths of files in `subdir` relative from `directory` whose extensions are in `white_list_formats`. 940 | 941 | # Arguments 942 | directory: absolute path to a directory containing the files to list. 943 | The directory name is used as class label and must be a key of `class_indices`. 944 | white_list_formats: set of strings containing allowed extensions for 945 | the files to be counted. 946 | class_indices: dictionary mapping a class name to its index. 947 | 948 | # Returns 949 | classes: a list of class indices 950 | filenames: the path of valid files in `directory`, relative from 951 | `directory`'s parent (e.g., if `directory` is "dataset/class1", 952 | the filenames will be ["class1/file1.jpg", "class1/file2.jpg", ...]). 953 | """ 954 | def _recursive_list(subpath): 955 | return sorted(os.walk(subpath, followlinks=follow_links), key=lambda tpl: tpl[0]) 956 | 957 | classes = [] 958 | filenames = [] 959 | subdir = os.path.basename(directory) 960 | basedir = os.path.dirname(directory) 961 | for root, _, files in _recursive_list(directory): 962 | for fname in sorted(files): 963 | is_valid = False 964 | for extension in white_list_formats: 965 | if fname.lower().endswith('.' + extension): 966 | is_valid = True 967 | break 968 | if is_valid: 969 | classes.append(class_indices[subdir]) 970 | # add filename relative to directory 971 | absolute_path = os.path.join(root, fname) 972 | filenames.append(os.path.relpath(absolute_path, basedir)) 973 | return classes, filenames 974 | 975 | 976 | class DirectoryIterator(Iterator): 977 | """Iterator capable of reading images from a directory on disbackend. 978 | 979 | # Arguments 980 | directory: Path to the directory to read images from. 981 | Each subdirectory in this directory will be 982 | considered to contain images from one class, 983 | or alternatively you could specify class subdirectories 984 | via the `classes` argument. 985 | image_data_generator: Instance of `ImageDataGenerator` 986 | to use for random transformations and normalization. 987 | target_size: tuple of integers, dimensions to resize input images to. 988 | color_mode: One of `"rgb"`, `"grayscale"`. Color mode to read images. 989 | classes: Optional list of strings, names of subdirectories 990 | containing images from each class (e.g. `["dogs", "cats"]`). 991 | It will be computed automatically if not set. 992 | class_mode: Mode for yielding the targets: 993 | `"binary"`: binary targets (if there are only two classes), 994 | `"categorical"`: categorical targets, 995 | `"sparse"`: integer targets, 996 | `"input"`: targets are images identical to input images (mainly 997 | used to work with autoencoders), 998 | `None`: no targets get yielded (only input images are yielded). 999 | batch_size: Integer, size of a batch. 1000 | shuffle: Boolean, whether to shuffle the data between epochs. 1001 | seed: Random seed for data shuffling. 1002 | data_format: String, one of `channels_first`, `channels_last`. 1003 | save_to_dir: Optional directory where to save the pictures 1004 | being yielded, in a viewable format. This is useful 1005 | for visualizing the random transformations being 1006 | applied, for debugging purposes. 1007 | save_prefix: String prefix to use for saving sample 1008 | images (if `save_to_dir` is set). 1009 | save_format: Format to use for saving sample images 1010 | (if `save_to_dir` is set). 1011 | """ 1012 | 1013 | def __init__(self, directory, image_data_generator, 1014 | target_size=(256, 256), color_mode='rgb', 1015 | classes=None, class_mode='categorical', 1016 | batch_size=32, shuffle=True, seed=None, 1017 | data_format=None, 1018 | save_to_dir=None, save_prefix='', save_format='png', 1019 | follow_links=False, h5=False): 1020 | if data_format is None: 1021 | data_format = backend.image_data_format() 1022 | self.directory = directory 1023 | self.image_data_generator = image_data_generator 1024 | self.target_size = tuple(target_size) 1025 | if color_mode not in {'rgb', 'grayscale'}: 1026 | raise ValueError('Invalid color mode:', color_mode, 1027 | '; expected "rgb" or "grayscale".') 1028 | self.color_mode = color_mode 1029 | self.data_format = data_format 1030 | if self.color_mode == 'rgb': 1031 | if self.data_format == 'channels_last': 1032 | self.image_shape = self.target_size + (3,) 1033 | else: 1034 | self.image_shape = (3,) + self.target_size 1035 | else: 1036 | if self.data_format == 'channels_last': 1037 | self.image_shape = self.target_size + (1,) 1038 | else: 1039 | self.image_shape = (1,) + self.target_size 1040 | self.classes = classes 1041 | if class_mode not in {'categorical', 'binary', 'sparse', 1042 | 'input', None}: 1043 | raise ValueError('Invalid class_mode:', class_mode, 1044 | '; expected one of "categorical", ' 1045 | '"binary", "sparse", "input"' 1046 | ' or None.') 1047 | self.class_mode = class_mode 1048 | self.save_to_dir = save_to_dir 1049 | self.save_prefix = save_prefix 1050 | self.save_format = save_format 1051 | self.h5 = h5 1052 | 1053 | white_list_formats = {'png', 'jpg', 'jpeg', 'bmp', 'ppm', 'npy'} 1054 | 1055 | # first, count the number of samples and classes 1056 | self.samples = 0 1057 | 1058 | if not classes: 1059 | classes = [] 1060 | for subdir in sorted(os.listdir(directory)): 1061 | if os.path.isdir(os.path.join(directory, subdir)): 1062 | classes.append(subdir) 1063 | self.num_classes = len(classes) 1064 | self.class_indices = dict(zip(classes, range(len(classes)))) 1065 | 1066 | pool = multiprocessing.pool.ThreadPool() 1067 | function_partial = partial(_count_valid_files_in_directory, 1068 | white_list_formats=white_list_formats, 1069 | follow_links=follow_links) 1070 | self.samples = sum(pool.map(function_partial, 1071 | (os.path.join(directory, subdir) 1072 | for subdir in classes))) 1073 | 1074 | print('Found %d images belonging to %d classes.' % (self.samples, self.num_classes)) 1075 | 1076 | # second, build an index of the images in the different class subfolders 1077 | results = [] 1078 | 1079 | self.filenames = [] 1080 | self.classes = np.zeros((self.samples,), dtype='int32') 1081 | i = 0 1082 | for dirpath in (os.path.join(directory, subdir) for subdir in classes): 1083 | results.append(pool.apply_async(_list_valid_filenames_in_directory, 1084 | (dirpath, white_list_formats, 1085 | self.class_indices, follow_links))) 1086 | for res in results: 1087 | classes, filenames = res.get() 1088 | self.classes[i:i + len(classes)] = classes 1089 | self.filenames += filenames 1090 | i += len(classes) 1091 | pool.close() 1092 | pool.join() 1093 | super(DirectoryIterator, self).__init__(self.samples, batch_size, shuffle, seed) 1094 | 1095 | def _get_batches_of_transformed_samples(self, index_array): 1096 | 1097 | def file_to_array(image_file, fname): 1098 | img_path = os.path.join(image_file, fname) 1099 | 1100 | img = load_img(os.path.join(img_path), 1101 | grayscale=grayscale, 1102 | target_size=self.target_size) 1103 | x = img_to_array(img, data_format=self.data_format) 1104 | x = self.image_data_generator.random_transform(x) 1105 | x = self.image_data_generator.standardize(x) 1106 | 1107 | return x 1108 | 1109 | # batch_x = np.zeros((len(index_array),) + self.image_shape, dtype=backend.floatx()) 1110 | grayscale = self.color_mode == 'grayscale' 1111 | if self.h5 == True: 1112 | batch_x = np.zeros((len(index_array),) + (5,224,224,3), dtype=backend.floatx()) 1113 | 1114 | 1115 | # build batch of h5 files 1116 | for i, j in enumerate(index_array): 1117 | fname = self.filenames[j] 1118 | 1119 | # Read h5 files 1120 | f = np.load(os.path.join(self.directory, fname)) 1121 | 1122 | file_path = self.directory.replace('consecutive', 'preface') 1123 | file_list = [os.path.basename(fi) for fi in f] 1124 | 1125 | x = [np.array(file_to_array(file_path, os.path.join(os.path.dirname(fname), file.replace('.h5', '.jpeg')))) for file in file_list] 1126 | # Normalize 1127 | batch_x[i] = x 1128 | 1129 | # optionally save augmented images to disk for debugging purposes 1130 | if self.save_to_dir: 1131 | for i, j in enumerate(index_array): 1132 | img = array_to_img(batch_x[i], self.data_format, scale=True) 1133 | fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix, 1134 | index=j, 1135 | hash=np.random.randint(1e4), 1136 | format=self.save_format) 1137 | img.save(os.path.join(self.save_to_dir, fname)) 1138 | # build batch of labels 1139 | if self.class_mode == 'input': 1140 | batch_y = batch_x.copy() 1141 | elif self.class_mode == 'sparse': 1142 | batch_y = self.classes[index_array] 1143 | elif self.class_mode == 'binary': 1144 | batch_y = self.classes[index_array].astype(backend.floatx()) 1145 | elif self.class_mode == 'categorical': 1146 | batch_y = np.zeros((len(batch_x), self.num_classes), dtype=backend.floatx()) 1147 | for i, label in enumerate(self.classes[index_array]): 1148 | batch_y[i, label] = 1. 1149 | else: 1150 | return batch_x 1151 | return batch_x, batch_y 1152 | 1153 | def next(self): 1154 | """For python 2.x. 1155 | 1156 | # Returns 1157 | The next batch. 1158 | """ 1159 | with self.lock: 1160 | index_array = next(self.index_generator) 1161 | # The transformation of images is not under thread lock 1162 | # so it can be done in parallel 1163 | return self._get_batches_of_transformed_samples(index_array) 1164 | -------------------------------------------------------------------------------- /Models/CNN+LSTM/pretrained_lstm.py: -------------------------------------------------------------------------------- 1 | import os 2 | os.environ['KERAS_BACKEND']='tensorflow' 3 | import keras 4 | from keras.engine import Model 5 | from keras_vggface.vggface import VGGFace 6 | from keras.layers import Flatten, Dense, Input, Dropout 7 | 8 | from utils import eucl_dist_ang, euclidean_distance 9 | 10 | 11 | def define_model(n_output, lr, dropout, n_units, features_model): 12 | """ 13 | Define model architecture, optimization and loss function 14 | """ 15 | 16 | # Initialize weights 17 | weight_init = keras.initializers.glorot_uniform(seed=3) 18 | 19 | # Initialize optimizer 20 | opt = keras.optimizers.adam(lr) 21 | 22 | 23 | # Load pre-trained model 24 | if features_model is not None: 25 | cnn_model = keras.models.load_model(features_model, custom_objects={'euclidean_distance': euclidean_distance, 26 | 'eucl_dist_ang': eucl_dist_ang}) 27 | else: 28 | return 0 29 | 30 | model_input = keras.layers.Input(shape=(None,224,224,3), name='seq_input') 31 | 32 | # little hack to remove the last layers - we know that there are 7 layers after the last cnn layer, so we remove them 33 | # using pop 34 | cnn_model.layers.pop() 35 | cnn_model.layers.pop() 36 | cnn_model.layers.pop() 37 | cnn_model.layers.pop() 38 | cnn_model.layers.pop() 39 | cnn_model.layers.pop() 40 | cnn_model.layers.pop() 41 | cnn_model.layers[-1].outbound_nodes = [] 42 | cnn_model.outputs = [cnn_model.layers[-1].output] 43 | 44 | for layer in cnn_model.layers: 45 | layer.trainable = False 46 | 47 | cnn_model.summary() 48 | 49 | # x = keras.layers.TimeDistributed(keras.layers.Lambda(lambda x: cnn_model(x)))(model_input) 50 | x = keras.layers.TimeDistributed(Flatten())(model_input) 51 | 52 | x = keras.layers.LSTM(n_units, dropout=dropout, name='lstm')(x) 53 | out = Dense(n_output, kernel_initializer=weight_init, name='out')(x) 54 | 55 | model = Model(inputs=[model_input], outputs=out) 56 | 57 | model.summary() 58 | 59 | print(len(model.layers)) 60 | print([n.name for n in model.layers]) 61 | 62 | # Use mean euclidean distance as loss and angular error and mse as metric 63 | model.compile(loss=euclidean_distance, 64 | optimizer=opt, 65 | metrics=[eucl_dist_ang, 'mse']) 66 | 67 | return model 68 | -------------------------------------------------------------------------------- /Models/CNN/vggFace.py: -------------------------------------------------------------------------------- 1 | # export CUDA_VISIBLE_DEVICES=1,2 2 | import os 3 | import sys 4 | import time 5 | import os.path as osp 6 | 7 | # Add the parent module to the import path 8 | sys.path.append(osp.realpath(osp.join(osp.dirname(__file__), '../'))) 9 | 10 | from vggface_model import VGGFace 11 | 12 | from keras.models import Model 13 | from keras.optimizers import adam 14 | from keras.layers import Flatten, Dense 15 | from keras.preprocessing.image import ImageDataGenerator 16 | from keras.callbacks import ModelCheckpoint, CSVLogger, TensorBoard 17 | 18 | from collections import OrderedDict 19 | from _saving.SaveModelDirectory import create_path 20 | from _saving.ModelParameters import ModelParameters 21 | 22 | __author__ = 'Daniel Garcia Zapata' 23 | 24 | 25 | # Datagen 26 | def obtain_datagen(datagen, path): 27 | return datagen.flow_from_directory( 28 | path, 29 | target_size=(img_height, img_width), 30 | batch_size=batch_size, 31 | class_mode='binary') 32 | 33 | if __name__ == '__main__': 34 | 35 | print "Starting:", time.ctime() 36 | 37 | ########################################### 38 | # Parameters 39 | 40 | epochs = 20 41 | batch_size = 40 42 | 43 | ########################################### 44 | # Data 45 | 46 | img_width, img_height, channel = 224, 224, 3 # Resolution of inputs 47 | nb_class = 6 48 | 49 | dataset = 'OULU-CASIA' 50 | partition = 'preface' # prealigned preface 51 | train_data_dir = osp.join('..', '..', '_Dataset', dataset, partition, 'training') 52 | validation_data_dir = osp.join('..', '..', '_Dataset', dataset, partition, 'validation') 53 | 54 | __model__ = 'CNN_' + partition 55 | 56 | if dataset == 'OULU-CASIA': 57 | nb_train_samples = 5517 58 | nb_validation_samples = 449 59 | else: 60 | nb_train_samples = 29798 61 | nb_validation_samples = 4428 62 | 63 | nb_train_samples = nb_train_samples / batch_size 64 | nb_validation_samples = nb_validation_samples / batch_size 65 | 66 | ''' 67 | Load the model 68 | ''' 69 | vgg_model = VGGFace( 70 | include_top=False, 71 | input_shape=(img_width, img_height, channel)) 72 | 73 | ''' 74 | Custom the model 75 | ''' 76 | last_layer = vgg_model.get_layer('pool5').output 77 | x = Flatten(name='flatten')(last_layer) 78 | x = Dense(4096, activation='relu', name='fc6')(x) 79 | x = Dense(4096, activation='relu', name='fc7')(x) 80 | x = Dense(2622, activation='relu', name='fc8')(x) 81 | out = Dense(nb_class, activation='softmax', name='fc9')(x) 82 | 83 | model = Model( 84 | vgg_model.input, 85 | out) 86 | 87 | ''' 88 | Dataset Generators 89 | ''' 90 | datagen = ImageDataGenerator( 91 | rescale= 1./224, 92 | shear_range= 0.2, 93 | zoom_range= 0.2, 94 | horizontal_flip= True) 95 | 96 | train_generator = obtain_datagen(datagen, train_data_dir) 97 | validation_generator = obtain_datagen(datagen, validation_data_dir) 98 | 99 | ''' 100 | Freeze previous layers 101 | ''' 102 | for i, layer in enumerate(vgg_model.layers): 103 | layer.trainable = False 104 | 105 | ''' 106 | Compile the model 107 | ''' 108 | lr = 0.0001 109 | optimizer = adam(lr=lr) 110 | loss = 'sparse_categorical_crossentropy' 111 | model.compile( loss=loss, 112 | optimizer=optimizer, 113 | metrics=['accuracy', 'top_k_categorical_accuracy']) 114 | 115 | ''' 116 | Callbacks 117 | ''' 118 | row_dict = OrderedDict({'model': __model__, 119 | 'dataset': dataset, 120 | 'partition': partition, 121 | 'loss': loss, 122 | 'lr': lr, 123 | 'date': time.ctime()}) 124 | 125 | # Create Version folder 126 | export_path = create_path(__model__, dataset) 127 | 128 | checkpointer = ModelCheckpoint(filepath=osp.join(export_path, __model__+'_epoch-{epoch:02d}_val-accu-{val_acc:.4f}.hdf5'), verbose=1) #, save_best_only=True) 129 | csv_logger = CSVLogger(osp.join(export_path, '_logs_'+__model__+'.log'), separator=',', append=False) 130 | tensorboard = TensorBoard(log_dir=export_path, histogram_freq=0, batch_size=batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) 131 | model_parameters = ModelParameters(osp.join(export_path, '_model_'+__model__+'.log'), row_dict) 132 | 133 | ''' 134 | Train the model 135 | ''' 136 | print('*************************************\nTraining \n*************************************') 137 | model.fit_generator( 138 | train_generator, 139 | steps_per_epoch=nb_train_samples, 140 | epochs=epochs, 141 | validation_data=validation_generator, 142 | validation_steps=nb_validation_samples, 143 | callbacks=[checkpointer, csv_logger, tensorboard, model_parameters]) 144 | 145 | print("Ending:", time.ctime()) 146 | -------------------------------------------------------------------------------- /Models/CNN/vggface_model.py: -------------------------------------------------------------------------------- 1 | '''VGGFace model for Keras. 2 | # Reference: 3 | - [Deep Face Recognition](http://www.robots.ox.ac.uk/~vgg/publications/2015/Parkhi15/parkhi15.pdf) 4 | ''' 5 | from __future__ import print_function 6 | import warnings 7 | 8 | from keras.applications.imagenet_utils import _obtain_input_shape 9 | from keras.engine.topology import get_source_inputs 10 | from keras.models import Model 11 | from keras.layers import Flatten, Dense, Input, GlobalAveragePooling2D, GlobalMaxPooling2D, Activation 12 | from keras.layers import Convolution2D, MaxPooling2D 13 | from keras.utils import layer_utils 14 | from keras.utils.data_utils import get_file 15 | from keras import backend as K 16 | 17 | RESNET50_WEIGHTS_PATH = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_tf_resnet50.h5' 18 | RESNET50_WEIGHTS_PATH_NO_TOP = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_tf_notop_resnet50.h5' 19 | 20 | def VGGFace(include_top=False, weights='vggface', 21 | input_tensor=None, input_shape=None, 22 | pooling='max', 23 | classes=2622): 24 | """Instantiates the VGGFace architecture. 25 | Optionally loads weights pre-trained 26 | on VGGFace dataset. Note that when using TensorFlow, 27 | for best performance you should set 28 | `image_data_format="channels_last"` in your Keras config 29 | at ~/.keras/keras.json. 30 | The model and the weights are compatible with both 31 | TensorFlow and Theano. The data format 32 | convention used by the model is the one 33 | specified in your Keras config file. 34 | # Arguments 35 | include_top: whether to include the 3 fully-connected 36 | layers at the top of the network. 37 | weights: one of `None` (random initialization) 38 | or "imagenet" (pre-training on ImageNet). 39 | input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) 40 | to use as image input for the model. 41 | input_shape: optional shape tuple, only to be specified 42 | if `include_top` is False (otherwise the input shape 43 | has to be `(224, 224, 3)` (with `channels_last` data format) 44 | or `(3, 224, 244)` (with `channels_first` data format). 45 | It should have exactly 3 inputs channels, 46 | and width and height should be no smaller than 48. 47 | E.g. `(200, 200, 3)` would be one valid value. 48 | pooling: Optional pooling mode for feature extraction 49 | when `include_top` is `False`. 50 | - `None` means that the output of the model will be 51 | the 4D tensor output of the 52 | last convolutional layer. 53 | - `avg` means that global average pooling 54 | will be applied to the output of the 55 | last convolutional layer, and thus 56 | the output of the model will be a 2D tensor. 57 | - `max` means that global max pooling will 58 | be applied. 59 | classes: optional number of classes to classify images 60 | into, only to be specified if `include_top` is True, and 61 | if no `weights` argument is specified. 62 | # Returns 63 | A Keras model instance. 64 | # Raises 65 | ValueError: in case of invalid argument for `weights`, 66 | or invalid input shape. 67 | """ 68 | if weights not in {'vggface', None}: 69 | raise ValueError('The `weights` argument should be either ' 70 | '`None` (random initialization) or `vggface` ' 71 | '(pre-training on VGGFace Dataset).') 72 | 73 | if weights == 'vggface' and include_top and classes != 2622: 74 | raise ValueError('If using `weights` as vggface original with `include_top`' 75 | ' as true, `classes` should be 2622') 76 | # Determine proper input shape 77 | input_shape = _obtain_input_shape(input_shape, 78 | default_size=224, 79 | min_size=48, 80 | data_format=K.image_data_format(), 81 | require_flatten=include_top) 82 | 83 | if input_tensor is None: 84 | img_input = Input(shape=input_shape) 85 | else: 86 | if not K.is_keras_tensor(input_tensor): 87 | img_input = Input(tensor=input_tensor, shape=input_shape) 88 | else: 89 | img_input = input_tensor 90 | 91 | # Block 1 92 | x = Convolution2D(64, (3, 3), activation='relu', padding='same', name='conv1_1')(img_input) 93 | x = Convolution2D(64, (3, 3), activation='relu', padding='same', name='conv1_2')(x) 94 | x = MaxPooling2D((2, 2), strides=(2, 2), name='pool1')(x) 95 | 96 | # Block 2 97 | x = Convolution2D(128, (3, 3), activation='relu', padding='same', name='conv2_1')(x) 98 | x = Convolution2D(128, (3, 3), activation='relu', padding='same', name='conv2_2')(x) 99 | x = MaxPooling2D((2, 2), strides=(2, 2), name='pool2')(x) 100 | 101 | # Block 3 102 | x = Convolution2D(256, (3, 3), activation='relu', padding='same', name='conv3_1')(x) 103 | x = Convolution2D(256, (3, 3), activation='relu', padding='same', name='conv3_2')(x) 104 | x = Convolution2D(256, (3, 3), activation='relu', padding='same', name='conv3_3')(x) 105 | x = MaxPooling2D((2, 2), strides=(2, 2), name='pool3')(x) 106 | 107 | # Block 4 108 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv4_1')(x) 109 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv4_2')(x) 110 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv4_3')(x) 111 | x = MaxPooling2D((2, 2), strides=(2, 2), name='pool4')(x) 112 | 113 | # Block 5 114 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv5_1')(x) 115 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv5_2')(x) 116 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv5_3')(x) 117 | x = MaxPooling2D((2, 2), strides=(2, 2), name='pool5')(x) 118 | 119 | if include_top: 120 | # Classification block 121 | x = Flatten(name='flatten')(x) 122 | x = Dense(4096, name='fc6')(x) 123 | x = Activation('relu', name='fc6/relu')(x) 124 | x = Dense(4096, name='fc7')(x) 125 | x = Activation('relu', name='fc7/relu')(x) 126 | x = Dense(2622, name='fc8')(x) 127 | x = Activation('relu', name='fc8/softmax')(x) 128 | else: 129 | if pooling == 'avg': 130 | x = GlobalAveragePooling2D()(x) 131 | elif pooling == 'max': 132 | x = GlobalMaxPooling2D()(x) 133 | 134 | # Ensure that the model takes into account 135 | # any potential predecessors of `input_tensor`. 136 | if input_tensor is not None: 137 | inputs = get_source_inputs(input_tensor) 138 | else: 139 | inputs = img_input 140 | # Create model. 141 | model = Model(inputs, x, name='VGGFace') # load weights 142 | if weights == 'vggface': 143 | if include_top: 144 | weights_path = get_file('rcmalli_vggface_tf_v2.h5', 145 | WEIGHTS_PATH, 146 | cache_subdir='models') 147 | else: 148 | weights_path = get_file('rcmalli_vggface_tf_notop_resnet50.h5', 149 | RESNET50_WEIGHTS_PATH_NO_TOP, 150 | cache_subdir='models') 151 | model.load_weights(weights_path, by_name=True) 152 | if K.backend() == 'theano': 153 | layer_utils.convert_all_kernels_in_model(model) 154 | 155 | if K.image_data_format() == 'channels_first': 156 | if include_top: 157 | maxpool = model.get_layer(name='pool5') 158 | shape = maxpool.output_shape[1:] 159 | dense = model.get_layer(name='fc6') 160 | layer_utils.convert_dense_weights_data_format(dense, shape, 'channels_first') 161 | 162 | if K.backend() == 'tensorflow': 163 | warnings.warn('You are using the TensorFlow backend, yet you ' 164 | 'are using the Theano ' 165 | 'image data format convention ' 166 | '(`image_data_format="channels_first"`). ' 167 | 'For best performance, set ' 168 | '`image_data_format="channels_last"` in ' 169 | 'your Keras config ' 170 | 'at ~/.keras/keras.json.') 171 | return model -------------------------------------------------------------------------------- /Models/CNN_2Stream/preprocessing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dangz90/Deep-Learning-for-Expression-Recognition-in-Image-Sequences/66c50292cac9a66eace32a040c49267b06c45de5/Models/CNN_2Stream/preprocessing/__init__.py -------------------------------------------------------------------------------- /Models/CNN_2Stream/vggFace_MM.py: -------------------------------------------------------------------------------- 1 | # export CUDA_VISIBLE_DEVICES=1,2 2 | import os 3 | import sys 4 | import time 5 | import os.path as osp 6 | 7 | # Add the parent module to the import path 8 | sys.path.append(osp.realpath(osp.join(osp.dirname(__file__), '../'))) 9 | 10 | from keras.optimizers import adam 11 | from keras.models import Model, load_model 12 | from keras.layers import Flatten, Dense, Input, concatenate 13 | from keras.callbacks import ModelCheckpoint, CSVLogger, TensorBoard 14 | 15 | from collections import OrderedDict 16 | from preprocessing.image import ImageDataGenerator 17 | 18 | from _saving.SaveModelDirectory import create_path 19 | from _saving.ModelParameters import ModelParameters 20 | 21 | __author__ = 'Daniel Garcia Zapata' 22 | 23 | 24 | # Datagen 25 | def obtain_datagen(datagen, path, h5=False, npy=False): 26 | return datagen.flow_from_directory( 27 | path, 28 | target_size=(img_height, img_width), 29 | batch_size=batch_size, 30 | class_mode='binary', 31 | h5=h5) 32 | 33 | # Yield for data generators 34 | def generate_data_generator_for_two_images(genX): 35 | while True: 36 | Xi = genX.next() 37 | yield [ Xi[0], Xi[1] ], Xi[2] 38 | 39 | if __name__ == '__main__': 40 | 41 | __model__ = 'CNN_s1-face_s2-aligned' 42 | 43 | print "Starting:", time.ctime() 44 | 45 | ########################################### 46 | # Parameters 47 | 48 | epochs = 20 49 | batch_size = 20 50 | 51 | ########################################### 52 | # Data 53 | 54 | img_width, img_height, channel = 224, 224, 3 # Resolution of inputs 55 | nb_class = 6 56 | 57 | dataset = 'OULU-CASIA' 58 | partition = 'preface' # prealigned preface 59 | # Dataset for Stream 1 60 | train_data_dir = osp.join('..', '..', '_Dataset', dataset, partition, 'training') 61 | validation_data_dir = osp.join('..', '..', '_Dataset', dataset, partition, 'validation') 62 | 63 | 64 | if dataset == 'OULU-CASIA': 65 | nb_train_samples = 8384 66 | nb_validation_samples = 1982 67 | else: 68 | nb_train_samples = 29798 69 | nb_validation_samples = 4428 70 | 71 | nb_train_samples = nb_train_samples / batch_size 72 | nb_validation_samples = nb_validation_samples / batch_size 73 | 74 | ########################################### 75 | # Create Models 76 | 77 | vgg_model_stream_1 = load_model(os.path.join('weights', 'CNN_patch_epoch-06_val-accu-0.34.hdf5')) 78 | vgg_model_stream_2 = load_model(os.path.join('weights', 'CNN_aligned_epoch-19_val-accu-0.2000.hdf5')) 79 | 80 | ''' 81 | Customize the model 82 | ''' 83 | stream_1 = vgg_model_stream_1.get_layer('pool5').output 84 | stream_1 = Flatten(name='flatten-1')(stream_1) 85 | 86 | for layer in vgg_model_stream_1.layers: 87 | layer.name = layer.name + "-model_stream_1" 88 | 89 | stream_2 = vgg_model_stream_2.get_layer('pool5').output 90 | stream_2 = Flatten(name='flatten-2')(stream_2) 91 | 92 | for layer in vgg_model_stream_1.layers: 93 | layer.name = layer.name + "-model_stream_2" 94 | 95 | # fuse_layer = stream_1 + stream_2 96 | fuse_layer = concatenate([stream_1, stream_2]) 97 | 98 | fuse_layer = Dense(1024, activation='relu', name='fc6')(fuse_layer) 99 | fuse_layer = Dense(1024, activation='relu', name='fc7')(fuse_layer) 100 | fuse_layer = Dense(512, activation='relu', name='fc8')(fuse_layer) 101 | out = Dense(nb_class, activation='softmax', name='fc9')(fuse_layer) 102 | 103 | model = Model( 104 | [vgg_model_stream_1.input, vgg_model_stream_2.input], 105 | out) 106 | 107 | 108 | ########################################### 109 | # Dataset Generators 110 | 111 | datagen = ImageDataGenerator( 112 | rescale=1. / 224, 113 | shear_range=0.2, 114 | zoom_range=0.2, 115 | horizontal_flip=True) 116 | 117 | # Data generators 118 | train_generator = obtain_datagen(datagen, train_data_dir, h5=True) 119 | validation_generator = obtain_datagen(datagen, validation_data_dir, h5=True) 120 | 121 | # Yield for data generators 122 | dataset_train_gen = generate_data_generator_for_two_images(train_generator) 123 | dataset_val_gen = generate_data_generator_for_two_images(validation_generator) 124 | 125 | ########################################### 126 | # Train the model 127 | 128 | # Freeze previous layers 129 | for i, layer in enumerate(vgg_model_stream_1.layers): 130 | layer.trainable = False 131 | for i, layer in enumerate(vgg_model_stream_2.layers): 132 | layer.trainable = False 133 | 134 | # Compile the model 135 | # Compile the model 136 | lr = 0.0001 137 | optimizer = adam(lr=lr) 138 | loss = 'sparse_categorical_crossentropy' 139 | model.compile( loss=loss, 140 | optimizer=optimizer, 141 | metrics=['accuracy', 'top_k_categorical_accuracy']) 142 | ''' 143 | Callbacks 144 | ''' 145 | row_dict = OrderedDict({'model': __model__, 146 | 'dataset': dataset, 147 | 'partition': partition, 148 | 'loss': loss, 149 | 'lr': lr, 150 | 'date': time.ctime()}) 151 | 152 | # Create Version folder 153 | export_path = create_path(__model__, dataset) 154 | 155 | checkpointer = ModelCheckpoint(filepath=osp.join(export_path, __model__+'_epoch-{epoch:02d}_val-accu-{val_acc:.4f}.hdf5'), verbose=1) #, save_best_only=True) 156 | csv_logger = CSVLogger(osp.join(export_path, '_logs_'+__model__+'.log'), separator=',', append=False) 157 | tensorboard = TensorBoard(log_dir=export_path, histogram_freq=0, batch_size=batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) 158 | model_parameters = ModelParameters(osp.join(export_path, '_model_'+__model__+'.log'), row_dict) 159 | 160 | ''' 161 | Train the model 162 | ''' 163 | print('*************************************\nFine-tuning the model \n*************************************') 164 | 165 | model.fit_generator( 166 | dataset_train_gen, 167 | steps_per_epoch=nb_train_samples, 168 | epochs=epochs, 169 | validation_data=dataset_val_gen, 170 | validation_steps=nb_validation_samples, 171 | callbacks=[checkpointer, csv_logger, model_parameters]) 172 | 173 | print("Ending:", time.ctime()) 174 | -------------------------------------------------------------------------------- /Models/CNN_2Stream/vggface_model_MM.py: -------------------------------------------------------------------------------- 1 | '''VGGFace model for Keras. 2 | # Reference: 3 | - [Deep Face Recognition](http://www.robots.ox.ac.uk/~vgg/publications/2015/Parkhi15/parkhi15.pdf) 4 | ''' 5 | from __future__ import print_function 6 | import warnings 7 | 8 | from keras.applications.imagenet_utils import _obtain_input_shape 9 | from keras.engine.topology import get_source_inputs 10 | from keras.models import Model 11 | from keras.layers import Flatten, Dense, Input, GlobalAveragePooling2D, GlobalMaxPooling2D, Activation 12 | from keras.layers import Convolution2D, MaxPooling2D 13 | from keras.utils import layer_utils 14 | from keras.utils.data_utils import get_file 15 | from keras import backend as K 16 | 17 | # WEIGHTS_PATH = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_tf_v2.h5' 18 | # WEIGHTS_PATH_NO_TOP = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_tf_notop_v2.h5' 19 | 20 | def VGGFace(include_top=False, weights='vggface', 21 | input_tensor=None, input_shape=None, 22 | pooling='max', 23 | classes=2622, 24 | WEIGHTS_FILE='None'): 25 | """Instantiates the VGGFace architecture. 26 | Optionally loads weights pre-trained 27 | on VGGFace dataset. Note that when using TensorFlow, 28 | for best performance you should set 29 | `image_data_format="channels_last"` in your Keras config 30 | at ~/.keras/keras.json. 31 | The model and the weights are compatible with both 32 | TensorFlow and Theano. The data format 33 | convention used by the model is the one 34 | specified in your Keras config file. 35 | # Arguments 36 | include_top: whether to include the 3 fully-connected 37 | layers at the top of the network. 38 | weights: one of `None` (random initialization) 39 | or "imagenet" (pre-training on ImageNet). 40 | input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) 41 | to use as image input for the model. 42 | input_shape: optional shape tuple, only to be specified 43 | if `include_top` is False (otherwise the input shape 44 | has to be `(224, 224, 3)` (with `channels_last` data format) 45 | or `(3, 224, 244)` (with `channels_first` data format). 46 | It should have exactly 3 inputs channels, 47 | and width and height should be no smaller than 48. 48 | E.g. `(200, 200, 3)` would be one valid value. 49 | pooling: Optional pooling mode for feature extraction 50 | when `include_top` is `False`. 51 | - `None` means that the output of the model will be 52 | the 4D tensor output of the 53 | last convolutional layer. 54 | - `avg` means that global average pooling 55 | will be applied to the output of the 56 | last convolutional layer, and thus 57 | the output of the model will be a 2D tensor. 58 | - `max` means that global max pooling will 59 | be applied. 60 | classes: optional number of classes to classify images 61 | into, only to be specified if `include_top` is True, and 62 | if no `weights` argument is specified. 63 | # Returns 64 | A Keras model instance. 65 | # Raises 66 | ValueError: in case of invalid argument for `weights`, 67 | or invalid input shape. 68 | """ 69 | 70 | if weights not in {'vggface', None}: 71 | raise ValueError('The `weights` argument should be either ' 72 | '`None` (random initialization) or `vggface` ' 73 | '(pre-training on VGGFace Dataset).') 74 | 75 | if weights == 'vggface' and include_top and classes != 2622: 76 | raise ValueError('If using `weights` as vggface original with `include_top`' 77 | ' as true, `classes` should be 2622') 78 | # Determine proper input shape 79 | input_shape = _obtain_input_shape(input_shape, 80 | default_size=224, 81 | min_size=48, 82 | data_format=K.image_data_format(), 83 | include_top=include_top) 84 | 85 | if input_tensor is None: 86 | img_input = Input(shape=input_shape) 87 | else: 88 | if not K.is_keras_tensor(input_tensor): 89 | img_input = Input(tensor=input_tensor, shape=input_shape) 90 | else: 91 | img_input = input_tensor 92 | 93 | # Block 1 94 | x = Convolution2D(64, (3, 3), activation='relu', padding='same', name='conv1_1')(img_input) 95 | x = Convolution2D(64, (3, 3), activation='relu', padding='same', name='conv1_2')(x) 96 | x = MaxPooling2D((2, 2), strides=(2, 2), name='pool1')(x) 97 | 98 | # Block 2 99 | x = Convolution2D(128, (3, 3), activation='relu', padding='same', name='conv2_1')(x) 100 | x = Convolution2D(128, (3, 3), activation='relu', padding='same', name='conv2_2')(x) 101 | x = MaxPooling2D((2, 2), strides=(2, 2), name='pool2')(x) 102 | 103 | # Block 3 104 | x = Convolution2D(256, (3, 3), activation='relu', padding='same', name='conv3_1')(x) 105 | x = Convolution2D(256, (3, 3), activation='relu', padding='same', name='conv3_2')(x) 106 | x = Convolution2D(256, (3, 3), activation='relu', padding='same', name='conv3_3')(x) 107 | x = MaxPooling2D((2, 2), strides=(2, 2), name='pool3')(x) 108 | 109 | # Block 4 110 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv4_1')(x) 111 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv4_2')(x) 112 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv4_3')(x) 113 | x = MaxPooling2D((2, 2), strides=(2, 2), name='pool4')(x) 114 | 115 | # Block 5 116 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv5_1')(x) 117 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv5_2')(x) 118 | x = Convolution2D(512, (3, 3), activation='relu', padding='same', name='conv5_3')(x) 119 | x = MaxPooling2D((2, 2), strides=(2, 2), name='pool5')(x) 120 | 121 | if include_top: 122 | # Classification block 123 | x = Flatten(name='flatten')(x) 124 | x = Dense(4096, name='fc6')(x) 125 | x = Activation('relu', name='fc6/relu')(x) 126 | x = Dense(4096, name='fc7')(x) 127 | x = Activation('relu', name='fc7/relu')(x) 128 | x = Dense(2622, name='fc8')(x) 129 | x = Activation('relu', name='fc8/softmax')(x) 130 | else: 131 | if pooling == 'avg': 132 | x = GlobalAveragePooling2D()(x) 133 | elif pooling == 'max': 134 | x = GlobalMaxPooling2D()(x) 135 | 136 | # Ensure that the model takes into account 137 | # any potential predecessors of `input_tensor`. 138 | if input_tensor is not None: 139 | inputs = get_source_inputs(input_tensor) 140 | else: 141 | inputs = img_input 142 | # Create model. 143 | model = Model(inputs, x, name='VGGFace') # load weights 144 | if weights == 'vggface': 145 | weights_path = 'weights/' + WEIGHTS_FILE 146 | model.load_weights(weights_path, by_name=True) 147 | if K.backend() == 'theano': 148 | layer_utils.convert_all_kernels_in_model(model) 149 | 150 | if K.image_data_format() == 'channels_first': 151 | if include_top: 152 | maxpool = model.get_layer(name='pool5') 153 | shape = maxpool.output_shape[1:] 154 | dense = model.get_layer(name='fc6') 155 | layer_utils.convert_dense_weights_data_format(dense, shape, 'channels_first') 156 | 157 | if K.backend() == 'tensorflow': 158 | warnings.warn('You are using the TensorFlow backend, yet you ' 159 | 'are using the Theano ' 160 | 'image data format convention ' 161 | '(`image_data_format="channels_first"`). ' 162 | 'For best performance, set ' 163 | '`image_data_format="channels_last"` in ' 164 | 'your Keras config ' 165 | 'at ~/.keras/keras.json.') 166 | return model -------------------------------------------------------------------------------- /Models/CNN_3D/3dcnn.py: -------------------------------------------------------------------------------- 1 | # import theano 2 | # theano.config.device = 'gpu:5' 3 | # theano.config.floatX = 'float32' 4 | 5 | # export CUDA_VISIBLE_DEVICES=1,2 6 | import os 7 | import sys 8 | import time 9 | import os.path as osp 10 | 11 | # Add the parent module to the import path 12 | sys.path.append(osp.realpath(osp.join(osp.dirname(__file__), '../'))) 13 | 14 | import warnings 15 | warnings.simplefilter("ignore", DeprecationWarning) 16 | 17 | from cnn3d_model import get_model 18 | from sklearn.metrics import confusion_matrix 19 | 20 | from keras.optimizers import adam 21 | from keras.callbacks import ModelCheckpoint, CSVLogger, Callback 22 | 23 | from collections import OrderedDict 24 | 25 | from _saving.SaveModelDirectory import create_path 26 | from _saving.ModelParameters import ModelParameters 27 | 28 | from preprocessing.image import ImageDataGenerator 29 | 30 | 31 | __author__ = 'Daniel Garcia Zapata' 32 | 33 | # Datagen 34 | def obtain_datagen(datagen, train_path): 35 | return datagen.flow_from_directory( 36 | train_path, 37 | target_size=(img_height, img_width), 38 | batch_size=batch_size, 39 | class_mode='binary', 40 | npy=True) 41 | 42 | # Yield for data generators 43 | def generate_data_generator_for_two_images(genX1): 44 | while True: 45 | X1i = genX1.next() 46 | yield X1i[0], X1i[1] 47 | 48 | if __name__ == '__main__': 49 | 50 | __model__ = '3DCNN' 51 | 52 | print('Starting:', time.ctime(), '\n') 53 | 54 | ########################################### 55 | # Parameters 56 | 57 | epochs = 20 58 | batch_size = 20 59 | 60 | ########################################### 61 | # Data 62 | 63 | img_width, img_height, channels = 112, 112, 3 # Resolution of inputs 64 | input_shape = 4096 65 | 66 | dataset = 'SASE-FE' # OULU-CASIA, SASE-FE 67 | if dataset == 'OULU-CASIA': 68 | partition = 'prealigned' 69 | train_data_dir = osp.join('..', '..', '_Dataset', dataset, 'consecutive', 'training') 70 | validation_data_dir = osp.join('..', '..', '_Dataset', dataset, 'consecutive', 'validation') 71 | 72 | frames = 5 73 | n_output = 6 74 | 75 | nb_train_samples = 6019 / batch_size 76 | nb_validation_samples = 1947 / batch_size 77 | 78 | else: 79 | partition = 'frontalization' 80 | train_data_dir = osp.join('..', '..', '_Dataset', dataset, '5frames', 'training') 81 | validation_data_dir = osp.join('..', '..', '_Dataset', dataset, '5frames', 'validation') 82 | 83 | frames = 5 84 | n_output = 12 85 | 86 | nb_train_samples = 29799 / batch_size 87 | nb_validation_samples = 4428 / batch_size 88 | 89 | 90 | # Custom parameters 91 | hidden_dim = 512 92 | 93 | 94 | ''' 95 | Load the models 96 | ''' 97 | model = get_model( 98 | frames=frames, 99 | summary=False) 100 | 101 | ''' 102 | Dataset Generators 103 | ''' 104 | datagen = ImageDataGenerator( 105 | rescale=1. / 224, 106 | shear_range=0.2, 107 | zoom_range=0.2, 108 | horizontal_flip=True) 109 | 110 | # Data Generators 111 | train_generator = obtain_datagen(datagen, train_data_dir) 112 | validation_generator = obtain_datagen(datagen, validation_data_dir) 113 | 114 | # Yield for data generators 115 | dataset_train_gen = generate_data_generator_for_two_images(train_generator) 116 | dataset_val_gen = generate_data_generator_for_two_images(validation_generator) 117 | 118 | 119 | ''' 120 | Fine-tune the model 121 | ''' 122 | # Freeze previous layers 123 | # for i, layer in enumerate(model.layers): 124 | # layer.trainable = False 125 | 126 | # Compile the model 127 | lr = 0.0001 128 | optimizer = adam(lr=lr) 129 | loss = 'sparse_categorical_crossentropy' 130 | model.compile( loss=loss, 131 | optimizer=optimizer, 132 | metrics=['accuracy', 'top_k_categorical_accuracy']) 133 | 134 | ''' 135 | Callbacks 136 | ''' 137 | row_dict = OrderedDict({'model': __model__, 138 | 'dataset': dataset, 139 | 'partition': partition, 140 | 'loss': loss, 141 | 'lr': lr, 142 | 'date': time.ctime()}) 143 | 144 | # Create Version folder 145 | export_path = create_path(__model__, dataset) 146 | 147 | checkpointer = ModelCheckpoint(filepath=osp.join(export_path, __model__+'_epoch-{epoch:02d}_val-accu-{val_acc:.4f}.hdf5'), verbose=1) #, save_best_only=True) 148 | csv_logger = CSVLogger(osp.join(export_path, '_logs_'+__model__+'.log'), separator=',', append=False) 149 | model_parameters = ModelParameters(osp.join(export_path, '_model_'+__model__+'.log'), row_dict) 150 | 151 | ''' 152 | Train the model 153 | ''' 154 | print('*************************************\nTraining \n*************************************') 155 | model.fit_generator(dataset_train_gen, 156 | steps_per_epoch=nb_train_samples, 157 | epochs=epochs, 158 | validation_data=dataset_val_gen, 159 | validation_steps=nb_validation_samples, 160 | callbacks=[checkpointer, csv_logger, model_parameters]) 161 | 162 | print('\nEnding:"', time.ctime()) 163 | -------------------------------------------------------------------------------- /Models/CNN_3D/cnn3d_model.py: -------------------------------------------------------------------------------- 1 | import os 2 | from keras.models import Sequential 3 | from keras.utils.data_utils import get_file 4 | from keras.layers.core import Dense, Dropout, Flatten 5 | from keras.layers.convolutional import Convolution3D, MaxPooling3D, ZeroPadding3D 6 | 7 | def get_model(summary=False, frames=16): 8 | WEIGHTS_PATH = 'sports1M_weights.h5' 9 | weights_path = os.path.join('weights', WEIGHTS_PATH) 10 | 11 | model = Sequential() 12 | # 1st layer group 13 | model.add(Convolution3D(64, (3, 3, 3), activation='relu', 14 | padding='same', name='conv1', 15 | strides=(1, 1, 1), 16 | input_shape=(frames, 112, 112, 3))) 17 | model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2), 18 | padding='valid', name='pool1')) 19 | # 2nd layer group 20 | model.add(Convolution3D(128, (3, 3, 3), activation='relu', 21 | padding='same', name='conv2', 22 | strides=(1, 1, 1))) 23 | model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), 24 | padding='valid', name='pool2')) 25 | # 3rd layer group 26 | model.add(Convolution3D(256, (3, 3, 3), activation='relu', 27 | padding='same', name='conv3a', 28 | strides=(1, 1, 1))) 29 | model.add(Convolution3D(256, (3, 3, 3), activation='relu', 30 | padding='same', name='conv3b', 31 | strides=(1, 1, 1))) 32 | model.add(MaxPooling3D(pool_size=(1, 1, 1), strides=(2, 2, 2), 33 | padding='valid', name='pool3')) 34 | # 4th layer group 35 | model.add(Convolution3D(512, (3, 3, 3), activation='relu', 36 | padding='same', name='conv4a', 37 | strides=(1, 1, 1))) 38 | model.add(Convolution3D(512, (3, 3, 3), activation='relu', 39 | padding='same', name='conv4b', 40 | strides=(1, 1, 1))) 41 | model.add(MaxPooling3D(pool_size=(1, 1, 1), strides=(2, 2, 2), 42 | padding='valid', name='pool4')) 43 | # 5th layer group 44 | model.add(Convolution3D(512, (3, 3, 3), activation='relu', 45 | padding='same', name='conv5a', 46 | strides=(1, 1, 1))) 47 | model.add(Convolution3D(512, (3, 3, 3), activation='relu', 48 | padding='same', name='conv5b', 49 | strides=(1, 1, 1))) 50 | model.add(ZeroPadding3D(padding=(0, 1, 1))) 51 | model.add(MaxPooling3D(pool_size=(1, 1, 1), strides=(2, 2, 2), 52 | padding='valid', name='pool5')) 53 | model.add(Flatten()) 54 | # FC layers group 55 | model.add(Dense(4096, activation='relu', name='fc6')) 56 | model.add(Dropout(.5)) 57 | model.add(Dense(4096, activation='relu', name='fc7')) 58 | model.add(Dropout(.5)) 59 | model.add(Dense(12, activation='softmax', name='fc8')) 60 | if summary: 61 | print(model.summary()) 62 | return model 63 | 64 | model = get_model(summary=True) -------------------------------------------------------------------------------- /Models/CNN_3D/preprocessing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dangz90/Deep-Learning-for-Expression-Recognition-in-Image-Sequences/66c50292cac9a66eace32a040c49267b06c45de5/Models/CNN_3D/preprocessing/__init__.py -------------------------------------------------------------------------------- /Models/_saving/ModelParameters.py: -------------------------------------------------------------------------------- 1 | import os 2 | import six, csv 3 | from keras.callbacks import Callback 4 | 5 | # Class for writing the model parameters 6 | class ModelParameters(Callback): 7 | def __init__(self, filename, row_dict, separator=',', append=False): 8 | self.sep = separator 9 | self.row_dict = row_dict 10 | self.filename = filename 11 | self.append = append 12 | self.writer = None 13 | self.keys = row_dict.keys() 14 | self.append_header = True 15 | self.file_flags = 'b' if six.PY2 and os.name == 'nt' else '' 16 | super(ModelParameters, self).__init__() 17 | 18 | def on_train_begin(self, logs=None): 19 | if self.append: 20 | if os.path.exists(self.filename): 21 | with open(self.filename, 'r' + self.file_flags) as f: 22 | self.append_header = not bool(len(f.readline())) 23 | self.csv_file = open(self.filename, 'a' + self.file_flags) 24 | else: 25 | self.csv_file = open(self.filename, 'w' + self.file_flags) 26 | 27 | if self.keys is None: 28 | self.keys = ['dataset', 'partition', 'loss', 'lr', 'date'] 29 | 30 | if not self.writer: 31 | class CustomDialect(csv.excel): 32 | delimiter = self.sep 33 | 34 | self.writer = csv.DictWriter(self.csv_file, 35 | fieldnames= self.keys, dialect=CustomDialect) 36 | if self.append_header: 37 | self.writer.writeheader() 38 | 39 | self.writer.writerow(self.row_dict) 40 | self.csv_file.flush() 41 | 42 | self.csv_file.close() 43 | self.writer = None -------------------------------------------------------------------------------- /Models/_saving/SaveModelDirectory.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import tensorflow as tf 4 | 5 | def create_path(__model__, dataset): 6 | 7 | ''' 8 | Directory base folder 9 | ''' 10 | export_path_base = os.path.dirname(sys.argv[0]) 11 | 12 | ''' 13 | Define model version 14 | ''' 15 | # Obtain most recent version 16 | export_base_path = os.path.join(export_path_base, '_versions', dataset, __model__) 17 | 18 | # Verify a folder already exists and assign version number 19 | if os.path.isdir(export_base_path): 20 | raw_all_subdirs = [d for d in os.listdir(export_base_path) if os.path.isdir(export_base_path)] 21 | all_subdirs = [subdir for subdir in raw_all_subdirs if not subdir.startswith('.')] 22 | 23 | __version__ = int(max(all_subdirs)) + 1 24 | else: 25 | __version__ = 1 26 | 27 | # Assign FLAG version number 28 | tf.app.flags.DEFINE_integer('model_version', __version__, 'Version number of the model.') 29 | FLAGS = tf.app.flags.FLAGS 30 | 31 | ''' 32 | Create version folder 33 | ''' 34 | export_path = os.path.join( export_base_path, 35 | str(FLAGS.model_version)) 36 | builder = tf.saved_model.builder.SavedModelBuilder(export_path) 37 | 38 | return export_path 39 | 40 | if __name__ == '__main__': 41 | create_path() -------------------------------------------------------------------------------- /Models/_saving/__init__.py: -------------------------------------------------------------------------------- 1 | from .SaveModelDirectory import create_path 2 | from .ModelParameters import ModelParameters -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deep Learning for Expression Recognition in Image Sequences 2 | 3 | Facial expressions convey lots of information, which can be used for identifying emotions. These facial expressions vary in time when they are being performed. Recognition of certain emotions is a very challenging task even for people. This thesis consists of using machine learning algorithms for recognizing emotions in image sequences. It uses the state-of-the-art deep learning on collected data for automatic analysis of emotions. Concretely, the thesis presents a comparison of current state-of-the-art learning strategies that can handle spatio-temporal data and adapt classical static approaches to deal with images sequences. Expanded versions of CNN, 3DCNN, and Recurrent approaches are evaluated and compared in two public datasets for universal emotion recognition, where the performances are shown, and pros and cons are discussed. 4 | 5 | 16 | 17 | ## Built With 18 | 19 | * [Keras](https://keras.io/) - Keras is a high-level neural networks API, written in Python 20 | * [Tensorflow](https://www.tensorflow.org/) - TensorFlow™ is an open source software library for numerical computation using data flow graphs. 21 | * [Theano](http://deeplearning.net/software/theano/) - Theano is a Python library that allows you to define, optimize, and evaluate mathematical expressions involving multi-dimensional arrays efficiently. 22 | 23 | ## Datasets 24 | 25 | * [SASE-FE](http://icv.tuit.ut.ee/sase-fe-database.html) - SASE-FE real and fake emotion database contains videos with six typical emotions depicting a real and a fake emotion. 26 | * [Oulu-Casia](http://www.cse.oulu.fi/CMV/Downloads/Oulu-CASIA) - Oulu-CASIA NIR&VIS facial expression database contains videos with the six typical expressions. 27 | 28 | ## Author 29 | 30 | * **Daniel Garcia Zapata** [[dangz90]](https://github.com/dangz90) 31 | 32 | ## License 33 | 34 | This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details 35 | 36 | ## Acknowledgments 37 | 38 | * The face frontalization [2] preprocess was performed using Douglas Souza [[dougsouza]](https://github.com/dougsouza/face-frontalization) implementation. 39 | * The 3D CNN model is based on Alberto Montes [[albertomontesg]](https://gist.github.com/albertomontesg/d8b21a179c1e6cca0480ebdf292c34d2) implementation of C3D model. 40 | * The CNN model is based on Refik Can Malli [[rcmalli]](https://github.com/rcmalli/keras-vggface) implementation of the VGG-Face. 41 | * The [VGG-Face](http://www.robots.ox.ac.uk/~vgg/software/vgg_face/) was first introduced by Omkar M. Parkhi, Andrea Vedaldi, Andrew Zisserman from University of Oxford. 42 | * The [C3D Model](https://www.cv-foundation.org/openaccess/content_iccv_2015/html/Tran_Learning_Spatiotemporal_Features_ICCV_2015_paper.html) first introduced by Du Tran, Lubomir Bourdev, Rob Fergus, Lorenzo Torresani, Manohar Paluri 43 | from Facebook AI Research and Dartmouth College. 44 | 45 | ## Bibliography 46 | * [1] Ofodile, I., Kulkarni, K., Corneanu, C. A., Escalera, S., Baro, X., Hyniewska, S., ... & Anbarjafari, G. (2017). Automatic recognition of deceptive facial expressions of emotion. arXiv preprint arXiv:1707.04061. 47 | * [2] Hassner, T., Harel, S., Paz, E., & Enbar, R. (2015). Effective face frontalization in unconstrained images. In Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (pp. 4295-4304). 48 | -------------------------------------------------------------------------------- /Thesis.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dangz90/Deep-Learning-for-Expression-Recognition-in-Image-Sequences/66c50292cac9a66eace32a040c49267b06c45de5/Thesis.pdf -------------------------------------------------------------------------------- /_Preprocess/block_video_to_data.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import time 4 | import numpy as np 5 | 6 | ''' 7 | Function to obtain frames from a video 8 | ''' 9 | def video_to_frames(dir): 10 | 11 | vidPath = '/path/foo/video.mp4' 12 | shotsPath = '/%d.avi' # output path (must be avi, otherwize choose other codecs) 13 | segRange = [(0,40),(50,100),(200,400)] # a list of starting/ending frame indices pairs 14 | 15 | cap = cv2.VideoCapture(dir) 16 | fps = int(cap.get(cv2.CAP_PROP_FPS)) 17 | size = (int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)),int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))) 18 | fourcc = int(cv2.VideoWriter_fourcc('X','V','I','D')) # XVID codecs 19 | 20 | for idx,(begFidx,endFidx) in enumerate(segRange): 21 | writer = cv2.VideoWriter(shotsPath%idx,fourcc,fps,size) 22 | cap.set(cv2.CAP_PROP_POS_FRAMES,begFidx) 23 | ret = True # has frame returned 24 | while(cap.isOpened() and ret and writer.isOpened()): 25 | ret, frame = cap.read() 26 | frame_number = cap.get(cv2.CAP_PROP_POS_FRAMES) - 1 27 | if frame_number < endFidx: 28 | writer.write(frame) 29 | else: 30 | break 31 | writer.release() 32 | 33 | 34 | ''' 35 | Function to obtain a list of files from a directory 36 | ''' 37 | def split_dataset(data_path, path_to_save): 38 | if (not os.path.isdir(data_path)): 39 | assert False 40 | if (not os.path.isdir(path_to_save)): 41 | os.mkdir(path_to_save) 42 | 43 | ''' 44 | Obtain Subdirs 45 | ''' 46 | subdirs = [x[0] for x in os.walk(data_path)][1:] 47 | 48 | for subdir in sorted(subdirs): 49 | files = os.walk(subdir).__next__()[2] 50 | 51 | ''' 52 | Obtain .MP4 file 53 | ''' 54 | for f in sorted(files): 55 | if f.endswith('.MP4'): 56 | 57 | print('Reading', os.path.join(subdir, f)) 58 | 59 | video_to_frames(os.path.join(subdir, f)) 60 | 61 | ''' 62 | Main: EGG 63 | ''' 64 | if __name__ == "__main__": 65 | print('Starting:', time.ctime(), '\n') 66 | 67 | import argparse 68 | 69 | parser = argparse.ArgumentParser(description='Change folder order for OULU-CASIA dataset') 70 | parser.add_argument('-dataset', type=str, default='Blocked_raw', 71 | help='Name of the dataset to use') 72 | parser.add_argument('-participant', type=str, default='', 73 | help='Name of the participant') 74 | args = parser.parse_args() 75 | 76 | path_to_dataset = os.path.join('..', '_Dataset', args.dataset, args.participant) 77 | path_to_save = os.path.join('..', '_Dataset', 'Blocked') 78 | 79 | """ 80 | Read video 81 | """ 82 | split_dataset(path_to_dataset, path_to_save) 83 | 84 | # for file in files: 85 | # save_as_numpy( path_to_dataset, file ) 86 | # split_dataset(path_to_dataset, path_to_save) 87 | 88 | print('\nEnding:', time.ctime()) 89 | -------------------------------------------------------------------------------- /_Preprocess/count-images.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import numpy as np 4 | 5 | from collections import defaultdict 6 | 7 | ''' 8 | Function to obtain frames from a video 9 | ''' 10 | def move_videos(classes, list_of_videos, dataset): 11 | 12 | new_base_path = os.path.join('..', '_Dataset', dataset, 'patch') 13 | 14 | for old_video in list_of_videos: 15 | participant_number = os.path.dirname(old_video).split('/')[6] 16 | 17 | # Validation 18 | if int(filter(str.isdigit, os.path.dirname(old_video).split('/')[6])) > 60: 19 | new_path = os.path.join(new_base_path, 'validation', classes) 20 | new_file_name = participant_number +'_'+ os.path.basename(old_video) 21 | 22 | # if not os.path.exists(new_path): 23 | # os.makedirs(new_path) 24 | 25 | full_path = os.path.join(new_path, new_file_name) 26 | # os.rename(old_video, full_path) 27 | 28 | # Training 29 | else: 30 | new_path = os.path.join(new_base_path, 'training', classes) 31 | new_file_name = participant_number +'_'+ os.path.basename(old_video) 32 | 33 | # if not os.path.exists(new_path): 34 | # os.makedirs(new_path) 35 | 36 | full_path = os.path.join(new_path, new_file_name) 37 | # os.rename(old_video, full_path) 38 | 39 | print old_video 40 | 41 | # frames_per_participants = defaultdict(int) 42 | # for k, video in enumerate(list_of_videos): 43 | # current_participant_number = os.path.basename(video).split('_')[0] 44 | 45 | # # Frames to use: frame_counter + 16 Size of the frames of the CNN 46 | # current_frame = int(video.replace('.jpg','').replace('.mp4frame', '=').split('=')[1]) 47 | # frame_limit = participant_frames[current_participant_number] 48 | 49 | # if frame_counter > number_of_frames and original_participant_number == current_participant_number: 50 | # ''' 51 | # Save 52 | # ''' 53 | # path_to_save = os.path.join( os.path.dirname(video).replace('2_Model','2_3DModel').replace(dataset, 'dataset'+number_of_frames+'_consecutive_3d') , os.path.basename(video).split('.')[0] + '_' + str(participant_frames_number) ) 54 | # print(path_to_save) 55 | # # np.save( path_to_save, vid ) 56 | 57 | # # Delete first video in order to keep always size 16 before appending the next video 58 | # # del vid[0] 59 | 60 | # if original_participant_number != current_participant_number or k == len(list_of_videos)-1: 61 | # ''' 62 | # Restore counter 63 | # ''' 64 | # vid = [] 65 | # frame_counter = 1 66 | # participant_frames_number = 1 67 | # original_participant_number = current_participant_number 68 | 69 | # vid.append(os.path.basename(video)) 70 | # frame_counter += 1 71 | 72 | ''' 73 | Function to obtain a list of files from a directory 74 | ''' 75 | def list_files(dir): 76 | from collections import defaultdict 77 | 78 | videos_by_class = defaultdict(list) 79 | subdirs = [x[0] for x in os.walk(dir)] 80 | 81 | for subdir in sorted(subdirs): 82 | files = os.walk(subdir).next()[2] 83 | for file in sorted(files): 84 | if file.endswith('.jpeg'): 85 | file_name = os.path.join(subdir, file) 86 | videos_by_class[os.path.basename(subdir)].append( file_name ) 87 | 88 | return videos_by_class 89 | 90 | 91 | if __name__ == '__main__': 92 | print 'Starting:', time.ctime(), '\n' 93 | 94 | import argparse 95 | 96 | parser = argparse.ArgumentParser(description='Change folder order for OULU-CASIA dataset') 97 | parser.add_argument('-dataset', type=str, default='OULU-CASIA', 98 | help='Name of the dataset to use') 99 | parser.add_argument('-partition', type=str, default='OriginalImg', 100 | help='Name of the data partition to use') 101 | parser.add_argument('-process', type=str, default='VL', 102 | help='Set to use (NI_Acropped, VL_Acropped)') 103 | args = parser.parse_args() 104 | 105 | path_to_dataset = os.path.join('..', '_Dataset', args.dataset, args.partition, args.process) 106 | 107 | # OULU-CASIA has an extra folder depth 108 | if args.dataset == 'OULU-CASIA': 109 | subfolder = 'Strong' 110 | path_to_dataset = os.path.join(path_to_dataset, subfolder) 111 | 112 | videos_by_class = list_files(path_to_dataset) 113 | 114 | for classes, videos in videos_by_class.items(): 115 | print videos 116 | # move_videos(classes, videos_by_class[classes], args.dataset) 117 | 118 | print '\nEnding:', time.ctime() -------------------------------------------------------------------------------- /_Preprocess/crop-to-aligned-and-geometry.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import pickle 4 | import numpy as np 5 | 6 | from PIL import Image 7 | from collections import defaultdict 8 | 9 | __author__ = 'Daniel Garcia Zapata' 10 | 11 | 12 | classes = { 0:'Anger', 13 | 1:'Disgust', 14 | 2:'Fear', 15 | 3:'Happiness', 16 | 4:'Sadness', 17 | 5:'Surprise'} 18 | 19 | ''' 20 | Function to obtain frames from a video 21 | ''' 22 | def move_videos(classes, list_of_files, dataset): 23 | 24 | for old_video in list_of_files: 25 | participant_number = os.path.dirname(old_video).split(os.sep)[4] 26 | 27 | # Validation 28 | if int(filter(str.isdigit, os.path.dirname(old_video).split(os.sep)[4])) > 60: 29 | new_path = os.path.join(new_base_path, 'validation', classes) 30 | new_file_name = participant_number +'_'+ os.path.basename(old_video) 31 | 32 | if not os.path.exists(new_path): 33 | os.makedirs(new_path) 34 | 35 | new_full_path = os.path.join(new_path, new_file_name) 36 | # os.rename(old_video, new_full_path) 37 | 38 | # Training 39 | else: 40 | new_path = os.path.join(new_base_path, 'training', classes) 41 | new_file_name = participant_number +'_'+ os.path.basename(old_video) 42 | 43 | if not os.path.exists(new_path): 44 | os.makedirs(new_path) 45 | 46 | new_full_path = os.path.join(new_path, new_file_name) 47 | # os.rename(old_video, new_full_path) 48 | 49 | # frames_per_participants = defaultdict(int) 50 | # for k, video in enumerate(list_of_files): 51 | # current_participant_number = os.path.basename(video).split('_')[0] 52 | 53 | # # Frames to use: frame_counter + 16 Size of the frames of the CNN 54 | # current_frame = int(video.replace('.jpg','').replace('.mp4frame', '=').split('=')[1]) 55 | # frame_limit = participant_frames[current_participant_number] 56 | 57 | # if frame_counter > number_of_frames and original_participant_number == current_participant_number: 58 | # ''' 59 | # Save 60 | # ''' 61 | # path_to_save = os.path.join( os.path.dirname(video).replace('2_Model','2_3DModel').replace(dataset, 'dataset'+number_of_frames+'_consecutive_3d') , os.path.basename(video).split('.')[0] + '_' + str(participant_frames_number) ) 62 | # print(path_to_save) 63 | # # np.save( path_to_save, vid ) 64 | 65 | # # Delete first video in order to keep always size 16 before appending the next video 66 | # # del vid[0] 67 | 68 | # if original_participant_number != current_participant_number or k == len(list_of_videos)-1: 69 | # ''' 70 | # Restore counter 71 | # ''' 72 | # vid = [] 73 | # frame_counter = 1 74 | # participant_frames_number = 1 75 | # original_participant_number = current_participant_number 76 | 77 | # vid.append(os.path.basename(video)) 78 | # frame_counter += 1 79 | 80 | def create_path(directory, subdirectory, classes, new_file_name, extension): 81 | new_path = os.path.join(directory, subdirectory, classes) 82 | full_path = os.path.join(new_path, new_file_name + extension) 83 | 84 | if not os.path.exists(new_path): 85 | os.makedirs(new_path) 86 | 87 | return full_path 88 | 89 | 90 | def read_pickle(subdir, file_path): 91 | with open(file_path, 'rb') as f: 92 | data = pickle.load(f) 93 | 94 | participant_number = os.path.splitext(os.path.basename(file_path))[0] 95 | 96 | for e, (emotion, geometry, face) in enumerate(zip(data['emos'], data['geoms'], data['faces'])): 97 | # Create file name 98 | new_file_name = participant_number +'_'+ str(e) 99 | 100 | # Geometry 101 | full_path_geometry = create_path(subdir, '_geometry', classes[emotion], new_file_name, '.npy') 102 | # np.save(full_path_geometry, geometry) 103 | 104 | # Face 105 | full_path_face = create_path(subdir, '_face', classes[emotion], new_file_name, '.jpg') 106 | 107 | print full_path_face 108 | 109 | 110 | # img = Image.fromarray(face, 'RGB') 111 | # result.save(img) 112 | 113 | ''' 114 | Function to obtain a list of files from a directory 115 | ''' 116 | def list_files(dir, extension): 117 | from collections import defaultdict 118 | 119 | # Explore sub directories 120 | subdirs = [x[0] for x in os.walk(dir) if len(x[0].split(os.sep)) > len(dir.split(os.sep))] 121 | 122 | list_of_files = defaultdict(list) 123 | # Cycle through sub directories 124 | for subdir in sorted(subdirs): 125 | # Obtain files from sub directories 126 | files = os.walk(subdir).next()[2] 127 | # Cycle through files 128 | for e, file in enumerate(sorted(files)): 129 | 130 | if file.endswith(extension): 131 | file_path = os.path.join(subdir, file) 132 | 133 | print file_path 134 | 135 | # # Obtain file class and put geometry and image in respective folder 136 | # read_pickle(os.path.dirname(subdir), file_path) 137 | 138 | # # list_of_files[os.path.basename(subdir)].append( file_path ) 139 | 140 | return list_of_files 141 | 142 | 143 | if __name__ == '__main__': 144 | print 'Starting:', time.ctime(), '\n' 145 | 146 | import argparse 147 | 148 | parser = argparse.ArgumentParser(description='Change folder order for OULU-CASIA dataset') 149 | parser.add_argument('-dataset', type=str, default='OULU-CASIA', 150 | help='Name of the dataset to use') 151 | parser.add_argument('-partition', type=str, default='croppedfaces', 152 | help='Name of the data partition to use') 153 | parser.add_argument('-process', type=str, default='', 154 | help='Set to use (NI_Acropped, VL_Acropped)') 155 | parser.add_argument('-depth', type=str, default='', 156 | help='Add if there is an extra depth') 157 | parser.add_argument('-extension', type=str, default='.jpeg', 158 | help='Add extension to extract') 159 | args = parser.parse_args() 160 | 161 | path_to_dataset = os.path.join('..', '_Dataset', args.dataset, args.partition, args.process, args.depth) 162 | print 'Using folder:', path_to_dataset 163 | 164 | list_of_files = list_files(path_to_dataset, args.extension) 165 | # for classes, videos in list_of_files.items(): 166 | # move_videos(classes, list_of_files[classes], args.dataset) 167 | 168 | print '\nEnding:', time.ctime() -------------------------------------------------------------------------------- /_Preprocess/egg_preprocessing.py: -------------------------------------------------------------------------------- 1 | import mne 2 | import csv 3 | import glob, os 4 | import time, datetime 5 | 6 | import numpy as np 7 | import pandas as pd 8 | import itertools as IT 9 | 10 | import matplotlib 11 | import plotly.plotly as py 12 | import matplotlib.pyplot as plt 13 | 14 | mne.set_log_level('CRITICAL') 15 | 16 | ############################################################################################### 17 | # 18 | #Reading eeg files. Each files contain informatiom for one trial (watching 60 images 19 | # wth 2 followong questions). Split file to events correspoding to watching specific 20 | #type of imae (see triggers). Assumme delay for video display for 0.5 sec 21 | # 22 | #OUT: "_____.bdf". if first symbol is 2, then the event had dublicate in the trial (can't identify an image) 23 | # 24 | ############################################################################################### 25 | 26 | triggers = {7:'disgust', 11:'surprise', 15:'sadness', 19:'anger', 23:'fear', 27:'neutral', 31:'happiness', 27 | 103:'disgust', 107:'surprise', 111:'sadness', 115:'anger', 119:'fear', 123:'neutral', 127:'happiness', 28 | 39:'disgust', 43:'surprise', 47:'sadness', 51:'anger', 55:'fear', 59:'neutral', 63:'happiness', 29 | 71:'disgust', 75:'surprise', 79:'sadness', 83:'anger', 87:'fear', 91:'neutral', 95:'happiness'} 30 | 31 | def valid(file, experiment): 32 | print(file) 33 | flag = 0 34 | with open(file, 'rb') as csvfile: 35 | reader = csv.reader(csvfile, delimiter=' ') 36 | orders = list(reader) 37 | 38 | d = [row[2::2] for row in orders if experiment in row] 39 | experiments = [row[0::40] for row in orders] 40 | 41 | if flag == 0: 42 | header = orders[0][1::2] 43 | flag = 1 44 | 45 | df = pd.DataFrame(d, columns=header) 46 | csvfile.close() 47 | return df, experiments 48 | 49 | def obtain_variables_from_video(subdir, files, experiment): 50 | for file in files: 51 | if 'sound' not in file and 'audio' not in file and file.endswith('.txt'): 52 | df, experiments = valid(os.path.join(subdir, file), experiment) 53 | 54 | return df, experiments 55 | 56 | def split_dataset(data_path, path_to_save): 57 | if (not os.path.isdir(data_path)): 58 | assert False 59 | if (not os.path.isdir(path_to_save)): 60 | os.mkdir(path_to_save) 61 | 62 | for v in triggers.values(): 63 | if not os.path.exists(os.path.join(path_to_save, v)): 64 | os.mkdir(os.path.join(path_to_save, v)) 65 | 66 | ''' 67 | Obtain Subdirs 68 | ''' 69 | subdirs = [x[0] for x in os.walk(data_path)][1:] 70 | 71 | for subdir in sorted(subdirs): 72 | files = os.walk(subdir).next()[2] 73 | 74 | experiment = 'EckmanFaces' 75 | df, experiments = obtain_variables_from_video(subdir, files, experiment) 76 | 77 | ''' 78 | Obtain .bdf file 79 | ''' 80 | for f in sorted(files): 81 | if f.endswith('.bdf'): 82 | 83 | print('Reading', os.path.join(subdir, f)) 84 | 85 | raw= mne.io.read_raw_edf(os.path.join(subdir, f), stim_channel='Status') 86 | picks = mne.pick_types(raw.info, eeg=True) 87 | events = mne.find_events(raw) 88 | epoch = mne.Epochs(raw, events, picks=picks) 89 | 90 | saved = [] 91 | rate = raw.info['sfreq'] #get sample frequency 92 | 93 | 94 | epoch = mne.Epochs(raw, events, picks=picks) 95 | data = epoch.get_data() 96 | 97 | ''' 98 | Iterate over events 99 | ''' 100 | for e, (event, d) in enumerate(zip(events, data)): 101 | emotions = triggers[event[2]] 102 | 103 | # print(d.shape, emotions, experiments[e]) 104 | 105 | 106 | # binary= bin(event[2]) 107 | # coded = binary[-9:-1] #extract 8 digits with trigger information 108 | # if coded not in ['10000000','01000000']: # not first and second rensponse 109 | # try: 110 | # stimul = triggers[coded[-3:]] # stimul class 111 | # start_time = int(event[0]/rate - 0.5) 112 | # stop_time = int(event[0]/rate + 2) #video was displaying for 1.5 sec + assuming the delay of trigger for 0.5 sec 113 | # repetition = '1_' 114 | # if coded in saved: 115 | # repetition = '2_' #the second image set has a bag in coding (some triggers codes are nor unique) 116 | # new_name = repetition + stimul+ '_' + str(coded) + '_' + f.split('.')[0] + '.bdf' 117 | # raw.save(path_to_save + stimul + '/' + new_name, tmin = start_time, tmax = stop_time, overwrite=True) 118 | # saved.append(coded) 119 | 120 | # print('saved') 121 | # except: 122 | # error_out.write('file: ' + f + ' ,' + binary + ' ,' + coded + '\n') 123 | 124 | def plot_channels(file, name): 125 | raw = mne.io.read_raw_edf(file) 126 | # raw.plot(block=True) 127 | event = mne.find_events(raw, stim_channel='Status') 128 | picks = mne.pick_types(raw.info, eeg=True, exclude='bads') 129 | names = raw.info['ch_names'] 130 | epoch = mne.Epochs(raw, event, tmax=1, picks=picks) 131 | # evoked = epoch.average() 132 | data = epoch.get_data() 133 | length = data.shape[2] 134 | 135 | for pick in range(data.shape[1]): 136 | y = np.reshape(data[0,pick, :], (length)) 137 | 138 | plt.subplot(15,41,pick+1) 139 | plt.plot(y) 140 | # plt.title(names[pick]) 141 | # plt.xlabel('time') 142 | # plt.ylabel('Hz') 143 | plt.subplots_adjust(hspace=1) 144 | plt.show() 145 | plt.savefig(name) 146 | plt.close() 147 | 148 | def plot_filtered(file): 149 | raw = mne.io.read_raw_fif(file) 150 | event = mne.find_events(raw) 151 | picks = mne.pick_types(raw.info, eeg=True, exclude='bads') 152 | epoch = mne.Epochs(raw, event, tmax=1, picks=[5]) 153 | evoked = epoch.average() 154 | evoked.plot() 155 | data = epoch.get_data() 156 | plt.show() 157 | 158 | def check_original_files(path): 159 | uniq = [] 160 | s = {'a' : 0, 'b' : 1} 161 | files = os.listdir(path) 162 | trials = {} 163 | for f in files: 164 | id = f[4:] 165 | image_set = s[f[2]] 166 | 167 | if id not in uniq: 168 | uniq.append(id) 169 | trials[id] = [0,0] 170 | try: 171 | trials[id][image_set-1] += 1 172 | except: 173 | print(f) 174 | print('Number of subjects: ', len(uniq)) 175 | print('Number of original files: ', len(files)) 176 | print(trials) 177 | 178 | def save_as_numpy(path_to_dataset, file): 179 | raw = mne.io.read_raw_edf( file ) 180 | 181 | event = mne.find_events(raw, stim_channel='Status') 182 | 183 | picks = mne.pick_types(raw.info, eeg=True) 184 | 185 | epoch = mne.Epochs(raw, event, picks=picks) 186 | 187 | data = epoch.get_data() 188 | # if data.shape[0] != 1: 189 | # print(file, event, data.shape) 190 | 191 | outfile = os.path.splitext(file)[0] + '.npy' 192 | np.save(outfile, data ) 193 | 194 | ''' 195 | Main: EGG 196 | ''' 197 | if __name__ == "__main__": 198 | print 'Starting:', time.ctime(), '\n' 199 | 200 | import argparse 201 | 202 | parser = argparse.ArgumentParser(description='Change folder order for OULU-CASIA dataset') 203 | parser.add_argument('-dataset', type=str, default='Blocked_raw', 204 | help='Name of the dataset to use') 205 | parser.add_argument('-participant', type=str, default='', 206 | help='Name of the participant') 207 | args = parser.parse_args() 208 | 209 | path_to_dataset = os.path.join('..', '_Dataset', args.dataset, args.participant) 210 | path_to_save = os.path.join('..', '_Dataset', 'Blocked') 211 | 212 | """ 213 | Convert EGG file into numpy array. 214 | """ 215 | split_dataset(path_to_dataset, path_to_save) 216 | 217 | # for file in files: 218 | # save_as_numpy( path_to_dataset, file ) 219 | # split_dataset(path_to_dataset, path_to_save) 220 | 221 | print '\nEnding:', time.ctime() -------------------------------------------------------------------------------- /_Preprocess/errors.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dangz90/Deep-Learning-for-Expression-Recognition-in-Image-Sequences/66c50292cac9a66eace32a040c49267b06c45de5/_Preprocess/errors.txt -------------------------------------------------------------------------------- /_Preprocess/faceDetection.py: -------------------------------------------------------------------------------- 1 | import os 2 | # import cv2 3 | import time 4 | import numpy as np 5 | 6 | ''' 7 | Function to obtain a list of files from a directory 8 | ''' 9 | def list_files(dir): 10 | from collections import defaultdict 11 | 12 | videos_by_class = defaultdict(list) 13 | subdirs = [x[0] for x in os.walk(dir) if len(x[0].split('/')) >= 8] 14 | 15 | for subdir in sorted(subdirs): 16 | files = os.walk(subdir).next()[2] 17 | for file in sorted(files): 18 | if file.endswith('.jpeg'): 19 | file_name = os.path.join(subdir, file) 20 | videos_by_class[os.path.basename(subdir)].append( file_name ) 21 | 22 | return videos_by_class 23 | 24 | def detect_face(list_of_files): 25 | # Obtain XML with Haar Features 26 | face_cascade = cv2.CascadeClassifier('C:\opencv\sources\data\haarcascades\haarcascade_frontalface_default.xml') 27 | 28 | # Cycle through the files 29 | for subdir, file in zip(r_subdir, r_file): 30 | print(subdir +'\\'+ file) 31 | 32 | img = cv2.imread(subdir +'\\'+ file) 33 | gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 34 | 35 | # Classifier for detecting faces 36 | faces = face_cascade.detectMultiScale(gray, 1.3, 5) 37 | for (x,y,w,h) in faces: 38 | cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2) 39 | # Create the face patch 40 | detected_color = img[y:y+h, x:x+w] 41 | detected_color = cv2.resize(detected_color, (224, 224)) 42 | # Save patch 43 | split_subdir = subdir.split('\\') 44 | cv2.imwrite(os.path.join('patches\\'+split_subdir[1]+'\\'+split_subdir[2]+'\\', file + '.jpg'), detected_color) # save frame as JPEG file 45 | 46 | if __name__ == '__main__': 47 | print 'Starting:', time.ctime(), '\n' 48 | 49 | import argparse 50 | 51 | parser = argparse.ArgumentParser(description='Change folder order for OULU-CASIA dataset') 52 | parser.add_argument('-dataset', type=str, default='OULU-CASIA', 53 | help='Name of the dataset to use') 54 | parser.add_argument('-partition', type=str, default='OriginalImg', 55 | help='Name of the data partition to use') 56 | parser.add_argument('-process', type=str, default='NI', 57 | help='Set to use (NI_Acropped, VL_Acropped)') 58 | args = parser.parse_args() 59 | 60 | path_to_dataset = os.path.join('..', '_Dataset', args.dataset, args.partition, args.process) 61 | 62 | # OULU-CASIA has an extra folder depth 63 | if args.dataset == 'OULU-CASIA': 64 | subfolder = 'Strong' 65 | path_to_dataset = os.path.join(path_to_dataset, subfolder) 66 | 67 | # Get list of files in dataset 68 | list_of_files = list_files(path_to_dataset) 69 | 70 | for classes, file in list_of_files.items(): 71 | detect_face(classes, list_of_files[classes], args.dataset) 72 | 73 | print '\nEnding:', time.ctime() -------------------------------------------------------------------------------- /_Preprocess/folder-person_to_folder-emotion.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import shutil 4 | import pickle 5 | import numpy as np 6 | 7 | from PIL import Image 8 | from collections import defaultdict 9 | 10 | __author__ = 'Daniel Garcia Zapata' 11 | 12 | 13 | classes = { 0:'Anger', 14 | 1:'Disgust', 15 | 2:'Fear', 16 | 3:'Happiness', 17 | 4:'Sadness', 18 | 5:'Surprise'} 19 | 20 | ''' 21 | Function to obtain frames from a video 22 | ''' 23 | def move_videos(classes, list_of_files, dataset): 24 | 25 | new_base_path = os.path.join('..', '_Dataset', dataset, 'face') 26 | 27 | for old_video in list_of_files: 28 | participant_number = os.path.dirname(old_video).split(os.sep)[4] 29 | 30 | # Validation 31 | if int(filter(str.isdigit, os.path.dirname(old_video).split(os.sep)[4])) > 60: 32 | new_path = os.path.join(new_base_path, 'validation', classes) 33 | new_file_name = participant_number +'_'+ os.path.basename(old_video) 34 | 35 | if not os.path.exists(new_path): 36 | os.makedirs(new_path) 37 | 38 | new_full_path = os.path.join(new_path, new_file_name) 39 | shutil.copy(old_video, new_full_path) 40 | 41 | # Training 42 | else: 43 | new_path = os.path.join(new_base_path, 'training', classes) 44 | new_file_name = participant_number +'_'+ os.path.basename(old_video) 45 | 46 | if not os.path.exists(new_path): 47 | os.makedirs(new_path) 48 | 49 | new_full_path = os.path.join(new_path, new_file_name) 50 | shutil.copy(old_video, new_full_path) 51 | 52 | # frames_per_participants = defaultdict(int) 53 | # for k, video in enumerate(list_of_files): 54 | # current_participant_number = os.path.basename(video).split('_')[0] 55 | 56 | # # Frames to use: frame_counter + 16 Size of the frames of the CNN 57 | # current_frame = int(video.replace('.jpg','').replace('.mp4frame', '=').split('=')[1]) 58 | # frame_limit = participant_frames[current_participant_number] 59 | 60 | # if frame_counter > number_of_frames and original_participant_number == current_participant_number: 61 | # ''' 62 | # Save 63 | # ''' 64 | # path_to_save = os.path.join( os.path.dirname(video).replace('2_Model','2_3DModel').replace(dataset, 'dataset'+number_of_frames+'_consecutive_3d') , os.path.basename(video).split('.')[0] + '_' + str(participant_frames_number) ) 65 | # print(path_to_save) 66 | # # np.save( path_to_save, vid ) 67 | 68 | # # Delete first video in order to keep always size 16 before appending the next video 69 | # # del vid[0] 70 | 71 | # if original_participant_number != current_participant_number or k == len(list_of_videos)-1: 72 | # ''' 73 | # Restore counter 74 | # ''' 75 | # vid = [] 76 | # frame_counter = 1 77 | # participant_frames_number = 1 78 | # original_participant_number = current_participant_number 79 | 80 | # vid.append(os.path.basename(video)) 81 | # frame_counter += 1 82 | 83 | def create_path(directory, subdirectory, classes, new_file_name, extension): 84 | new_path = os.path.join(directory, subdirectory, classes) 85 | full_path = os.path.join(new_path, new_file_name + extension) 86 | 87 | if not os.path.exists(new_path): 88 | os.makedirs(new_path) 89 | 90 | return full_path 91 | 92 | 93 | def read_pickle(subdir, file_path): 94 | with open(file_path, 'rb') as f: 95 | data = pickle.load(f) 96 | 97 | participant_number = os.path.splitext(os.path.basename(file_path))[0] 98 | 99 | for e, (emotion, geometry, face) in enumerate(zip(data['emos'], data['geoms'], data['faces'])): 100 | # Create file name 101 | new_file_name = participant_number +'_'+ str(e) 102 | 103 | # Geometry 104 | full_path_geometry = create_path(subdir, '_geometry', classes[emotion], new_file_name, '.npy') 105 | # np.save(full_path_geometry, geometry) 106 | 107 | # Face 108 | full_path_face = create_path(subdir, '_face', classes[emotion], new_file_name, '.jpg') 109 | 110 | print full_path_face 111 | 112 | 113 | # img = Image.fromarray(face, 'RGB') 114 | # result.save(img) 115 | 116 | ''' 117 | Function to obtain a list of files from a directory 118 | ''' 119 | def list_files(dir, extension): 120 | from collections import defaultdict 121 | 122 | # Explore sub directories 123 | subdirs = [x[0] for x in os.walk(dir)] # if len(x[0].split(os.sep)) >= 6 124 | 125 | list_of_files = defaultdict(list) 126 | # Cycle through sub directories 127 | for subdir in sorted(subdirs): 128 | # Obtain files from sub directories 129 | files = os.walk(subdir).next()[2] 130 | # Cycle through files 131 | for e, file in enumerate(sorted(files)): 132 | 133 | if file.endswith(extension): 134 | file_path = os.path.join(subdir, file) 135 | 136 | # Obtain file class and put geometry and image in respective folder 137 | # read_pickle(os.path.dirname(subdir), file_path) 138 | 139 | list_of_files[os.path.basename(subdir)].append( file_path ) 140 | 141 | return list_of_files 142 | 143 | 144 | if __name__ == '__main__': 145 | print 'Starting:', time.ctime(), '\n' 146 | 147 | import argparse 148 | 149 | parser = argparse.ArgumentParser(description='Change folder order for OULU-CASIA dataset') 150 | parser.add_argument('-dataset', type=str, default='OULU-CASIA', 151 | help='Name of the dataset to use') 152 | parser.add_argument('-partition', type=str, default='croppedfaces', 153 | help='Name of the data partition to use') 154 | parser.add_argument('-process', type=str, default='', 155 | help='Set to use (NI_Acropped, VL_Acropped)') 156 | parser.add_argument('-depth', type=str, default='', 157 | help='Add if there is an extra depth') 158 | parser.add_argument('-extension', type=str, default='.jpeg', 159 | help='Add extension to extract') 160 | args = parser.parse_args() 161 | 162 | path_to_dataset = os.path.join('..', '_Dataset', args.dataset, args.partition, args.process, args.depth) 163 | print 'Using folder:', path_to_dataset 164 | 165 | list_of_files = list_files(path_to_dataset, args.extension) 166 | for classes, videos in list_of_files.items(): 167 | print(classes) 168 | move_videos(classes, list_of_files[classes], args.dataset) 169 | 170 | print '\nEnding:', time.ctime() -------------------------------------------------------------------------------- /_Preprocess/folder-person_to_geometry-face.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import pickle 4 | import numpy as np 5 | 6 | from PIL import Image 7 | from collections import defaultdict 8 | 9 | __author__ = 'Daniel Garcia Zapata' 10 | 11 | 12 | classes = { 0:'Anger', 13 | 1:'Disgust', 14 | 2:'Fear', 15 | 3:'Happiness', 16 | 4:'Sadness', 17 | 5:'Surprise'} 18 | 19 | def create_path(directory, subdirectory, classes, partition, new_file_name, extension): 20 | new_path = os.path.join(directory, subdirectory, partition, classes) 21 | full_path = os.path.join(new_path, new_file_name + extension) 22 | 23 | if not os.path.exists(new_path): 24 | os.makedirs(new_path) 25 | 26 | return full_path 27 | 28 | 29 | def read_pickle(subdir, file_path): 30 | with open(file_path, 'rb') as f: 31 | data = pickle.load(f) 32 | 33 | participant_number = os.path.splitext(os.path.basename(file_path))[0] 34 | 35 | previous_emotion = -1 36 | for emotion, geometry, face in zip(data['emos'], data['geoms'], data['faces']): 37 | # Check count of emotion 38 | if emotion != previous_emotion: 39 | count = 0 40 | else: 41 | count += 1 42 | previous_emotion = emotion 43 | 44 | # Create file name 45 | if int(filter(str.isdigit, participant_number)) > 65: 46 | partition = 'validation' 47 | else: 48 | partition = 'training' 49 | 50 | new_file_name = participant_number +'_'+ format(count, '02') 51 | 52 | # Geometry 53 | full_path_geometry = create_path(subdir, '_Geometry', classes[emotion], partition, new_file_name, '.npy') 54 | np.save(full_path_geometry, geometry) 55 | 56 | # Face 57 | full_path_face = create_path(subdir, '_Face', classes[emotion], partition, new_file_name, '.jpg') 58 | img = Image.fromarray(face, 'RGB') 59 | img.save(full_path_face) 60 | 61 | ''' 62 | Function to obtain a list of files from a directory 63 | ''' 64 | def list_files(dir, extension): 65 | from collections import defaultdict 66 | 67 | # Explore sub directories 68 | subdirs = [x[0] for x in os.walk(dir)] # if len(x[0].split(os.sep)) >= 6 69 | 70 | list_of_files = defaultdict(list) 71 | # Cycle through sub directories 72 | for subdir in sorted(subdirs): 73 | # Obtain files from sub directories 74 | files = os.walk(subdir).next()[2] 75 | # Cycle through files 76 | for e, file in enumerate(sorted(files)): 77 | print 'Participant', e 78 | 79 | if file.endswith(extension): 80 | file_path = os.path.join(subdir, file) 81 | 82 | # Obtain file class and put geometry and image in respective folder 83 | read_pickle(os.path.dirname(subdir), file_path) 84 | 85 | return list_of_files 86 | 87 | if __name__ == '__main__': 88 | import argparse 89 | 90 | print 'Starting:', time.ctime(), '\n' 91 | 92 | parser = argparse.ArgumentParser(description='Change folder order for OULU-CASIA dataset') 93 | parser.add_argument('-dataset', type=str, default='OULU-CASIA', 94 | help='Name of the dataset to use') 95 | parser.add_argument('-preprocess', type=str, default='croppedfaces', 96 | help='Name of the data process to use') 97 | parser.add_argument('-process', type=str, default='', 98 | help='Set to use (NI_Acropped, VL_Acropped)') 99 | parser.add_argument('-depth', type=str, default='', 100 | help='Add if there is an extra depth') 101 | parser.add_argument('-extension', type=str, default='.jpeg', 102 | help='Add extension to extract') 103 | args = parser.parse_args() 104 | 105 | path_to_dataset = os.path.join('..', '_Dataset', args.dataset, args.preprocess, args.process, args.depth) 106 | print 'Using folder:', path_to_dataset, '\n' 107 | 108 | ''' 109 | Get the list of files 110 | ''' 111 | list_of_files = list_files(path_to_dataset, args.extension) 112 | 113 | print '\nEnding:', time.ctime() -------------------------------------------------------------------------------- /_Preprocess/frame_to_16frames.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import time 4 | import json 5 | import numpy as np 6 | 7 | from collections import defaultdict 8 | 9 | ''' 10 | Function to obtain frames from a video 11 | ''' 12 | def video_to_frames(classes, list_of_videos, dataset, number_of_frames): 13 | 14 | print(classes) 15 | 16 | vid = [] 17 | frame_counter = 1 18 | participant_frames_number = 1 19 | original_participant_number = os.path.basename(list_of_videos[0]).split('_')[0] 20 | 21 | # Frames per participants 22 | participant_frames = np.load(classes+'_frames_per_participants.npy').tolist() 23 | 24 | frames_per_participants = defaultdict(int) 25 | for k, video in enumerate(list_of_videos): 26 | current_participant_number = os.path.basename(video).split('_')[0] 27 | 28 | # Frames to use: frame_counter + 16 Size of the frames of the CNN 29 | current_frame = int(video.replace('.jpg','').replace('.mp4frame', '=').split('=')[1]) 30 | frame_limit = participant_frames[current_participant_number] 31 | 32 | if frame_counter > number_of_frames and original_participant_number == current_participant_number: 33 | ''' 34 | Save 35 | ''' 36 | path_to_save = os.path.join( os.path.dirname(video).replace('2_Model','2_3DModel').replace(dataset, 'dataset'+number_of_frames+'_consecutive_3d') , os.path.basename(video).split('.')[0] + '_' + str(participant_frames_number) ) 37 | print(path_to_save) 38 | # np.save( path_to_save, vid ) 39 | 40 | # Delete first video in order to keep always size 16 before appending the next video 41 | del vid[0] 42 | 43 | if original_participant_number != current_participant_number or k == len(list_of_videos)-1: 44 | ''' 45 | Restore counter 46 | ''' 47 | vid = [] 48 | frame_counter = 1 49 | participant_frames_number = 1 50 | original_participant_number = current_participant_number 51 | 52 | vid.append(os.path.basename(video)) 53 | frame_counter += 1 54 | 55 | print('\n') 56 | 57 | ''' 58 | Function to obtain a list of files from a directory 59 | ''' 60 | def list_files(dir): 61 | from collections import defaultdict 62 | 63 | videos_by_class = defaultdict(list) 64 | 65 | subdirs = [x[0] for x in os.walk(dir)] 66 | for subdir in subdirs: 67 | files = os.walk(subdir).__next__()[2] 68 | if (len(files) > 0): 69 | for file in files: 70 | videos_by_class[os.path.basename(subdir)].append( os.path.join(subdir, file) ) 71 | 72 | return videos_by_class 73 | 74 | 75 | if __name__ == '__main__': 76 | print('Starting:', time.ctime(), '\n') 77 | 78 | import argparse 79 | 80 | parser = argparse.ArgumentParser(description='Generate the dataset for the 3D CNN') 81 | parser.add_argument('-dataset', type=str, default='patches', 82 | help='Name of the dataset to use') 83 | parser.add_argument('-set', type=str, default='validation', 84 | help='Set to use (training, validation, test)') 85 | parser.add_argument('-frames', type=int, default=5, 86 | help='Number of frames') 87 | args = parser.parse_args() 88 | 89 | path_to_dataset = os.path.join('..', '_Dataset', args.dataset, args.set) 90 | print(path_to_dataset) 91 | videos_by_class = list_files(path_to_dataset) 92 | 93 | 94 | for classes, videos in videos_by_class.items(): 95 | video_to_frames(classes, videos_by_class[classes], args.dataset, args.frames) 96 | 97 | print('\nEnding:', time.ctime()) -------------------------------------------------------------------------------- /_Preprocess/frames_to_dataset_3d.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import numpy as np 4 | 5 | from collections import defaultdict 6 | 7 | __author__ = 'Daniel Garcia Zapata' 8 | 9 | ''' 10 | Function to obtain frames from a video 11 | ''' 12 | def video_to_frames(classes, list_of_videos, dataset, data_set, number_of_frames): 13 | 14 | print('Class:', classes) 15 | 16 | vid = [] 17 | frame_counter = 1 18 | participant_frames_number = 1 19 | original_participant_number = os.path.basename(list_of_videos[0]).split('_')[0] # Number of the participant 20 | 21 | # Frames per participants 22 | frames_per_participants_path = os.path.join('..', '_Dataset', 'frames_per_participants', classes+'_frames_per_participants.npy') 23 | participant_frames = np.load(frames_per_participants_path).tolist() 24 | 25 | frames_per_participants = defaultdict(int) 26 | for k, video in enumerate(list_of_videos): 27 | current_participant_number = os.path.basename(video).split('_')[0] 28 | 29 | # Frames to use: frame_counter + number_of_frames Size of the frames 30 | if data_set == 'validation': 31 | current_frame = int(video.replace('.jpg','').replace('.mp4frame', '=').split('=')[1]) 32 | else: 33 | current_frame = int( video.replace('.jpg.jpg','').replace('.MP4frame', '=').split('=')[1] ) 34 | 35 | frame_limit = participant_frames[current_participant_number] 36 | 37 | if frame_counter > number_of_frames and original_participant_number == current_participant_number: 38 | ''' 39 | Save 40 | ''' 41 | folder_name = os.path.dirname(video).replace(dataset, str(number_of_frames)+'frames') 42 | participant_and_class = os.path.basename(video).split('.')[0] 43 | 44 | if not os.path.exists(folder_name): 45 | os.makedirs(folder_name) 46 | 47 | path_to_save = os.path.join( folder_name, participant_and_class + '_' + str(participant_frames_number) ) 48 | np.save( path_to_save, vid ) 49 | 50 | # Delete first video in order to keep always size 16 before appending the next video 51 | del vid[0] 52 | participant_frames_number += 1 53 | 54 | if original_participant_number != current_participant_number or k == len(list_of_videos)-1: 55 | ''' 56 | Restore counter 57 | ''' 58 | vid = [] 59 | frame_counter = 1 60 | participant_frames_number = 1 61 | original_participant_number = current_participant_number 62 | 63 | vid.append(os.path.basename(video)) 64 | frame_counter += 1 65 | 66 | 67 | ''' 68 | Function to obtain a list of files from a directory 69 | ''' 70 | def list_files(dir): 71 | from collections import defaultdict 72 | 73 | videos_by_class = defaultdict(list) 74 | 75 | subdirs = [x[0] for x in os.walk(dir)] 76 | for subdir in subdirs: 77 | files = os.walk(subdir).__next__()[2] 78 | if (len(files) > 0): 79 | for file in sorted(files): 80 | videos_by_class[os.path.basename(subdir)].append( os.path.join(subdir, file) ) 81 | 82 | return videos_by_class 83 | 84 | 85 | if __name__ == '__main__': 86 | import argparse 87 | 88 | parser = argparse.ArgumentParser(description='Generate a dataset of frames') 89 | parser.add_argument('-dataset', 90 | type=str, default='patches', 91 | help='Name of the dataset to use') 92 | parser.add_argument('-set', 93 | type=str, default='validation', 94 | help='Set to use (training, validation)') 95 | parser.add_argument('-number_of_frames', 96 | type=int, default=5, 97 | help='Number of frames to be used to create the dataset') 98 | args = parser.parse_args() 99 | 100 | ''' __main__ ''' 101 | print("Starting:", time.ctime()) 102 | 103 | # Obtain list of files in dataset directories 104 | path_to_dataset = os.path.join('..', '_Dataset', args.dataset, args.set) 105 | videos_by_class = list_files(path_to_dataset) 106 | 107 | # Create npy file with {number_of_frames} frames to be used as an input 108 | for classes, videos in videos_by_class.items(): 109 | video_to_frames(classes, videos_by_class[classes], args.dataset, args.set, args.number_of_frames) 110 | 111 | print("Ending:", time.ctime()) -------------------------------------------------------------------------------- /_Preprocess/oulu-obtain_face.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import numpy as np 4 | 5 | from collections import defaultdict 6 | 7 | ''' 8 | Function to obtain frames from a video 9 | ''' 10 | def move_videos(classes, list_of_videos, dataset): 11 | 12 | new_base_path = os.path.join('..', '_Dataset', dataset, 'patch') 13 | 14 | for old_video in list_of_videos: 15 | participant_number = os.path.dirname(old_video).split('/')[6] 16 | 17 | # Validation 18 | if int(filter(str.isdigit, os.path.dirname(old_video).split('/')[6])) > 60: 19 | new_path = os.path.join(new_base_path, 'validation', classes) 20 | new_file_name = participant_number +'_'+ os.path.basename(old_video) 21 | 22 | # if not os.path.exists(new_path): 23 | # os.makedirs(new_path) 24 | 25 | full_path = os.path.join(new_path, new_file_name) 26 | # os.rename(old_video, full_path) 27 | 28 | # Training 29 | else: 30 | new_path = os.path.join(new_base_path, 'training', classes) 31 | new_file_name = participant_number +'_'+ os.path.basename(old_video) 32 | 33 | # if not os.path.exists(new_path): 34 | # os.makedirs(new_path) 35 | 36 | full_path = os.path.join(new_path, new_file_name) 37 | # os.rename(old_video, full_path) 38 | 39 | print old_video 40 | 41 | # frames_per_participants = defaultdict(int) 42 | # for k, video in enumerate(list_of_videos): 43 | # current_participant_number = os.path.basename(video).split('_')[0] 44 | 45 | # # Frames to use: frame_counter + 16 Size of the frames of the CNN 46 | # current_frame = int(video.replace('.jpg','').replace('.mp4frame', '=').split('=')[1]) 47 | # frame_limit = participant_frames[current_participant_number] 48 | 49 | # if frame_counter > number_of_frames and original_participant_number == current_participant_number: 50 | # ''' 51 | # Save 52 | # ''' 53 | # path_to_save = os.path.join( os.path.dirname(video).replace('2_Model','2_3DModel').replace(dataset, 'dataset'+number_of_frames+'_consecutive_3d') , os.path.basename(video).split('.')[0] + '_' + str(participant_frames_number) ) 54 | # print(path_to_save) 55 | # # np.save( path_to_save, vid ) 56 | 57 | # # Delete first video in order to keep always size 16 before appending the next video 58 | # # del vid[0] 59 | 60 | # if original_participant_number != current_participant_number or k == len(list_of_videos)-1: 61 | # ''' 62 | # Restore counter 63 | # ''' 64 | # vid = [] 65 | # frame_counter = 1 66 | # participant_frames_number = 1 67 | # original_participant_number = current_participant_number 68 | 69 | # vid.append(os.path.basename(video)) 70 | # frame_counter += 1 71 | 72 | ''' 73 | Function to obtain a list of files from a directory 74 | ''' 75 | def list_files(dir): 76 | from collections import defaultdict 77 | 78 | videos_by_class = defaultdict(list) 79 | subdirs = [x[0] for x in os.walk(dir) if len(x[0].split('/')) >= 8] 80 | 81 | for subdir in sorted(subdirs): 82 | files = os.walk(subdir).next()[2] 83 | for file in sorted(files): 84 | if file.endswith('.jpeg'): 85 | file_name = os.path.join(subdir, file) 86 | videos_by_class[os.path.basename(subdir)].append( file_name ) 87 | 88 | return videos_by_class 89 | 90 | 91 | if __name__ == '__main__': 92 | print 'Starting:', time.ctime(), '\n' 93 | 94 | import argparse 95 | 96 | parser = argparse.ArgumentParser(description='Change folder order for OULU-CASIA dataset') 97 | parser.add_argument('-dataset', type=str, default='OULU-CASIA', 98 | help='Name of the dataset to use') 99 | parser.add_argument('-partition', type=str, default='OriginalImg', 100 | help='Name of the data partition to use') 101 | parser.add_argument('-process', type=str, default='VL', 102 | help='Set to use (NI_Acropped, VL_Acropped)') 103 | args = parser.parse_args() 104 | 105 | path_to_dataset = os.path.join('..', '_Dataset', args.dataset, args.partition, args.process) 106 | 107 | # OULU-CASIA has an extra folder depth 108 | if args.dataset == 'OULU-CASIA': 109 | subfolder = 'Strong' 110 | path_to_dataset = os.path.join(path_to_dataset, subfolder) 111 | 112 | videos_by_class = list_files(path_to_dataset) 113 | 114 | for classes, videos in videos_by_class.items(): 115 | move_videos(classes, videos_by_class[classes], args.dataset) 116 | 117 | print '\nEnding:', time.ctime() -------------------------------------------------------------------------------- /_Preprocess/preprocess_EGG.py: -------------------------------------------------------------------------------- 1 | import os 2 | import mne 3 | import numpy as np 4 | 5 | import matplotlib 6 | import plotly.plotly as py 7 | import matplotlib.pyplot as plt 8 | 9 | mne.set_log_level('CRITICAL') 10 | 11 | ############################################################################################### 12 | # 13 | #Reading eeg files. Each files contain informatiom for one trial (watching 60 images 14 | # wth 2 followong questions). Split file to events correspoding to watching specific 15 | #type of imae (see triggers). Assumme delay for video display for 0.5 sec 16 | # 17 | #OUT: "_____.bdf". if first symbol is 2, then the event had dublicate in the trial (can't identify an image) 18 | # 19 | ############################################################################################### 20 | 21 | def split_dataset(data_path, path_to_save): 22 | error_out = open('errors.txt','w') 23 | if (not os.path.isdir(data_path)): 24 | assert False 25 | if (not os.path.isdir(path_to_save)): 26 | os.mkdir(path_to_save) 27 | triggers = {'001':'ntr', '010':'sex','100':'veri','011':'posM', '101':'negM'} 28 | for v in triggers.values(): 29 | os.mkdir(path_to_save+v) 30 | for f in os.listdir(data_path): 31 | raw= mne.io.read_raw_edf(data_path+ f, preload=False) 32 | events = mne.find_events(raw) 33 | saved = [] 34 | rate = raw.info['sfreq'] #get sample frequency 35 | for event in events: 36 | binary= bin(event[2]) 37 | coded = binary[-9:-1] #extract 8 digits with trigger information 38 | if coded not in ['10000000','01000000']: # not first and second rensponse 39 | try: 40 | stimul = triggers[coded[-3:]] # stimul class 41 | start_time = int(event[0]/rate - 0.5) 42 | stop_time = int(event[0]/rate + 2) #video was displaying for 1.5 sec + assuming the delay of trigger for 0.5 sec 43 | repetition = '1_' 44 | if coded in saved: 45 | repetition = '2_' #the second image set has a bag in coding (some triggers codes are nor unique) 46 | new_name = repetition + stimul+ '_' + str(coded) + '_' + f.split('.')[0] + '.bdf' 47 | raw.save(path_to_save + stimul + '/' + new_name, tmin = start_time, tmax = stop_time, overwrite=True) 48 | saved.append(coded) 49 | except: 50 | error_out.write('file: ' + f + ' ,' + binary + ' ,' + coded + '\n') 51 | error_out.close() 52 | 53 | 54 | def plot_channels(file, name): 55 | raw = mne.io.read_raw_fif(file) 56 | # raw.plot(block=True) 57 | event = mne.find_events(raw) 58 | picks = mne.pick_types(raw.info, eeg=True, exclude='bads') 59 | names = raw.info['ch_names'] 60 | epoch = mne.Epochs(raw, event, tmax=1, picks=picks) 61 | # evoked = epoch.average() 62 | data = epoch.get_data() 63 | length = data.shape[2] 64 | for pick in range(data.shape[1]): 65 | y = np.reshape(data[:,pick, :], (length)) 66 | plt.subplot(8,6,pick+1) 67 | plt.plot(y) 68 | # plt.title(names[pick]) 69 | # plt.xlabel('time') 70 | # plt.ylabel('Hz') 71 | plt.subplots_adjust(hspace=1) 72 | plt.savefig(name) 73 | plt.close() 74 | 75 | def plot_filtered(file): 76 | raw = mne.io.read_raw_fif(file) 77 | event = mne.find_events(raw) 78 | picks = mne.pick_types(raw.info, eeg=True, exclude='bads') 79 | epoch = mne.Epochs(raw, event, tmax=1, picks=[5]) 80 | evoked = epoch.average() 81 | evoked.plot() 82 | data = epoch.get_data() 83 | plt.show() 84 | 85 | def check_original_files(path): 86 | uniq = [] 87 | s = {'a' : 0, 'b' : 1} 88 | files = os.listdir(path) 89 | trials = {} 90 | for f in files: 91 | id = f[4:] 92 | image_set = s[f[2]] 93 | 94 | if id not in uniq: 95 | uniq.append(id) 96 | trials[id] = [0,0] 97 | try: 98 | trials[id][image_set-1] += 1 99 | except: 100 | print(f) 101 | print('Number of subjects: ', len(uniq)) 102 | print('Number of original files: ', len(files)) 103 | print(trials) 104 | 105 | def save_as_numpy(path, where_to_save): 106 | for sub in os.listdir(path): 107 | for file in os.listdir(os.path.join(path, sub)): 108 | 109 | print(os.path.join(path, sub, file)) 110 | 111 | raw = mne.io.read_raw_edf( os.path.join(path, sub, file) ) 112 | 113 | event = mne.find_events(raw) 114 | picks = mne.pick_types(raw.info, eeg=True) 115 | epoch = mne.Epochs(raw, event, picks=picks) 116 | data = epoch.get_data() 117 | if data.shape[0] != 1: 118 | print(sub, file, event, data.shape) 119 | 120 | np.save(where_to_save + sub +'/'+ file.split('.')[0] + '.npy', np.reshape(data, (data.shape[1], data.shape[2]))) 121 | 122 | ''' 123 | ___init___ 124 | ''' 125 | 126 | # check_original_files(os.path.join('dataset', 'EGG')) 127 | 128 | # plot_filtered('dataset\EGG\h1a3ki42.bdf') 129 | 130 | save_as_numpy( os.path.join('dataset'), os.path.join('dataset','numpy_data') ) 131 | 132 | print('\nEND') -------------------------------------------------------------------------------- /_Preprocess/resize16.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import numpy as np 4 | from shutil import copyfile 5 | 6 | ''' 7 | Function to obtain frames from a video 8 | ''' 9 | def video_to_frames(classes, list_of_videos, dataset): 10 | print(classes) 11 | 12 | for videos in list_of_videos: 13 | f = np.load(videos) 14 | images_list = [fi for fi in f] 15 | 16 | for images in images_list: 17 | # Obtain original image 18 | source_path = os.path.dirname(videos) 19 | source = os.path.join(source_path.replace('dataset16_consecutive_3d', 'patches'), images) 20 | 21 | # Read source image and resize to 112,112 22 | img = cv2.imread(source) 23 | img = cv2.resize(img, (112, 112)) 24 | 25 | # Save modified image 26 | destination = os.path.join(source_path.replace('dataset16_consecutive_3d', 'patches_consecutive112'), images) 27 | cv2.imwrite(destination, img) # save frame as JPEG file 28 | 29 | ''' 30 | Function to obtain a list of files from a directory 31 | ''' 32 | def list_files(dir): 33 | from collections import defaultdict 34 | 35 | videos_by_class = defaultdict(list) 36 | 37 | subdirs = [x[0] for x in os.walk(dir)] 38 | for subdir in subdirs: 39 | files = os.walk(subdir).__next__()[2] 40 | if (len(files) > 0): 41 | for file in files: 42 | videos_by_class[os.path.basename(subdir)].append( os.path.join(subdir, file) ) 43 | 44 | return videos_by_class 45 | 46 | 47 | if __name__ == '__main__': 48 | import argparse 49 | 50 | parser = argparse.ArgumentParser(description='Generate the dataset for the 3D CNN') 51 | parser.add_argument('-dataset', type=str, default='patches', 52 | help='Name of the dataset to use') 53 | parser.add_argument('-set', type=str, default='validation', 54 | help='Set to use (training, validation, test)') 55 | args = parser.parse_args() 56 | 57 | args.dataset = 'dataset16_consecutive_3d' 58 | path_to_dataset = os.path.join('..', '2_3DModel', '2_Dataset', args.dataset, args.set) 59 | videos_by_class = list_files(path_to_dataset) 60 | 61 | for classes, videos in videos_by_class.items(): 62 | video_to_frames(classes, videos_by_class[classes], args.dataset) 63 | 64 | print('\nEND') -------------------------------------------------------------------------------- /_Preprocess/verify_16.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import numpy as np 4 | from shutil import copyfile 5 | 6 | ''' 7 | Function to obtain frames from a video 8 | ''' 9 | def video_to_frames(classes, list_of_videos, dataset): 10 | print(classes) 11 | 12 | for videos in list_of_videos: 13 | f = np.load(videos) 14 | images_list = [fi for fi in f] 15 | 16 | if len(images_list) == 16: 17 | source = videos 18 | destination = videos.replace('dataset_3d', 'dataset16_3d') 19 | 20 | # Copy File from dataset_3d to patches 21 | copyfile(source, destination) 22 | 23 | ''' 24 | Function to obtain a list of files from a directory 25 | ''' 26 | def list_files(dir): 27 | from collections import defaultdict 28 | 29 | videos_by_class = defaultdict(list) 30 | 31 | subdirs = [x[0] for x in os.walk(dir)] 32 | for subdir in subdirs: 33 | files = os.walk(subdir).__next__()[2] 34 | if (len(files) > 0): 35 | for file in files: 36 | videos_by_class[os.path.basename(subdir)].append( os.path.join(subdir, file) ) 37 | 38 | return videos_by_class 39 | 40 | 41 | if __name__ == '__main__': 42 | import argparse 43 | 44 | parser = argparse.ArgumentParser(description='Generate the dataset for the 3D CNN') 45 | parser.add_argument('-dataset', type=str, default='patches', 46 | help='Name of the dataset to use') 47 | parser.add_argument('-set', type=str, default='validation', 48 | help='Set to use (training, validation, test)') 49 | args = parser.parse_args() 50 | 51 | args.dataset = 'dataset_3d' 52 | path_to_dataset = os.path.join('..', '2_3DModel', '2_Dataset', args.dataset, args.set) 53 | videos_by_class = list_files(path_to_dataset) 54 | 55 | for classes, videos in videos_by_class.items(): 56 | video_to_frames(classes, videos_by_class[classes], args.dataset) 57 | 58 | print('END') -------------------------------------------------------------------------------- /_Preprocess/video_to_frame.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import os 3 | import numpy as np 4 | import re 5 | 6 | ''' 7 | Function to obtain frames from a video 8 | ''' 9 | def video_to_frames(dir, dirname, file): 10 | 11 | vidcap = cv2.VideoCapture(dir) 12 | 13 | ''' 14 | Get length of the videos in frames: 15 | - Get frames from 50% of the video's length until 90% of the length 16 | ''' 17 | length_of_video = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT)) 18 | 19 | success, image = vidcap.read() 20 | 21 | 22 | new_dir = dir.replace('patches','frames3D').split('.MP4')[0] 23 | 24 | # print( new_dir ) 25 | print( os.path.basename(new_dir) ) 26 | 27 | count = 0 28 | frame_counter = 0 29 | vid = [] 30 | while success: 31 | success,image = vidcap.read() 32 | 33 | 34 | 35 | if frame_counter > 16: 36 | frame_counter = 0 37 | print(file) 38 | # np.save(, vid) 39 | vid = [] 40 | 41 | if count > int(length_of_video*.6) and count < int(length_of_video*.9): 42 | vid.append(dirname) 43 | # cv2.imwrite(os.path.join(dirname, file + 'frame%d.jpg' % count), image) # save frame as JPEG file 44 | 45 | 46 | if cv2.waitKey(10) == 27: # exit if Escape is hit 47 | break 48 | 49 | count += 1 50 | frame_counter += 1 51 | 52 | 53 | # vid = [] 54 | # while True: 55 | # ret, img = cap.read() 56 | # if not ret: 57 | # break 58 | # vid.append(cv2.resize(img, (171, 128))) 59 | # vid = np.array(vid, dtype=np.float32) 60 | 61 | 62 | ''' 63 | Function to obtain a list of files from a directory 64 | ''' 65 | def list_files(dir): 66 | r = [] 67 | r_subdir = [] 68 | r_file = [] 69 | subdirs = [x[0] for x in os.walk(dir)] 70 | for subdir in subdirs: 71 | files = os.walk(subdir).__next__()[2] 72 | if (len(files) > 0): 73 | for file in files: 74 | r.append( os.path.join(subdir, file) ) 75 | r_subdir.append(subdir) 76 | r_file.append(file) 77 | return r, r_subdir, r_file 78 | 79 | 80 | if __name__ == '__main__': 81 | import argparse 82 | 83 | parser = argparse.ArgumentParser(description='Generate the dataset for the 3D CNN') 84 | parser.add_argument('-dataset', type=str, default='patches', 85 | help='Name of the dataset to use') 86 | parser.add_argument('-set', type=str, default='training', 87 | help='Set to use (training, validation, test)') 88 | args = parser.parse_args() 89 | 90 | ''' 91 | @ r: list of directory path 92 | @ r_subdir: list of subdirectories 93 | @ r_file: list of files 94 | ''' 95 | path_to_dataset = os.path.join('..', '2_Model', '2_Dataset', args.dataset, args.set) 96 | r, r_subdir, r_file = list_files(path_to_dataset) 97 | 98 | for video, dirname, file in zip(r, r_subdir, r_file): 99 | video_to_frames(video, dirname, file) --------------------------------------------------------------------------------