├── CNN_Jaffe_test.py ├── CNN_lfw_test.py ├── CNN_single_image_test.py ├── CNN_train.py ├── README.md ├── augment_jaffe_data.py ├── autoencoder_train_appender.py ├── my_model_360_iter_batch_100.h5 ├── one_layer_autoencoder_test.py ├── one_layer_autoencoder_train.py ├── predictions_lfw.txt ├── test_files.txt ├── train_files.txt ├── train_test_splitter.py ├── two_layer_autoencoder_test.py ├── two_layer_autoencoder_train.py └── weights.txt /CNN_Jaffe_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Sat Apr 15 16:27:02 2017 5 | 6 | @author: prudhvi 7 | """ 8 | 9 | """ 10 | The program loads the trained CNN model 11 | and tests it on the JAFFE test set. 12 | 13 | This should print the number of correct 14 | predictions, total numner of test files, 15 | the accuracy percentage, and a confusion 16 | matrix 17 | """ 18 | 19 | import numpy as np 20 | import os 21 | from PIL import Image 22 | np.random.rand(2) 23 | #from keras.layers import Conv1D 24 | files = os.listdir('aug_data_64_by_48') 25 | 26 | 27 | tag_list = ['AN', 'SA', 'SU', 'HA', 'DI', 'FE', 'NE'] 28 | 29 | def targets(filename): 30 | targets = [] 31 | for f in filename: 32 | if tag_list[0] in f: 33 | targets.append(0) 34 | if tag_list[1] in f: 35 | targets.append(1) 36 | if tag_list[2] in f: 37 | targets.append(2) 38 | if tag_list[3] in f: 39 | targets.append(3) 40 | if tag_list[4] in f: 41 | targets.append(4) 42 | if tag_list[5] in f: 43 | targets.append(5) 44 | if tag_list[6] in f: 45 | targets.append(6) 46 | return np.array(targets) 47 | 48 | 49 | def data(filename): 50 | train_images = [] 51 | for f in filename: 52 | current = f 53 | train_images.append(np.array(Image.open('aug_data_64_by_48/'+current).getdata())) 54 | return np.array(train_images) 55 | 56 | y = targets(files) 57 | print "Fetching Data. Please wait......" 58 | x = data(files) 59 | print "Fetching Complete." 60 | 61 | 62 | x = np.reshape(x, (3408, 48, 48, 1)) 63 | #from sklearn.cross_validation import train_test_split 64 | #x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.25, random_state =124) 65 | from keras.utils import np_utils 66 | y = np_utils.to_categorical(y) 67 | #y_test = np_utils.to_categorical(y_test) 68 | #x_train = np.reshape(x_train, (2556, 48, 48, 1)) 69 | #x_test = np.reshape(x_test, (852, 48, 48, 1)) 70 | #print x.shape 71 | #print y.shape 72 | x_train = x[0:2556, :, :, :] 73 | x_test = x[2556:, :, :, :] 74 | y_train = y[0:2556, :] 75 | y_test = y[2556:, :] 76 | #print x_train.shape 77 | #print y_train.shape 78 | #print x_test.shape 79 | #print y_test.shape 80 | from keras.models import load_model 81 | print "Loading Trained CNN Model" 82 | model= load_model('my_model_360_iter_batch_100.h5') 83 | print "Loading Complete" 84 | labels = model.predict_classes(x_test) 85 | originals = np_utils.categorical_probas_to_classes(y_test) 86 | 87 | #print labels.shape 88 | #print originals.shape 89 | 90 | #print len(labels), np.sum(labels == originals) 91 | 92 | predictions = labels 93 | y_test = originals 94 | accuracy = np.sum(predictions == y_test) 95 | print '\n' 96 | print "Number of Correct Predictions : " +str(accuracy), 97 | print '\n' 98 | print "Total Number of Images : " + str(len(predictions)) 99 | percentage = accuracy * 1.0/len(predictions) 100 | from sklearn.metrics import confusion_matrix 101 | print '\n' 102 | print 'Accuracy : ' +str(percentage) 103 | 104 | print "\nConfusion Matrix\n" 105 | print confusion_matrix(y_test, predictions) 106 | 107 | ''' 108 | from keras.models import Sequential 109 | from keras.layers import Conv2D, AveragePooling2D, MaxPooling2D, Flatten, Dense, Dropout 110 | model = Sequential() 111 | model.add(Conv2D(10, 5, 5, activation = 'relu', input_shape = x.shape[1:])) 112 | model.add(AveragePooling2D(pool_size=(2, 2))) 113 | model.add(Conv2D(10, 5, 5, activation = 'relu')) 114 | model.add(MaxPooling2D(pool_size= (2, 2))) 115 | 116 | #model.add(Conv2D(10, 3, 3, activation = 'relu')) 117 | #model.add(AveragePooling2D(pool_size=(2, 2))) 118 | model.add(Conv2D(10, 3, 3, activation = 'relu')) 119 | model.add(MaxPooling2D(pool_size= (2, 2))) 120 | 121 | model.add(Flatten()) 122 | model.add(Dense(256, activation = 'relu')) 123 | model.add(Dropout(0.5)) 124 | model.add(Dense(128, activation = 'relu')) 125 | model.add(Dropout(0.5)) 126 | model.add(Dense(7, activation = 'softmax')) 127 | 128 | from keras import optimizers 129 | 130 | #ada = optimizers.adam(lr = 0.1, beta_1 = 0.9, beta_2 = 0.999, epsilon = 1e-8, decay= 0) 131 | #ada = optimizers.adam(lr = 0.005) 132 | model.compile(optimizer= 'adam' , loss = 'categorical_crossentropy', 133 | metrics= ['accuracy']) 134 | 135 | model.fit(x, y, batch_size= 100, nb_epoch= 50, validation_split=0.25) 136 | ''' 137 | -------------------------------------------------------------------------------- /CNN_lfw_test.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from PIL import Image 3 | import sys 4 | from keras.models import load_model 5 | #360 824 6 | """ 7 | 8 | This program takes the LFW test set 9 | folder as the input, and returns the 10 | number of correct predictions, 11 | total number of predictions, 12 | the accuracy percentage, 13 | and the confusion matrix for the 14 | LFW test set 15 | 16 | This program also creates a file called 17 | predictions_lfw.txt, which consists of 18 | the image name followed by the model's 19 | top three predictions. 20 | 21 | """ 22 | 23 | print "Loading trained CNN model" 24 | model = load_model('my_model_360_iter_batch_100.h5') 25 | print "Loading Complete" 26 | import os 27 | files = os.listdir(sys.argv[1]) 28 | 29 | tag_list = ['AN', 'SA', 'SU', 'HA', 'DI', 'FE', 'NE'] 30 | 31 | def targets(filename): 32 | targets = [] 33 | for f in filename: 34 | if tag_list[0] in f: 35 | targets.append(0) 36 | if tag_list[1] in f: 37 | targets.append(1) 38 | if tag_list[2] in f: 39 | targets.append(2) 40 | if tag_list[3] in f: 41 | targets.append(3) 42 | if tag_list[4] in f: 43 | targets.append(4) 44 | if tag_list[5] in f: 45 | targets.append(5) 46 | if tag_list[6] in f: 47 | targets.append(6) 48 | return np.array(targets) 49 | 50 | true_labels = targets(files) 51 | 52 | #for i in range(0, 7): 53 | #print np.sum(true_labels == i) 54 | predictions = [] 55 | 56 | #print files 57 | writer = open('predictions_lfw.txt', 'w') 58 | for f in files: 59 | #print f 60 | current = Image.open(sys.argv[1] + f) 61 | #current 62 | current = current.convert('L') 63 | current = current.resize((48, 48)) 64 | data = np.array(current.getdata()) 65 | #10 on top 66 | #11 on top 67 | #13 okay 68 | #15 okay 69 | data = np.reshape(data, (1, 48, 48, 1)) 70 | #from keras.models import load_model 71 | #360 82.4 72 | #model = load_model('my_model_360_iter_batch_100.h5') 73 | 74 | prediction = model.predict(data) 75 | order = np.argsort(prediction)[0,:] 76 | #print order 77 | #print prediction 78 | #first = np.argmax(prediction) 79 | #m keras.models import load_model 80 | #360 82.4 81 | #model = load_model('my_model_360_iter_batch_100.h5') 82 | 83 | 84 | #prediction[0, first] = 0 85 | #second = np.argmax(prediction) 86 | 87 | tag_dict = {0: 'Angry', 1: 'Sadness', 2: 'Surprise', 3: 'Happiness', 4: 'Disgust', 5: 'Fear', 6: 'Neutral'} 88 | 89 | prediction = prediction[0, :] 90 | writer.write(f + ' ' + tag_dict[order[-1]]+ ' ' + str( prediction[order[-1]]) + ' ' + tag_dict[order[-2]] + ' ' + str(prediction[order[-2]]) + ' ' + tag_dict[order[-3]] + ' ' +str( prediction[order[-3]])) 91 | writer.write('\n') 92 | predictions.append([order[-1], order[-2], order[-3]]) 93 | 94 | #print predictions 95 | writer.close() 96 | true_predictions = [] 97 | count = 0 98 | tot_count = 0 99 | for i in range(0, len(true_labels)): 100 | tot_count +=1 101 | if true_labels[i] in predictions[i]: 102 | count += 1 103 | true_predictions.append(true_labels[i]) 104 | else: 105 | true_predictions.append(predictions[i][2]) 106 | 107 | print "Number of Correct Predictions: " +str(count) 108 | print "Total number of Images: " +str(tot_count) 109 | #print count, tot_count 110 | accuracy = 1.0 * count / tot_count 111 | print "Accuracy: " +str(accuracy) 112 | from sklearn.metrics import confusion_matrix 113 | print "\nConfusion Matrix\n" 114 | print confusion_matrix(true_labels, true_predictions) 115 | -------------------------------------------------------------------------------- /CNN_single_image_test.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from PIL import Image 3 | import sys 4 | 5 | """ 6 | This program takes a single image as its input 7 | and returns the top three predicted emotions 8 | """ 9 | current = Image.open(sys.argv[1]) 10 | 11 | #current 12 | current = current.convert('L') 13 | current = current.resize((48, 48)) 14 | data = np.array(current.getdata()) 15 | #10 on top 16 | #11 on top 17 | #13 okay 18 | #15 okay 19 | data = np.reshape(data, (1, 48, 48, 1)) 20 | from keras.models import load_model 21 | #360 824 22 | model = load_model('my_model_360_iter_batch_100.h5') 23 | 24 | prediction = model.predict(data) 25 | order = np.argsort(prediction)[0,:] 26 | #print order 27 | #print prediction 28 | #first = np.argmax(prediction) 29 | #prediction[0, first] = 0 30 | #second = np.argmax(prediction) 31 | 32 | tag_dict = {0: 'Angry', 1: 'Sadness', 2: 'Surprise', 3: 'Happiness', 4: 'Disgust', 5: 'Fear', 6: 'Neutral'} 33 | 34 | #print tag_dict 35 | print "First: " + tag_dict[order[-1]] 36 | print "Second: " +tag_dict[order[-2]] 37 | print "Third: " +tag_dict[order[-3]] 38 | -------------------------------------------------------------------------------- /CNN_train.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Sat Apr 15 16:27:02 2017 5 | 6 | @author: prudhvi 7 | """ 8 | 9 | """ 10 | The program trains the CNN 11 | on the Jaffe training set. 12 | 13 | The number of iterations can be increased, 14 | but as default, this runs for 5 iterations on a 15 | 20% divided validation set. 16 | 17 | We trained this for around 400 iterations, 18 | 20 epochs at a time. Each epoch takes 75 seconds 19 | 20 | Arguments: 21 | 22 | first : Number of epochs 23 | """ 24 | 25 | import numpy as np 26 | import os 27 | import sys 28 | from PIL import Image 29 | np.random.rand(2) 30 | #from keras.layers import Conv1D 31 | files = os.listdir('aug_data_64_by_48') 32 | 33 | n_iter = int(sys.argv[1]) 34 | tag_list = ['AN', 'SA', 'SU', 'HA', 'DI', 'FE', 'NE'] 35 | 36 | def targets(filename): 37 | targets = [] 38 | for f in filename: 39 | if tag_list[0] in f: 40 | targets.append(0) 41 | if tag_list[1] in f: 42 | targets.append(1) 43 | if tag_list[2] in f: 44 | targets.append(2) 45 | if tag_list[3] in f: 46 | targets.append(3) 47 | if tag_list[4] in f: 48 | targets.append(4) 49 | if tag_list[5] in f: 50 | targets.append(5) 51 | if tag_list[6] in f: 52 | targets.append(6) 53 | return np.array(targets) 54 | 55 | 56 | def data(filename): 57 | train_images = [] 58 | for f in filename: 59 | current = f 60 | train_images.append(np.array(Image.open('aug_data_64_by_48/'+current).getdata())) 61 | return np.array(train_images) 62 | 63 | y = targets(files) 64 | print "Fetching Data. Please wait......" 65 | x = data(files) 66 | print "Fetching Complete." 67 | 68 | 69 | x = np.reshape(x, (3408, 48, 48, 1)) 70 | #from sklearn.cross_validation import train_test_split 71 | #x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.25, random_state =124) 72 | from keras.utils import np_utils 73 | y = np_utils.to_categorical(y) 74 | #y_test = np_utils.to_categorical(y_test) 75 | #x_train = np.reshape(x_train, (2556, 48, 48, 1)) 76 | #x_test = np.reshape(x_test, (852, 48, 48, 1)) 77 | #print x.shape 78 | #print y.shape 79 | x_train = x[0:2556, :, :, :] 80 | x_test = x[2556:, :, :, :] 81 | y_train = y[0:2556, :] 82 | y_test = y[2556:, :] 83 | 84 | #print "\nConfusion Matrix\n" 85 | #print confusion_matrix(y_test, predictions) 86 | from keras.models import Sequential 87 | from keras.layers import Conv2D, AveragePooling2D, MaxPooling2D, Flatten, Dense, Dropout 88 | model = Sequential() 89 | model.add(Conv2D(10, 5, 5, activation = 'relu', input_shape = x.shape[1:])) 90 | model.add(AveragePooling2D(pool_size=(2, 2))) 91 | model.add(Conv2D(10, 5, 5, activation = 'relu')) 92 | model.add(MaxPooling2D(pool_size= (2, 2))) 93 | 94 | #model.add(Conv2D(10, 3, 3, activation = 'relu')) 95 | #model.add(AveragePooling2D(pool_size=(2, 2))) 96 | model.add(Conv2D(10, 3, 3, activation = 'relu')) 97 | model.add(MaxPooling2D(pool_size= (2, 2))) 98 | 99 | model.add(Flatten()) 100 | model.add(Dense(256, activation = 'relu')) 101 | model.add(Dropout(0.5)) 102 | model.add(Dense(128, activation = 'relu')) 103 | model.add(Dropout(0.5)) 104 | model.add(Dense(7, activation = 'softmax')) 105 | 106 | 107 | #ada = optimizers.adam(lr = 0.1, beta_1 = 0.9, beta_2 = 0.999, epsilon = 1e-8, decay= 0) 108 | #ada = optimizers.adam(lr = 0.005) 109 | model.compile(optimizer= 'adam' , loss = 'categorical_crossentropy', 110 | metrics= ['accuracy']) 111 | 112 | model.fit(x_train, y_train, batch_size= 100, nb_epoch= n_iter, validation_split=0.2) 113 | model.save("CNN_Jaffe_model_" + str(n_iter) + "_epoch.h5") 114 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Facial-Emotion-Detection 2 | This work showcases two independent methods for recognizing emotions from faces. The first method using representational autoencoder units, a fairly original idea, to classify an image among one of the seven different emotions. The second method uses a 8-layer convolutional neural network which has an original and unique design, and was developed from scratch. The models were trained on the JAFFE dataset, and we tested on a seperate JAFFE test set and a subjectively labeled Labeled Faces in the Wild data set. 3 | 4 | Link to the paper: https://arxiv.org/abs/1706.01509 5 | -------------------------------------------------------------------------------- /augment_jaffe_data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Sat Apr 15 16:07:48 2017 5 | 6 | @author: prudhvi 7 | """ 8 | 9 | """ 10 | 11 | Takes a dataset of 64 x 64 sized images 12 | and returns 48 x 48 patches of each image 13 | 14 | Arguments: 15 | 16 | First: train_files.txt (or) test_files.txt 17 | 18 | Be sure to create a new directory 19 | of the name given below 20 | before executing this program 21 | """ 22 | 23 | 24 | import numpy as np 25 | from PIL import Image 26 | import os 27 | from scipy.misc import imsave 28 | import sys 29 | 30 | files = sys.argv[1] 31 | filenames = open(files, 'r').read().splitlines() 32 | 33 | for i in range(0, len(filenames)): 34 | print i 35 | current = filenames[i] 36 | if current != '.DS_Store': 37 | image = np.array((Image.open('resized_JAFFE_data_64_by_64/' + current)).getdata()) 38 | image = np.reshape(image, (64, 64)) 39 | for i in range(0, 16): 40 | if files == 'train_files.txt': 41 | imsave('aug_data_64_by_48/' + str(i) + '_' + current, image[i:i+48, i:i+48]) 42 | #imsave('aug_train/' + str(i) + '_' + current, image[i:i+48, i:i+48]) 43 | 44 | else: 45 | #imsave('aug_test/' + str(i) + '_' + current, image[i:i+48, i:i+48]) 46 | 47 | imsave('aug_test_data_64_by_48/' + str(i) + '_' + current, image[i:i+48, i:i+48]) 48 | 49 | -------------------------------------------------------------------------------- /autoencoder_train_appender.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pickle 3 | import os 4 | import sys 5 | 6 | """ 7 | 8 | This program appends all the representations of 9 | emotions into one file 10 | 11 | Arguments: 12 | 13 | first: Number of layers (one/two) 14 | second: Number of hidden units (300/500) 15 | 16 | PLEASE MAKE SURE TO RUN THIS BEFORE RUNNING 17 | THE AUTOENCODER TEST FILES 18 | """ 19 | 20 | layers = sys.argv[1] 21 | hu = sys.argv[2] 22 | #data = os.listdir('two_layer_ae_500_hn_rep/') 23 | data = os.listdir(str(layers) + '_layer_ae_' +str(hu) +'_hn_rep/') 24 | 25 | new_dict = {} 26 | 27 | emotions = ['AN', 'SA', 'SU', 'HA', 'DI', 'FE', 'NE'] 28 | 29 | for e in emotions: 30 | for d in data: 31 | if e in d: 32 | f = open(layers+'_layer_ae_'+ hu + '_hn_rep/' +d, 'rb') 33 | new_dict[e] = pickle.load(f) 34 | 35 | dict_write = open(layers + '_layer_ae_' + hu + '_hn_rep/trained_' +hu + '_representations.txt', 'wb') 36 | pickle.dump(new_dict, dict_write) 37 | dict_write.close() 38 | 39 | 40 | -------------------------------------------------------------------------------- /my_model_360_iter_batch_100.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PrudhviRaj12/Facial-Emotion-Detection-Using-Convolutional-Neural-Networks-and-Representational-Autoencoder-Units/f7fe3c11dc67153d566831ee196058084b7815b8/my_model_360_iter_batch_100.h5 -------------------------------------------------------------------------------- /one_layer_autoencoder_test.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pickle 3 | import os 4 | from PIL import Image 5 | from scipy.spatial.distance import cosine, euclidean, correlation 6 | import operator 7 | import sys 8 | 9 | """ 10 | 11 | The program is used for testing the 12 | shallow autoencoder network. 13 | 14 | This takes three arguments. 15 | 16 | first: dataset (jaffe for JAFFE test set, and lfw for LFW data set) 17 | second: Number of Hidden Layers (300/500) 18 | third: Number of iterations for each example (100 was giving the best results) 19 | 20 | Once it takes all the images, 21 | it converts them into 300/500 dimensional 22 | space and uses cosine distance as a metric 23 | to find the distance between two emotions. 24 | 25 | """ 26 | np.random.rand(30) 27 | c = sys.argv[1] 28 | hu = sys.argv[2] 29 | if c == 'lfw': 30 | test = os.listdir('lfw2/') 31 | elif c == 'jaffe': 32 | test = open('test_files.txt', 'r').read().split() 33 | 34 | from keras.models import Sequential, Model 35 | from keras.layers import Input, Dense, Activation 36 | from keras import backend as K 37 | #data = np.array(list(Image.open('resized_JAFFE_data_64_by_64/' +test[0]).getdata())) 38 | train_dict = pickle.load(open('one_layer_ae_' + str(hu) + '_hn_rep/trained_' +str(hu)+ '_representations.txt', 'rb')) 39 | 40 | tag_list = ['AN', 'SA', 'SU', 'HA', 'DI', 'FE', 'NE'] 41 | targets_list = [] 42 | for t in test: 43 | for tag in tag_list: 44 | if tag in t: 45 | targets_list.append(tag) 46 | 47 | 48 | #print len(targets_list) 49 | #print targets_list 50 | #print data.shape 51 | #print test[0] 52 | #print e 53 | 54 | hidden_units = int(hu) 55 | n_iter = int(sys.argv[3]) 56 | #data = np.reshape(data, (1, 4096)) 57 | np.random.rand(30) 58 | inputs = Input(shape = (4096, )) 59 | encoder = Dense(hidden_units)(inputs) 60 | decoder = Dense(4096)(encoder) 61 | model = Model(input = inputs, output = decoder) 62 | 63 | model.compile(optimizer = 'adam', loss = 'mse') 64 | 65 | a = [] 66 | b = [] 67 | for t in range(len(test)): 68 | print t 69 | if c == 'lfw': 70 | im = Image.open('lfw2/' + test[t]) 71 | im = im.convert('L') 72 | data= im.resize((64, 64)) 73 | data = np.array(data.getdata()) 74 | elif c == 'jaffe': 75 | data = np.array(list(Image.open('resized_JAFFE_data_64_by_64/' +test[t]).getdata())) 76 | data = np.reshape(data, (1, 4096)) 77 | model.fit(data, data, nb_epoch = n_iter) 78 | 79 | #from keras import backend as K 80 | encoder_func = K.function([model.layers[0].input], [model.layers[1].output]) 81 | encoder_output = np.array(encoder_func([data])) 82 | 83 | hidden_rep = encoder_output[0, 0, :] 84 | 85 | #print hidden_rep 86 | 87 | #train_dict = pickle.load(open('one_layer_ae_300_hn_rep/trained_300_representations.txt', 'rb')) 88 | 89 | new_dict = {} 90 | 91 | for p in train_dict: 92 | new_dict[p] = cosine(hidden_rep,train_dict[p]) 93 | 94 | order = sorted(new_dict.items(), key=operator.itemgetter(1)) 95 | 96 | a.append((order[0][0], order[1][0], order[2][0])) 97 | 98 | #print a 99 | #print targets_list 100 | 101 | a = np.array(a) 102 | targets_list = np.array(targets_list) 103 | 104 | count = 0 105 | for t in range(len(targets_list)): 106 | if targets_list[t] in a[t]: 107 | count +=1 108 | 109 | print "Number of Correct Predictions (Top 2): " + str(count) 110 | print "Total Number of Images: " +str(len(targets_list)) 111 | 112 | print "Accuracy : " + str((1.0 * count)/len(targets_list)) 113 | #print np.sum(a == targets_list) 114 | #print len(targets_list) 115 | -------------------------------------------------------------------------------- /one_layer_autoencoder_train.py: -------------------------------------------------------------------------------- 1 | ''' 2 | References: 3 | 4 | 1)https://keras.io/getting-started/faq/#how-can-i-obtain-the-output-of-an-intermediate-layer 5 | ''' 6 | 7 | """ 8 | 9 | The program is for the shallow 10 | autoencoder network. 11 | 12 | The strucutre of this autoencoder is 13 | 4096 - 300/500 - 4096. 14 | 15 | Arguments: 16 | 17 | First: Emotion - [AN, SA, SU, NE, FE, DI, NE] 18 | Second: Number of Iterations 19 | Third: Number of Hidden Units - 300 (or) 500 20 | 21 | For each given emotion, this program creates a condensed 22 | representation that is said to capture some unique 23 | information about that particular emotion. 24 | 25 | The loads the train_files.txt file, which consists of 26 | the filenames of the training data and then collects each 27 | image from the directory. 28 | 29 | We used pickle to save all these hidden representations. 30 | 31 | MAKE SURE TO RUN THE AUTOENCODER APPENDER WITH CORRECT 32 | ARGUMENTS BEFORE TESTING 33 | """ 34 | import pickle 35 | import sys 36 | from keras.models import Sequential, Model 37 | from keras.layers import Input, Dense, Activation 38 | import numpy as np 39 | from scipy.misc import imread, imsave 40 | from PIL import Image 41 | import os 42 | from sklearn.model_selection import train_test_split 43 | np.random.seed(30) 44 | 45 | new_files = [] 46 | #files = os.listdir('resized_JAFFE_data_64_by_64/') 47 | files = open('train_files.txt', 'r').read().split() 48 | 49 | #print f 50 | #print e 51 | emotion = str(sys.argv[1]) 52 | n_iter = int(sys.argv[2]) 53 | hidden_units = int(sys.argv[3]) 54 | print n_iter 55 | print emotion 56 | #print e 57 | for f in files: 58 | if emotion in f: 59 | new_files.append(f) 60 | 61 | #print new_files 62 | print len(new_files) 63 | print np.array(new_files).shape 64 | #print e 65 | #x, y = train_test_split(files, test_size = 0.25, random_state = 12) 66 | #print len(x) 67 | #print len(y) 68 | #print y 69 | #print e 70 | data = [] 71 | 72 | #print new_files 73 | #print e 74 | for f in new_files: 75 | print f 76 | im = np.array(list(Image.open('resized_JAFFE_data_64_by_64/' +f).getdata()))#imread("/resized_JAFFE_data_64_by_64/TM.NE1.177.tiff") 77 | #data = np.concatentate((data, im)) 78 | data.append(im) 79 | #img2 = np.reshape(np.array(im), (1, 4096)) 80 | #data = open('test_data.txt', 'r').read().split() 81 | #data = np.array(data).astype(np.int) 82 | #data = np.reshape(data, (1, 4096)) 83 | 84 | print np.array(data).shape 85 | #print img2.shape 86 | #print data.shape 87 | #print np.concatenate((data, img2)).shape 88 | #print data.shape 89 | #print e 90 | #data = np.concatenate((data, img2)) 91 | 92 | data = np.array(data) 93 | inputs = Input(shape = (4096, )) 94 | encoder = Dense(hidden_units)(inputs) 95 | decoder = Dense(4096)(encoder) 96 | model = Model(input = inputs, output = decoder) 97 | 98 | model.compile(optimizer = 'adam', loss = 'mse') 99 | 100 | model.fit(data, data, nb_epoch = n_iter) 101 | 102 | from keras import backend as K 103 | encoder_func = K.function([model.layers[0].input], [model.layers[1].output]) 104 | encoder_output = np.array(encoder_func([data])) 105 | 106 | hidden_rep = encoder_output[0, 0, :] 107 | 108 | reps = open('one_layer_ae_'+str(hidden_units)+'_hn_rep/' + emotion + '_' + str(hidden_units) + '_rep.txt', 'wb') 109 | hidden_rep = np.array(hidden_rep) 110 | 111 | pickle.dump(hidden_rep, reps) 112 | reps.close() 113 | 114 | weights = np.array(((model.layers[1]).get_weights())[0]) 115 | 116 | new_file = open('weights.txt', 'wb') 117 | pickle.dump(weights, new_file) 118 | new_file.close() 119 | 120 | #print data 121 | val = model.predict(data) 122 | #print emotion 123 | #print np.max(hidden_rep), np.min(hidden_rep) 124 | #print hidden_rep.shape 125 | #print val.shape 126 | #print data.shape 127 | #print e 128 | #imsave('original.jpg', np.reshape(data[0, :], (64, 64))) 129 | #imsave('predicted.jpg', np.reshape(val[0, :], (64, 64))) 130 | #print np.array(model.predict(data)).shape 131 | 132 | 133 | 134 | #new = open('AN_10_rep.txt', 'rb') 135 | #print pickle.load(new) 136 | ''' 137 | model = Sequential() 138 | model.add(Dense(30, input_dim = 4096, init = 'uniform')) 139 | model.add(Activation('linear')) 140 | model.add(Dense(4096, init = data[0])) 141 | model.add(Activation('linear')) 142 | ''' 143 | -------------------------------------------------------------------------------- /predictions_lfw.txt: -------------------------------------------------------------------------------- 1 | Tim_Chapman_0002_NE.jpg Neutral 0.802869 Surprise 0.197112 Sadness 1.37691e-05 2 | Valerie_Harper_0002_SU.jpg Fear 1.0 Surprise 1.23353e-21 Disgust 7.90957e-29 3 | Jennifer_Aniston_0011.jpg_NE Neutral 0.975515 Happiness 0.0174839 Surprise 0.0049558 4 | Wes_Craven_0001_NE.jpg Neutral 0.520305 Happiness 0.368612 Sadness 0.0999579 5 | Sourav_Ganguly_0002_DI.jpg Fear 0.999527 Disgust 0.000470793 Sadness 1.99569e-06 6 | Tracy_McGrady_0001_SA.jpg Happiness 0.999864 Neutral 8.30882e-05 Sadness 5.06896e-05 7 | Aaron_Sorkin_0002_HA.jpg Sadness 0.949495 Happiness 0.032987 Fear 0.0168283 8 | Tiger_Woods_0004_SA.jpg Fear 0.628598 Sadness 0.32956 Surprise 0.0417534 9 | Jackie_Chan_0006_FE.jpg Sadness 0.999677 Angry 0.000222926 Disgust 7.11926e-05 10 | Nicole_Parker_0001_SA.jpg Sadness 0.818709 Surprise 0.0840639 Fear 0.0564731 11 | Alicia_Witt_0001_HA.jpg Sadness 0.735404 Fear 0.20031 Happiness 0.0589806 12 | Sally_Field_0003_DI.jpg Fear 1.0 Disgust 1.46116e-20 Sadness 1.17105e-33 13 | Angela_Bassett_0001_HA.jpg Fear 0.998643 Disgust 0.000845052 Happiness 0.000214688 14 | Pamela_Anderson_0002_NE.jpg Neutral 0.834296 Happiness 0.135972 Sadness 0.0220147 15 | Gwyneth_Paltrow_0003_HA.jpg Happiness 0.604956 Fear 0.32861 Surprise 0.062182 16 | Sourav_Ganguly_0005_HA.jpg Fear 0.998998 Sadness 0.000987746 Disgust 1.40265e-05 17 | Jamie_Carey_0001_HA.jpg Sadness 0.999157 Happiness 0.000598565 Fear 0.000146569 18 | Tom_Scully_0001_SA.jpg Fear 1.0 Surprise 8.1024e-14 Disgust 1.99148e-21 19 | Harrison_Ford_0010_HA.jpg Happiness 0.960344 Fear 0.0230736 Sadness 0.0106026 20 | Jodie_Foster_0001_HA.jpg Sadness 0.554195 Fear 0.254912 Happiness 0.189529 21 | George_Lucas_0001_SA.jpg Fear 1.0 Disgust 1.17466e-13 Sadness 1.09613e-14 22 | Ain_Seppik_0001_NE.jpg Happiness 0.999999 Fear 5.29268e-07 Sadness 4.83089e-08 23 | Howard_Smith_0001_AN.jpg Disgust 0.575893 Angry 0.176982 Fear 0.150696 24 | Nicholoas_DiMarzio_0001_HA.jpg Surprise 0.999275 Fear 0.000397947 Sadness 0.000326677 25 | Robert_Douglas_0001_SA.jpg Sadness 0.999777 Fear 0.000175037 Surprise 3.27589e-05 26 | Ben_Affleck_0005_HA.jpg Neutral 0.98692 Happiness 0.00756402 Surprise 0.00420098 27 | Jacques_Kallis_0001_SA.jpg Happiness 1.0 Angry 3.03842e-10 Sadness 2.72627e-11 28 | Roh_Moo-hyun_0026_DI.jpg Fear 1.0 Sadness 5.08097e-09 Angry 1.05526e-12 29 | Hrithik_Roshan_0001_SA.jpg Sadness 0.999998 Fear 1.83551e-06 Angry 3.66706e-07 30 | Tiger_Woods_0007_FE.jpg Happiness 1.0 Disgust 1.0361e-08 Angry 2.20238e-09 31 | Ana_Guevara_0006_DI.jpg Disgust 0.996038 Sadness 0.0030725 Fear 0.000860926 32 | Sarah_Michelle_Gellar_0001_HA.jpg Happiness 0.999492 Neutral 0.000432281 Surprise 5.45896e-05 33 | Jeffrey_Archer_0001_SA.jpg Sadness 0.953701 Happiness 0.0276916 Fear 0.0173002 34 | Nathan_Lane_0001_SU.jpg Disgust 0.639028 Angry 0.360971 Fear 1.64469e-06 35 | Al_Gore_0002_AN.jpg Happiness 0.999999 Sadness 4.29989e-07 Neutral 6.56551e-08 36 | Tiger_Woods_0013_SA.jpg Happiness 0.889739 Sadness 0.10785 Neutral 0.00129628 37 | Florencia_Kirchner_0001_SA.jpg Fear 0.975933 Sadness 0.0240644 Disgust 2.64809e-06 38 | Gwyneth_Paltrow_0001_NE.jpg Fear 0.999974 Disgust 2.60952e-05 Sadness 1.17926e-35 39 | Denzel_Washington_0004_NE.jpg Fear 1.0 Disgust 1.72462e-24 Surprise 1.2639e-25 40 | Gianni_Agnelli_0001_AN.jpg Happiness 0.964132 Surprise 0.0289015 Neutral 0.00622548 41 | Tiger_Woods_0010_HA.jpg Happiness 0.962652 Fear 0.0344618 Sadness 0.00288576 42 | Harbhajan_Singh_0001_HA.jpg Happiness 0.999994 Sadness 6.45349e-06 Fear 4.93416e-08 43 | Dirk_Kempthorne_0001_DI.jpg Disgust 1.0 Fear 6.61303e-16 Angry 3.44413e-18 44 | Jeff_Van_Gundy_0001_AN.jpg Fear 0.98251 Disgust 0.0150126 Sadness 0.00247789 45 | Adelina_Avila_0001_AN.jpg Fear 0.967995 Sadness 0.0320023 Disgust 2.46009e-06 46 | Alastair_Campbell_0001_AN.jpg Fear 0.998538 Surprise 0.000778113 Sadness 0.000680981 47 | Tina_Fey_0001_SU.jpg Neutral 0.999838 Surprise 0.000135859 Sadness 2.37607e-05 48 | Alan_Zemaitis_0001_AN.jpg Happiness 0.998781 Sadness 0.000565114 Fear 0.000332138 49 | Bill_Paxton_0004_SU.jpg Fear 1.0 Disgust 6.02488e-41 Sadness 2.8026e-45 50 | Gary_Paer_0001_NE.jpg Surprise 0.972126 Fear 0.0278736 Sadness 1.02626e-22 51 | Ruth_Pearce_0001_FE.jpg Sadness 0.982596 Fear 0.0170947 Happiness 0.000174385 52 | Trista_Rehn_0001_DI.jpg Fear 1.0 Sadness 1.80418e-07 Disgust 6.96938e-12 53 | Alberto_Fujimori_0002_NE.jpg Happiness 0.999047 Sadness 0.000951048 Angry 1.62014e-06 54 | Aaron_Pena_0001_DI.jpg Fear 0.893319 Sadness 0.0563479 Disgust 0.0465747 55 | Sachin_Tendulkar_0001_SA.jpg Sadness 0.545232 Surprise 0.337846 Neutral 0.0733332 56 | Adriana_Perez_Navarro_0001_HA.jpg Fear 1.0 Surprise 8.78933e-11 Sadness 2.24109e-12 57 | Will_Ofenheusle_0001_NE.jpg Fear 0.999736 Sadness 0.00025969 Happiness 3.35025e-06 58 | Theresa_May_0002_SA.jpg Sadness 0.879706 Fear 0.0886048 Happiness 0.0232492 59 | Cameron_Diaz_0005_HA.jpg Neutral 0.986592 Happiness 0.00921785 Surprise 0.00339636 60 | Courtney_Cox_0001_FE.jpg Fear 0.866311 Surprise 0.133689 Sadness 3.53191e-11 61 | Tiger_Woods_0022_SA.jpg Happiness 0.989924 Sadness 0.00800407 Angry 0.00190854 62 | Tiger_Woods_0001_SA.jpg Disgust 0.999999 Sadness 4.45301e-07 Angry 1.32584e-07 63 | Sarah_Michelle_Gellar_0002_DI.jpg Neutral 0.987391 Happiness 0.00880399 Surprise 0.00312507 64 | Zico_0003_HA.jpg Sadness 0.999997 Fear 2.41003e-06 Happiness 4.7024e-07 65 | Tom_Reilly_0003_SA.jpg Sadness 0.746478 Neutral 0.121061 Happiness 0.0690922 66 | Adrianna_Zuzic_0001_SA.jpg Happiness 0.978611 Angry 0.0177784 Sadness 0.00360714 67 | Alejandro_Gonzalez_Inarritu_0001_HA.jpg Happiness 1.0 Disgust 1.22542e-07 Fear 9.53384e-08 68 | Steve_Austin_0001_AN.jpg Fear 1.0 Disgust 1.42926e-16 Angry 2.99791e-30 69 | Aaron_Guiel_0001_HA.jpg Happiness 0.622632 Sadness 0.37273 Fear 0.0029791 70 | Alain_Ducasse_0001_HA.jpg Happiness 0.97271 Sadness 0.0250013 Neutral 0.00198181 71 | Tatjana_Gsell_0001_DI.jpg Fear 0.852958 Disgust 0.147042 Angry 4.41023e-23 72 | Douglas_Faneuil_0001_HA.jpg Sadness 0.999715 Fear 0.000285122 Happiness 1.72614e-07 73 | Tian_Liang_0001_NE.jpg Fear 1.0 Disgust 2.48292e-23 Sadness 6.16571e-44 74 | Sarah_Michelle_Gellar_0003_HA.jpg Happiness 0.983476 Sadness 0.0160729 Angry 0.000311194 75 | Todd_Wike_0001_SA.jpg Happiness 1.0 Sadness 3.84701e-12 Angry 2.93765e-12 76 | Harrison_Ford_0012_SU.jpg Neutral 0.986388 Happiness 0.00947734 Surprise 0.00342081 77 | Ian_McKellen_0001_HA.jpg Neutral 0.75732 Sadness 0.124703 Happiness 0.0618008 78 | Sigourney_Weaver_0001_NE.jpg Neutral 0.778578 Sadness 0.173729 Happiness 0.0359103 79 | Wilfredo_Moreno_0001_AN.jpg Happiness 0.992076 Sadness 0.0071676 Angry 0.000643112 80 | Heidi_Klum_0004_HA.jpg Happiness 0.875668 Neutral 0.0916886 Sadness 0.0294397 81 | Hideki_Matsui_0002_SA.jpg Sadness 0.99932 Fear 0.000679856 Disgust 5.82496e-07 82 | Ainsworth_Dyer_0001_NE.jpg Sadness 0.969292 Happiness 0.0298433 Fear 0.000388276 83 | Vanessa_Williams_0002_SU.jpg Surprise 0.643683 Sadness 0.27236 Neutral 0.060965 84 | Agnes_Bruckner_0001_AN.jpg Fear 0.996455 Sadness 0.00239451 Disgust 0.00114508 85 | Ahmed_Chalabi_0003_AN.jpg Sadness 0.939262 Fear 0.0604416 Happiness 0.000260206 86 | Ahmed_Ahmed_0001_HA.jpg Disgust 0.999817 Sadness 0.0001072 Angry 6.75912e-05 87 | Guy_Ritchie_0002_NE.jpg Fear 0.999997 Disgust 2.68466e-06 Happiness 3.19582e-08 88 | Bernard_Lord_0002_SA.jpg Fear 0.999947 Sadness 4.50085e-05 Disgust 7.71318e-06 89 | Bob_Graham_0003_AN.jpg Happiness 0.758579 Sadness 0.237704 Neutral 0.00358078 90 | Jeff_Van_Gundy_0002_SA.jpg Fear 0.999999 Disgust 6.88724e-07 Sadness 1.79838e-10 91 | Yuvraj_Singh_0001_HA.jpg Fear 1.0 Disgust 8.63766e-08 Sadness 4.1071e-09 92 | Franz_Fischler_0003_AN.jpg Happiness 0.996683 Sadness 0.00329025 Neutral 1.38293e-05 93 | George_Clooney_0004_HA.jpg Sadness 0.994873 Happiness 0.0045885 Fear 0.000298834 94 | Sue_Grafton_0001_NE.jpg Neutral 0.998807 Surprise 0.00117876 Sadness 1.331e-05 95 | Jacob_Frenkel_0001_FE.jpg Surprise 0.950488 Fear 0.0495121 Sadness 5.5976e-09 96 | Alan_Zemaitis_0001_NE.jpg Happiness 0.998781 Sadness 0.000565114 Fear 0.000332138 97 | Cate_Blanchett_0002_SU.jpg Surprise 0.986372 Fear 0.0136279 Sadness 1.92939e-07 98 | Ellen_DeGeneres_0002_AN.jpg Fear 0.414961 Sadness 0.307755 Disgust 0.277278 99 | Hilary_Duff_0003_HA.jpg Neutral 0.987391 Happiness 0.00880401 Surprise 0.00312507 100 | Turner_Gill_0001_HA.jpg Neutral 0.981558 Happiness 0.0112225 Surprise 0.0051493 101 | Carrie-Anne_Moss_0005_FE.jpg Fear 0.999998 Surprise 1.54715e-06 Sadness 8.28247e-24 102 | Jennifer_Aniston_0020_HA.jpg Happiness 0.876856 Neutral 0.118385 Surprise 0.00250702 103 | Jackie_Chan_0005_AN.jpg Sadness 0.677557 Angry 0.148248 Happiness 0.120221 104 | Winona_Ryder_0019_NE.jpg Surprise 0.987926 Sadness 0.00925568 Fear 0.0028177 105 | Sarah_Jessica_Parker_0003_SU.jpg Neutral 0.970369 Happiness 0.019129 Surprise 0.0084613 106 | -------------------------------------------------------------------------------- /test_files.txt: -------------------------------------------------------------------------------- 1 | NM.NE3.94.tiff 2 | KA.SA2.34.tiff 3 | TM.AN3.192.tiff 4 | KM.SU1.14.tiff 5 | KM.AN3.19.tiff 6 | KR.NE3.73.tiff 7 | NM.SA1.98.tiff 8 | KA.DI1.42.tiff 9 | MK.HA3.118.tiff 10 | YM.AN2.62.tiff 11 | KL.DI2.171.tiff 12 | NM.DI3.109.tiff 13 | NA.SA2.206.tiff 14 | YM.SA1.55.tiff 15 | KL.FE1.174.tiff 16 | KA.SU1.36.tiff 17 | NM.FE3.112.tiff 18 | TM.SA1.184.tiff 19 | KR.HA1.74.tiff 20 | NA.HA1.202.tiff 21 | TM.NE3.179.tiff 22 | TM.FE1.196.tiff 23 | KA.SU2.37.tiff 24 | MK.NE3.115.tiff 25 | KA.HA2.30.tiff 26 | KM.FE2.24.tiff 27 | NM.SU2.102.tiff 28 | UY.HA3.139.tiff 29 | KR.FE3.91.tiff 30 | MK.FE2.132.tiff 31 | UY.FE1.152.tiff 32 | YM.DI1.64.tiff 33 | UY.AN2.147.tiff 34 | KR.FE2.90.tiff 35 | TM.SA3.186.tiff 36 | YM.SU1.58.tiff 37 | UY.AN1.146.tiff 38 | TM.SU2.188.tiff 39 | NA.NE3.201.tiff 40 | NM.SA2.99.tiff 41 | KL.AN2.168.tiff 42 | KR.AN1.83.tiff 43 | KM.SA2.10.tiff 44 | NA.FE3.219.tiff 45 | MK.SA1.119.tiff 46 | KL.DI1.170.tiff 47 | YM.NE1.49.tiff 48 | KR.NE1.71.tiff 49 | NM.SA3.100.tiff 50 | KM.AN1.17.tiff 51 | TM.AN2.191.tiff 52 | NA.AN1.211.tiff 53 | KL.HA1.158.tiff 54 | KM.NE2.2.tiff 55 | -------------------------------------------------------------------------------- /train_files.txt: -------------------------------------------------------------------------------- 1 | TM.HA2.181.tiff 2 | KM.DI3.22.tiff 3 | NA.DI3.216.tiff 4 | NM.HA1.95.tiff 5 | MK.HA1.116.tiff 6 | KM.SA5.13.tiff 7 | NM.FE2.111.tiff 8 | NM.SU3.103.tiff 9 | MK.FE3.133.tiff 10 | NM.SU1.101.tiff 11 | KA.AN2.40.tiff 12 | TM.SA2.185.tiff 13 | NA.HA3.204.tiff 14 | TM.AN1.190.tiff 15 | NM.DI1.107.tiff 16 | NA.FE1.217.tiff 17 | NA.NE2.200.tiff 18 | KA.DI2.43.tiff 19 | KR.DI1.86.tiff 20 | KL.FE2.175.tiff 21 | KM.HA1.4.tiff 22 | KL.NE3.157.tiff 23 | NA.SU3.210.tiff 24 | MK.DI3.130.tiff 25 | UY.FE3.154.tiff 26 | MK.SA2.120.tiff 27 | KR.DI3.88.tiff 28 | NA.AN2.212.tiff 29 | NA.HA2.203.tiff 30 | TM.FE2.197.tiff 31 | KM.SU2.15.tiff 32 | NA.SA1.205.tiff 33 | NA.NE1.199.tiff 34 | KM.SA3.11.tiff 35 | KL.DI4.173.tiff 36 | NM.HA3.97.tiff 37 | YM.NE2.50.tiff 38 | KM.DI1.20.tiff 39 | TM.DI3.195.tiff 40 | KL.SA3.163.tiff 41 | KA.SA1.33.tiff 42 | TM.SU3.189.tiff 43 | KL.NE2.156.tiff 44 | TM.HA1.180.tiff 45 | NA.SA3.207.tiff 46 | UY.NE3.136.tiff 47 | KR.SU1.80.tiff 48 | MK.SA3.121.tiff 49 | KM.FE3.25.tiff 50 | KA.FE1.45.tiff 51 | KL.SU1.164.tiff 52 | KL.DI3.172.tiff 53 | KL.SA2.162.tiff 54 | MK.HA2.117.tiff 55 | KA.NE1.26.tiff 56 | KL.HA2.159.tiff 57 | KR.SA3.79.tiff 58 | KA.FE3.47.tiff 59 | KA.FE4.48.tiff 60 | UY.SA2.141.tiff 61 | KR.SA1.77.tiff 62 | KA.SA3.35.tiff 63 | KL.NE1.155.tiff 64 | KA.SU3.38.tiff 65 | UY.SU2.144.tiff 66 | KM.NE3.3.tiff 67 | NM.NE2.93.tiff 68 | YM.HA3.54.tiff 69 | KA.HA1.29.tiff 70 | MK.AN2.126.tiff 71 | YM.FE2.68.tiff 72 | MK.NE2.114.tiff 73 | KR.AN2.84.tiff 74 | KM.HA4.7.tiff 75 | YM.FE1.67.tiff 76 | TM.HA3.182.tiff 77 | MK.SU1.122.tiff 78 | YM.SA3.57.tiff 79 | MK.AN1.125.tiff 80 | YM.HA1.52.tiff 81 | NA.AN3.213.tiff 82 | KL.AN3.169.tiff 83 | TM.SU1.187.tiff 84 | UY.NE1.134.tiff 85 | NM.NE1.92.tiff 86 | KA.DI3.44.tiff 87 | TM.NE2.178.tiff 88 | TM.DI2.194.tiff 89 | KL.FE3.176.tiff 90 | KR.AN3.85.tiff 91 | MK.SU3.124.tiff 92 | KA.HA4.32.tiff 93 | UY.SU1.143.tiff 94 | NA.SU2.209.tiff 95 | KM.HA3.6.tiff 96 | KA.NE2.27.tiff 97 | KL.SA1.161.tiff 98 | KL.AN1.167.tiff 99 | MK.NE1.113.tiff 100 | UY.NE2.135.tiff 101 | MK.DI2.129.tiff 102 | NA.DI1.214.tiff 103 | MK.DI1.128.tiff 104 | NM.HA2.96.tiff 105 | UY.HA1.137.tiff 106 | NA.SU1.208.tiff 107 | MK.AN3.127.tiff 108 | YM.SU2.59.tiff 109 | NM.AN2.105.tiff 110 | YM.SA2.56.tiff 111 | YM.NE3.51.tiff 112 | KM.AN2.18.tiff 113 | TM.FE3.198.tiff 114 | YM.AN1.61.tiff 115 | KA.FE2.46.tiff 116 | UY.AN3.148.tiff 117 | KR.FE1.89.tiff 118 | NA.FE2.218.tiff 119 | TM.NE1.177.tiff 120 | YM.FE4.70.tiff 121 | KR.SA2.78.tiff 122 | KR.HA2.75.tiff 123 | YM.AN3.63.tiff 124 | UY.SA3.142.tiff 125 | UY.HA2.138.tiff 126 | NM.AN1.104.tiff 127 | UY.FE2.153.tiff 128 | KM.HA2.5.tiff 129 | KL.SU3.166.tiff 130 | NM.FE1.110.tiff 131 | KM.SA1.9.tiff 132 | KR.NE2.72.tiff 133 | YM.DI2.65.tiff 134 | UY.SA1.140.tiff 135 | UY.DI1.149.tiff 136 | YM.SU3.60.tiff 137 | NM.AN3.106.tiff 138 | KL.SU2.165.tiff 139 | YM.DI3.66.tiff 140 | KL.HA3.160.tiff 141 | KR.SU2.81.tiff 142 | MK.SU2.123.tiff 143 | KM.NE1.1.tiff 144 | NA.DI2.215.tiff 145 | KA.NE3.28.tiff 146 | UY.DI3.151.tiff 147 | KA.HA3.31.tiff 148 | MK.FE1.131.tiff 149 | KM.FE1.23.tiff 150 | KA.AN3.41.tiff 151 | UY.SU3.145.tiff 152 | UY.DI2.150.tiff 153 | KR.SU3.82.tiff 154 | YM.FE3.69.tiff 155 | TM.DI1.193.tiff 156 | YM.HA2.53.tiff 157 | KR.DI2.87.tiff 158 | KA.AN1.39.tiff 159 | KM.SU3.16.tiff 160 | -------------------------------------------------------------------------------- /train_test_splitter.py: -------------------------------------------------------------------------------- 1 | import os 2 | from sklearn.model_selection import train_test_split 3 | import numpy as np 4 | 5 | """ 6 | 7 | Splits files for autoencoder modules 8 | 9 | """ 10 | files = os.listdir('resized_JAFFE_data_64_by_64/') 11 | 12 | train_data, test_data = train_test_split(files, test_size = 0.25, \ 13 | random_state = 123) 14 | 15 | def counter(tag, files): 16 | count = 0 17 | for f in files: 18 | if tag in f: 19 | count +=1 20 | return count 21 | 22 | tag_list = ['AN', 'SA', 'SU', 'HA', 'DI', 'FE', 'NE'] 23 | 24 | for t in tag_list: 25 | print t, counter(t, train_data), counter(t, test_data) 26 | 27 | train_file = open('train_files.txt', 'w') 28 | for t in train_data: 29 | train_file.write(t + '\n') 30 | train_file.close() 31 | 32 | test_file = open('test_files.txt', 'w') 33 | for t in test_data: 34 | test_file.write(t + '\n') 35 | test_file.close() 36 | -------------------------------------------------------------------------------- /two_layer_autoencoder_test.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pickle 3 | import os 4 | from PIL import Image 5 | from scipy.spatial.distance import cosine, euclidean, correlation 6 | import operator 7 | import sys 8 | """ 9 | 10 | The program is used for testing the 11 | two layer dense autoencoder network. 12 | 13 | This takes three arguments. 14 | 15 | first: dataset (jaffe for JAFFE test set, and lfw for LFW data set) 16 | second: Number of Hidden Layers (300/500) 17 | third: Number of iterations for each example (100 was giving the best results) 18 | 19 | Once it takes all the images, 20 | it converts them into 300/500 dimensional 21 | space and uses cosine distance as a metric 22 | to find the distance between two emotions. 23 | 24 | """ 25 | np.random.rand(30) 26 | c = sys.argv[1] 27 | hu = sys.argv[2] 28 | if c == 'lfw': 29 | test = os.listdir('lfw2/') 30 | elif c == 'jaffe': 31 | test = open('test_files.txt', 'r').read().split() 32 | 33 | #np.random.rand(30) 34 | #test = os.listdir('lfw2/') 35 | #test = open('test_files.txt', 'r').read().split() 36 | 37 | from keras.models import Sequential, Model 38 | from keras.layers import Input, Dense, Activation 39 | from keras import backend as K 40 | #data = np.array(list(Image.open('resized_JAFFE_data_64_by_64/' +test[0]).getdata())) 41 | train_dict = pickle.load(open('two_layer_ae_' +str(hu) + '_hn_rep/trained_'+str(hu) +'_representations.txt', 'rb')) 42 | 43 | tag_list = ['AN', 'SA', 'SU', 'HA', 'DI', 'FE', 'NE'] 44 | targets_list = [] 45 | for t in test: 46 | for tag in tag_list: 47 | if tag in t: 48 | targets_list.append(tag) 49 | 50 | 51 | #print len(targets_list) 52 | #print targets_list 53 | #print data.shape 54 | #print test[0] 55 | #print e 56 | 57 | 58 | hidden_units = int(hu) 59 | ''' 60 | n_iter = 40 61 | #data = np.reshape(data, (1, 4096)) 62 | np.random.rand(30) 63 | inputs = Input(shape = (4096, )) 64 | encoder = Dense(hidden_units)(inputs) 65 | decoder = Dense(4096)(encoder) 66 | ''' 67 | h2 = 2800 68 | #hidden_units = 300 69 | n_iter = int(sys.argv[3]) 70 | #data = np.array(data) 71 | inputs = Input(shape = (4096, )) 72 | #encoder = Dense(hidden_units)(inputs) 73 | encoder = Dense(h2)(inputs) 74 | encoder_2 = Dense(hidden_units)(encoder) 75 | decoder_2 = Dense(h2)(encoder_2) 76 | decoder = Dense(4096)(decoder_2) 77 | #model = Model(input = inputs, output = decoder) 78 | 79 | 80 | model = Model(input = inputs, output = decoder) 81 | 82 | model.compile(optimizer = 'adam', loss = 'mse') 83 | 84 | a = [] 85 | b = [] 86 | for t in range(len(test)): 87 | print t 88 | if c == 'lfw': 89 | im = Image.open('lfw2/' + test[t]) 90 | im = im.convert('L') 91 | data= im.resize((64, 64)) 92 | data = np.array(data.getdata()) 93 | elif c == 'jaffe': 94 | data = np.array(list(Image.open('resized_JAFFE_data_64_by_64/' +test[t]).getdata())) 95 | data = np.reshape(data, (1, 4096)) 96 | model.fit(data, data, nb_epoch = n_iter) 97 | 98 | #from keras import backend as K 99 | encoder_func = K.function([model.layers[1].input], [model.layers[2].output]) 100 | encoder_output = np.array(encoder_func([data])) 101 | 102 | hidden_rep = encoder_output[0, 0, :] 103 | 104 | #print hidden_rep 105 | 106 | #train_dict = pickle.load(open('one_layer_ae_300_hn_rep/trained_300_representations.txt', 'rb')) 107 | 108 | new_dict = {} 109 | 110 | for p in train_dict: 111 | new_dict[p] = cosine(hidden_rep,train_dict[p]) 112 | 113 | order = sorted(new_dict.items(), key=operator.itemgetter(1), reverse = True) 114 | order_2 = sorted(new_dict.items(), key=operator.itemgetter(1)) 115 | #a.append((order_2[0][0], order_2[1][0]))#, order[2][0])) 116 | b.append((order_2[0][0], order_2[1][0], order_2[2][0])) 117 | 118 | #print a 119 | #print b 120 | #print targets_list 121 | 122 | #a = np.array(a) 123 | b = np.array(b) 124 | targets_list = np.array(targets_list) 125 | 126 | #count_a = 0 127 | count_b = 0 128 | for t in range(len(targets_list)): 129 | #if targets_list[t] in a[t]: 130 | # count_a +=1 131 | if targets_list[t] in b[t]: 132 | count_b +=1 133 | 134 | #print count_a 135 | 136 | print "Number of Correct Predictions (Top 2): " + str(count_b) 137 | print "Total Number of Predictions: " +str(len(targets_list)) 138 | 139 | print "Accuracy : " + str((1.0 * count_b)/len(targets_list)) 140 | #print np.sum(a == targets_list) 141 | #print len(targets_list) 142 | -------------------------------------------------------------------------------- /two_layer_autoencoder_train.py: -------------------------------------------------------------------------------- 1 | ''' 2 | References: 3 | 4 | 1)https://keras.io/getting-started/faq/#how-can-i-obtain-the-output-of-an-intermediate-layer 5 | ''' 6 | #import theano 7 | #theano.config.device = 'gpu' 8 | #theano.config.floatX = 'float32' 9 | """ 10 | The program is for the two layer dense 11 | autoencoder network. 12 | 13 | The strucutre of this autoencoder is 14 | 4096 - 2800 - 300/500 - 2800 - 4096. 15 | 16 | Arguments: 17 | 18 | First: Emotion - [AN, SA, SU, NE, FE, DI, NE] 19 | Second: Number of Iterations 20 | Third: Number of Hidden Units - 300 (or) 500 21 | 22 | For each given emotion, this program creates a condensed 23 | representation that is said to capture some unique 24 | information about that particular emotion. 25 | 26 | The loads the train_files.txt file, which consists of 27 | the filenames of the training data and then collects each 28 | image from the directory. 29 | 30 | We used pickle to save all these hidden representations. 31 | 32 | MAKE SURE TO RUN THE AUTOENCODER APPENDER WITH CORRECT 33 | ARGUMENTS BEFORE TESTING 34 | 35 | """ 36 | 37 | import pickle 38 | import sys 39 | from keras.models import Sequential, Model 40 | from keras.layers import Input, Dense, Activation 41 | import numpy as np 42 | from scipy.misc import imread, imsave 43 | from PIL import Image 44 | import os 45 | from sklearn.model_selection import train_test_split 46 | np.random.seed(30) 47 | 48 | new_files = [] 49 | #files = os.listdir('resized_JAFFE_data_64_by_64/') 50 | files = open('train_files.txt', 'r').read().split() 51 | 52 | #print f 53 | #print e 54 | emotion = str(sys.argv[1]) 55 | n_iter = int(sys.argv[2]) 56 | hidden_units = int(sys.argv[3]) 57 | print n_iter 58 | print emotion 59 | #print e 60 | for f in files: 61 | if emotion in f: 62 | new_files.append(f) 63 | 64 | #print new_files 65 | print len(new_files) 66 | print np.array(new_files).shape 67 | #print e 68 | #x, y = train_test_split(files, test_size = 0.25, random_state = 12) 69 | #print len(x) 70 | #print len(y) 71 | #print y 72 | #print e 73 | data = [] 74 | 75 | #print new_files 76 | #print e 77 | for f in new_files: 78 | print f 79 | im = np.array(list(Image.open('resized_JAFFE_data_64_by_64/' +f).getdata()))#imread("/resized_JAFFE_data_64_by_64/TM.NE1.177.tiff") 80 | #data = np.concatentate((data, im)) 81 | data.append(im) 82 | #img2 = np.reshape(np.array(im), (1, 4096)) 83 | #data = open('test_data.txt', 'r').read().split() 84 | #data = np.array(data).astype(np.int) 85 | #data = np.reshape(data, (1, 4096)) 86 | 87 | print np.array(data).shape 88 | #print img2.shape 89 | #print data.shape 90 | #print np.concatenate((data, img2)).shape 91 | #print data.shape 92 | #print e 93 | #data = np.concatenate((data, img2)) 94 | h2 = 2800 95 | data = np.array(data) 96 | inputs = Input(shape = (4096, )) 97 | #encoder = Dense(hidden_units)(inputs) 98 | encoder = Dense(h2)(inputs) 99 | encoder_2 = Dense(hidden_units)(encoder) 100 | decoder_2 = Dense(h2)(encoder_2) 101 | decoder = Dense(4096)(decoder_2) 102 | model = Model(input = inputs, output = decoder) 103 | 104 | model.compile(optimizer = 'adam', loss = 'mse') 105 | 106 | model.fit(data, data, nb_epoch = n_iter) 107 | 108 | from keras import backend as K 109 | encoder_func = K.function([model.layers[1].input], [model.layers[2].output]) 110 | encoder_output = np.array(encoder_func([data])) 111 | 112 | hidden_rep = encoder_output[0, 0, :] 113 | 114 | reps = open('two_layer_ae_' +str(hidden_units) +'_hn_rep/' + emotion + '_' + str(hidden_units) + '_reptoo.txt', 'wb') 115 | hidden_rep = np.array(hidden_rep) 116 | 117 | pickle.dump(hidden_rep, reps) 118 | reps.close() 119 | 120 | #print data 121 | #val = model.predict(data) 122 | #print emotion 123 | #print np.max(hidden_rep), np.min(hidden_rep) 124 | #print hidden_rep.shape 125 | #print val.shape 126 | #print data.shape 127 | #print e 128 | #imsave('original.jpg', np.reshape(data[0, :], (64, 64))) 129 | #imsave('predicted.jpg', np.reshape(val[0, :], (64, 64))) 130 | #print np.array(model.predict(data)).shape 131 | 132 | 133 | 134 | #new = open('AN_10_rep.txt', 'rb') 135 | #print pickle.load(new) 136 | ''' 137 | model = Sequential() 138 | model.add(Dense(30, input_dim = 4096, init = 'uniform')) 139 | model.add(Activation('linear')) 140 | model.add(Dense(4096, init = data[0])) 141 | model.add(Activation('linear')) 142 | ''' 143 | --------------------------------------------------------------------------------