├── .gitattributes ├── EEG ├── process_DEAP │ └── DEAP_EEG_example.py ├── process_EEG │ └── EEG_tool.py ├── process_MAHNOB_HCI │ └── MAHNOB_HCI_EEG_example.py └── process_ONLINE │ └── ONLINE_EEG_example.py ├── LICENSE ├── README.md ├── adaboost_fusion ├── process_DEAP │ └── DEAP_adboost_example.py ├── process_MAHNOB_HCI │ └── MANNOB_HCI_adaboost_example.py ├── process_ONLINE │ └── ONLINE_adaboost_example.py └── process_adaboost │ └── adaboost_tool.py ├── dataset └── fer2013 │ └── X_mean.npy ├── enum_fusion ├── process_DEAP │ └── DEAP_enum_example.py ├── process_MANHOB_HCI │ └── MANHOB_HCI_enum_example.py ├── process_ONLINE │ └── ONLINE_enum_example.py └── process_enum │ └── enum_tool.py ├── facial_expression ├── process_DEAP │ ├── DEAP_facial_expression_example.py │ └── __init__.py ├── process_MAHNOB_HCI │ └── MAHNOB_HCI_facial_expression_example.py.py ├── process_ONLINE │ └── ONLINE_facial_expression_example.py.py ├── process_facial_expression │ └── face_tool.py └── train_baseline_model_inFer2013 │ ├── base_accuray.png │ ├── base_loss.png │ ├── build_model.py │ ├── prepare_data.py │ └── train.py └── model └── CNN_expression_baseline.h5 /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /EEG/process_DEAP/DEAP_EEG_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Sun Sep 30 13:41:10 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | ''' 9 | This is an example of handle EEG data in DEAP dataset, whose EEG features have been preprocessed. 10 | the shape of 'EEG.npy' stored an array with shape (60, 85) 11 | 60 represents 60 seconds, the origin dataset is 128Hz and we get the average for each feature to avoid noise. 12 | 85 represents 85 feature for each second. The meaning of each feature was presented in paper. 13 | The DEAP dataset contains 40 trials for each subject. 14 | In this example, for each subject, 20 trials are for training whereas the other trials are for testing. 15 | This is an example of using 20 trials for training and the other for testing. 16 | ''' 17 | 18 | import numpy as np 19 | import sys 20 | sys.path.append('../process_EEG') 21 | import EEG_tool 22 | 23 | if __name__ == '__main__': 24 | ROOT_PATH = '../../dataset/DEAP/' 25 | 26 | for subject_id in range(1, 23): 27 | subject_path = ROOT_PATH + str(subject_id)+'/' 28 | EEG_model = EEG_tool.EEG_model() 29 | 30 | #random select 20 trial for training, the other trials for testing 31 | train_idxs_set = set(np.random.choice(np.arange(1, 41), size = 20, replace = False)) 32 | all_set = set(np.arange(1, 41)) 33 | test_idxs_set = all_set - train_idxs_set 34 | 35 | #training 36 | for trial_id in train_idxs_set: 37 | trial_path = subject_path + 'trial_' + str(trial_id) + '/' 38 | EEG_model.add_one_trial_data(trial_path, preprocessed = True) 39 | 40 | EEG_model.train() 41 | 42 | #testing 43 | acc_valence, acc_arousal = 0., 0. 44 | for trial_id in test_idxs_set: 45 | trial_path = subject_path + 'trial_' + str(trial_id) + '/' 46 | valence_correct, arousal_correct = EEG_model.predict_one_trial(trial_path, preprocessed=True) 47 | acc_valence += 1 if valence_correct else 0 48 | acc_arousal += 1 if arousal_correct else 0 49 | 50 | print (subject_id, acc_valence/20, acc_arousal/20) 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /EEG/process_EEG/EEG_tool.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Oct 2 12:55:02 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | from sklearn.svm.classes import SVC 9 | import numpy as np 10 | import pandas as pd 11 | import mne 12 | 13 | def extract_EEG_feature(raw_EEG_obj): 14 | ''' 15 | Extract PSD feature from raw EEG data 16 | Parameter: 17 | raw_EEG_obj: raw objects from mne library 18 | Rerturn: 19 | average_feature: extracted feature 20 | 21 | ''' 22 | 23 | #select 14 electrodes 24 | EEG_raw = raw_EEG_obj.pick_channels(['Fp1', 'T7', 'CP1', 'Oz', 'Fp2', 'F8', 'FC6', 'FC2', 'Cz', 'C4', 'T8', 'CP6', 'CP2', 'PO4']) 25 | 26 | EEG_data_frame = EEG_raw.to_data_frame() 27 | 28 | #calculate three symmetric pairs 29 | EEG_data_frame['T7-T8'] = EEG_data_frame['T7'] - EEG_data_frame['T8'] 30 | EEG_data_frame['Fp1-Fp2'] = EEG_data_frame['Fp1'] - EEG_data_frame['Fp2'] 31 | EEG_data_frame['CP1-CP2'] = EEG_data_frame['CP1'] - EEG_data_frame['CP2'] 32 | 33 | #extract PSD feature from different frequecy 34 | EEG_raw_numpy = np.array(EEG_data_frame).T 35 | EEG_theta = mne.filter.filter_data(EEG_raw_numpy, sfreq = 256, l_freq=4, h_freq=8, verbose = 'ERROR') 36 | EEG_slow_alpha = mne.filter.filter_data(EEG_raw_numpy, sfreq=256, l_freq=8, h_freq=10, verbose = 'ERROR') 37 | EEG_alpha = mne.filter.filter_data(EEG_raw_numpy, sfreq=256, l_freq=8, h_freq=12, verbose = 'ERROR') 38 | EEG_beta = mne.filter.filter_data(EEG_raw_numpy, sfreq=256, l_freq=12, h_freq=30, verbose = 'ERROR') 39 | EEG_gamma = mne.filter.filter_data(EEG_raw_numpy, sfreq=256, l_freq=30, h_freq=4, verbose = 'ERROR') 40 | 41 | #concat them together 42 | features = np.concatenate((EEG_theta, EEG_slow_alpha, EEG_alpha, EEG_beta, EEG_gamma), axis=0) 43 | 44 | #get average in each second for decreasing noise and reduce the number of sample for quicker training. 45 | left_idx = 0 46 | len_features = features.shape[1] 47 | features_list = [] 48 | while left_idx < len_features: 49 | sub_features = features[:, left_idx:left_idx+256] if left_idx+256 < len_features else features[:, left_idx:] 50 | features_list.append(np.average(sub_features, axis = 1)) 51 | left_idx += 256 52 | average_feature = np.array(features_list) 53 | 54 | return average_feature 55 | 56 | class EEG_model: 57 | ''' 58 | This class allow EEG model become an independent model like facial 59 | expression model rathan than two separated model. 60 | Attributes: 61 | valence_model: model for classifying valence 62 | arousal_model: model for classifying arousal 63 | X: the list that saves all EEGs features 64 | y_valence: the valence label list, ground true 65 | y_arousal: the arousal label list, ground true 66 | ''' 67 | 68 | valence_model = None 69 | arousal_model = None 70 | X = None 71 | y_valence = None 72 | y_arousal = None 73 | 74 | def __init__(self): 75 | self.valence_model = SVC(C = 15) 76 | self.arousal_model = SVC(C = 15) 77 | self.X = [] 78 | self.y_valence = [] 79 | self.y_arousal = [] 80 | 81 | def train(self): 82 | ''' 83 | train valence_model and arousal_model using EEG data 84 | ''' 85 | self.valence_model.fit(self.X, self.y_valence) 86 | self.arousal_model.fit(self.X, self.y_arousal) 87 | 88 | def add_one_trial_data(self, trial_path, preprocessed = False): 89 | ''' 90 | read one-trial 91 | data from trial_path and put them into X, valence_y, arousal_y 92 | Parameter: 93 | trial_path: the file path of the trial 94 | preprocessed: whether the EEG data is preprocessed 95 | 96 | ''' 97 | 98 | #load EEG data 99 | if preprocessed is False: 100 | raw_EEG_obj = mne.io.read_raw_fif(trial_path + 'EEG.raw.fif', preload=True, verbose='ERROR') 101 | EEGs = extract_EEG_feature(raw_EEG_obj) 102 | else: 103 | EEGs = np.load(trial_path + 'EEG.npy') 104 | label = pd.read_csv(trial_path + 'label.csv') 105 | 106 | for EEG in EEGs: 107 | self.X.append(EEG) 108 | self.y_valence.append(int(label['valence'] > 5)) 109 | self.y_arousal.append(int(label['arousal'] > 5)) 110 | 111 | 112 | def predict_one_trial(self, trial_path, preprocessed = False): 113 | ''' 114 | use model to predict one trial 115 | Parameter: 116 | trial_path: the trial's path 117 | preprocessed: whether the EEG data is preprocessed 118 | Return: 119 | A: whether the valence was correctly predict. 120 | (1 stands for correct 0 otherwise) 121 | B: whether the arousal was correctly predict. 122 | (1 stands for correct 0 otherwise) 123 | ''' 124 | 125 | #load trial data 126 | if preprocessed is False: 127 | raw_EEG_obj = mne.io.read_raw_fif(trial_path + 'EEG.raw.fif', preload=True, verbose='ERROR') 128 | EEGs = extract_EEG_feature(raw_EEG_obj) 129 | else: 130 | EEGs = np.load(trial_path + 'EEG.npy') 131 | 132 | label = pd.read_csv(trial_path + 'label.csv') 133 | predict_valences, predict_arousals = self.valence_model.predict(EEGs), self.arousal_model.predict(EEGs) 134 | predict_valence = np.sum(predict_valences)/float(len(predict_valences)) > 0.5 135 | predict_arousal = np.sum(predict_arousals)/float(len(predict_arousals)) > 0.5 136 | ground_true_valence = int(label['valence']) > 5 137 | ground_true_arousal = int(label['arousal']) > 5 138 | 139 | return (predict_valence == ground_true_valence), (predict_arousal == ground_true_arousal) 140 | 141 | 142 | def predict_one_trial_scores(self, trial_path, preprocessed = False): 143 | ''' 144 | use model to predict one trial 145 | Parameter: 146 | trial_path: the trial's path 147 | preprocessed: whether the EEG data is preprocessed 148 | Return: 149 | score_valence: the scores of valence predicted by face model 150 | score_arousal: the scores of arousal predicted by EEG model 151 | ''' 152 | #load trial data 153 | if preprocessed is False: 154 | raw_EEG_obj = mne.io.read_raw_fif(trial_path + 'EEG.raw.fif', preload=True, verbose='ERROR') 155 | EEGs = extract_EEG_feature(raw_EEG_obj) 156 | else: 157 | EEGs = np.load(trial_path + 'EEG.npy') 158 | 159 | predict_valences, predict_arousals = self.valence_model.predict(EEGs), self.arousal_model.predict(EEGs) 160 | 161 | score_valence = np.sum(predict_valences)/float(len(predict_valences)) 162 | score_arousal = np.sum(predict_arousals)/float(len(predict_arousals)) 163 | 164 | return score_valence, score_arousal 165 | 166 | def predict_one_trial_results(self, trial_path, preprocessed = False): 167 | ''' 168 | use model to predict one trial 169 | Parameter: 170 | trial_path: the trial's path 171 | preprocessed: whether the EEG data is preprocessed 172 | Return: 173 | result_valence: the results of valence predicted by face model 174 | result_arousal: the results of arousal predicted by EEG model 175 | ''' 176 | score_valence, score_arousal = self.predict_one_trial_scores(trial_path, preprocessed) 177 | result_valence = score_valence > 0.5 178 | result_arousal = score_arousal > 0.5 179 | 180 | return result_valence, result_arousal -------------------------------------------------------------------------------- /EEG/process_MAHNOB_HCI/MAHNOB_HCI_EEG_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Sun Sep 30 13:41:58 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | """ 9 | This is an example for processing EEG data in MAHNOB_HCI dataset using the function in ../process_facial_expression/processing script 10 | Note that the dataset's format should be same as the format performed in 'dataset' folder 11 | The MAHNOB_HCI dataset contains 20 trials for each subject. 12 | In this example, a leave-one-out cross validation is performed for each subject. 13 | This is an example for leave-one-trial-out cross validation 14 | """ 15 | 16 | import sys 17 | sys.path.append('../process_EEG') 18 | import EEG_tool 19 | 20 | if __name__ == '__main__': 21 | root_path = '../../dataset/' + 'MAHNOB_HCI/' 22 | 23 | #for each subject we train 20 model 24 | for subject_id in range(1, 21): 25 | #calculate accuracy 26 | acc_valence, acc_arousal = 0, 0 27 | 28 | subject_path = root_path + str(subject_id) + '/' 29 | #each trial has one change to become validation set. leave-one-trial out 30 | for validation_trial_id in range(1, 21): 31 | 32 | EEG_model = EEG_tool.EEG_model() 33 | #use other 19 trial as train set 34 | for train_trial_id in range(1, 21): 35 | #can't put the validation trial into train set 36 | if train_trial_id == validation_trial_id: 37 | continue 38 | 39 | #load train data 40 | path = subject_path + 'trial_' + str(train_trial_id) + '/' 41 | EEG_model.add_one_trial_data(path, preprocessed=False) 42 | 43 | 44 | #feed data to model 45 | EEG_model.train() 46 | 47 | #validation on one trial 48 | path = subject_path + 'trial_' + str(validation_trial_id) + '/' 49 | 50 | #predict one trial 51 | valence_correct, arousal_correct = EEG_model.predict_one_trial(path) 52 | 53 | acc_valence += 1 if valence_correct else 0 54 | acc_arousal += 1 if arousal_correct else 0 55 | print (valence_correct, arousal_correct) 56 | 57 | print ('subject: ' + str(subject_id)) 58 | print (acc_valence/20., acc_arousal/20.) -------------------------------------------------------------------------------- /EEG/process_ONLINE/ONLINE_EEG_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Sun Sep 30 13:42:21 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | ''' 9 | This is an example of handle EEG data in ONLINE dataset, whose EEG features have been preprocessed. 10 | the shape of 'EEG.npy' stored an array with shape (60, 85) 11 | 60 represents 60 seconds, the origin dataset is 128Hz and we get the average for each feature to avoid noise. 12 | 85 represents 85 feature for each second. The meaning of each feature was presented in paper. 13 | The DEAP dataset contains 40 trials for each subject. 14 | In this example, for each subject, 20 trials are for training whereas the other trials are for testing. 15 | ''' 16 | 17 | import numpy as np 18 | import sys 19 | sys.path.append('../process_EEG') 20 | import EEG_tool 21 | 22 | if __name__ == '__main__': 23 | ROOT_PATH = '../../dataset/ONLINE/' 24 | 25 | for subject_id in range(1, 23): 26 | subject_path = ROOT_PATH + str(subject_id)+'/' 27 | EEG_model = EEG_tool.EEG_model() 28 | 29 | #random select 20 trial for training, the other trials for testing 30 | train_idxs_set = set(np.random.choice(np.arange(1, 41), size = 20, replace = False)) 31 | all_set = set(np.arange(1, 41)) 32 | test_idxs_set = all_set - train_idxs_set 33 | 34 | #training 35 | for trial_id in train_idxs_set: 36 | trial_path = subject_path + 'trial_' + str(trial_id) + '/' 37 | EEG_model.add_one_trial_data(trial_path, preprocessed = True) 38 | 39 | EEG_model.train() 40 | 41 | #testing 42 | acc_valence, acc_arousal = 0., 0. 43 | for trial_id in test_idxs_set: 44 | trial_path = subject_path + 'trial_' + str(trial_id) + '/' 45 | valence_correct, arousal_correct = EEG_model.predict_one_trial(trial_path, preprocessed=True) 46 | 47 | acc_valence += 1 if valence_correct else 0 48 | acc_arousal += 1 if arousal_correct else 0 49 | 50 | print (subject_id, acc_valence/20, acc_arousal/20) 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 yongruihuang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Eumpy-experiment 2 | This is a project of combining facial expression and EEG for emotion recognition using python in emotion experiment. 3 | Data from emotion experiment, such as [DEAP](http://www.eecs.qmul.ac.uk/mmv/datasets/deap/), [MAHNOB-HCI](https://mahnob-db.eu/hci-tagging/) are suitable for using this tool to processce. 4 | 5 | ## Prerequisites 6 | 7 | What things you need to install the software and how to install them 8 | 9 | - [Anaconda (Python 3.7 version)](https://www.anaconda.com/download/#windows) 10 | - [Keras 2.2.4](https://pypi.org/project/Keras/) 11 | - [MNE](https://www.martinos.org/mne/stable/install_mne_python.html) 12 | 13 | ## Note 14 | Before processing the data, the structure of data should be organized in the following format and put it under the directory 'dataset'. 15 | 16 | ``` 17 | dataset 18 | |- MAHNOB-HCI 19 | | |- 1 20 | | | |- trial_1 21 | | | | |- faces.npy 22 | | | | |- EEG.npy 23 | | | | |- label.csv 24 | | | |- trial_2 25 | | | |- trial_... 26 | | |- 2 27 | | | |- trial_1 28 | | | |- trial_2 29 | | | |- trial_.. 30 | ``` 31 | 32 | Where MAHNOB-HCI represents the name of the dataset, '1' represents the id of the subject and trial_1 represets one trial. In each trial, 3 files are presented. 'faces.npy' contains the facial expression data, in our dataset, it is a numpy array-like with shape (?, 48, 48). 'label.csv' contains the ground truth label for this trial. 33 | 34 | As for EEG data, when it presents itself as 'EEG.npy', it means that the data is preprocessed and the data can be directly feed to classifier. However, when it is presented as 'EEG.raw.fif', it means that the data is not preprocessed. We have provided some preprocess approach and the user can choose to use our approach or theirs. 35 | In our released dataset, the dataset DEAP and ONLINE (recorded by us) are used precessed EEG data while the MAHNOB-HCI dataset is used raw EEG data. 36 | 37 | Due to the limit of uploading file in Github, we put our dataset in [here](https://pan.baidu.com/s/1a6k5_tRXk3niZqrxXyNIhg), password 7dep. 38 | Please download the dataset and put it under the directory 'dataset'. 39 | 40 | ## License 41 | 42 | This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /adaboost_fusion/process_DEAP/DEAP_adboost_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Oct 2 17:02:01 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | """ 9 | This is an example for processing adaboost fusion algorithm in DEAP dataset using the function in ../process_adaboost/adaboost_tool 10 | Note that the dataset's format should be same as the format performed in 'dataset' folder 11 | The ONLINE dataset contains 40 trials for each subject. 12 | In this example, for each subject, 20 trials are for training whereas the other trials are for testing. 13 | """ 14 | 15 | import sys 16 | sys.path.append('../process_adaboost') 17 | import numpy as np 18 | import adaboost_tool 19 | 20 | if __name__ == '__main__': 21 | ROOT_PATH = '../../dataset/DEAP/' 22 | 23 | for subject_id in range(1, 23): 24 | subject_path = ROOT_PATH + str(subject_id)+'/' 25 | 26 | adaboost_model = adaboost_tool.Adaboost_model(preprocessed=True) 27 | #random select 20 trial for training, the other trials for testing 28 | train_idxs_set = set(np.random.choice(np.arange(1, 41), size = 20, replace = False)) 29 | all_set = set(np.arange(1, 41)) 30 | test_idxs_set = all_set - train_idxs_set 31 | 32 | #training 33 | for trial_id in train_idxs_set: 34 | 35 | trial_path = subject_path + 'trial_' + str(trial_id) + '/' 36 | adaboost_model.add_one_trial_data(trial_path) 37 | 38 | adaboost_model.train() 39 | 40 | #testing 41 | acc_valence, acc_arousal = 0., 0. 42 | for trial_id in test_idxs_set: 43 | trial_path = subject_path + 'trial_' + str(trial_id) + '/' 44 | valence_correct, arousal_correct = adaboost_model.predict_one_trial(trial_path) 45 | acc_valence += 1 if valence_correct else 0 46 | acc_arousal += 1 if arousal_correct else 0 47 | 48 | print (subject_id, acc_valence/20, acc_arousal/20) -------------------------------------------------------------------------------- /adaboost_fusion/process_MAHNOB_HCI/MANNOB_HCI_adaboost_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Oct 2 17:02:20 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | """ 9 | This is an example for processing fusion algorithm in MAHNOB_HCI dataset using the function in ../process_adaboost/adaboost_tool script 10 | Note that the dataset's format should be same as the format performed in 'dataset' folder 11 | The MAHNOB_HCI dataset contains 20 trials for each subject. 12 | In this example, a leave-one-out cross validation is performed for each subject. 13 | """ 14 | 15 | import sys 16 | sys.path.append('../process_adaboost') 17 | import adaboost_tool 18 | 19 | if __name__ == '__main__': 20 | 21 | root_path = '../../dataset/' + 'MAHNOB_HCI/' 22 | 23 | #for each subject we train 20 model 24 | for subject_id in range(1, 10): 25 | #calculate accuracy 26 | acc_valence, acc_arousal = 0, 0 27 | 28 | subject_path = root_path + str(subject_id) + '/' 29 | #each trial has one change to become validation set. leave-one-trial out 30 | for validation_trial_id in range(1, 21): 31 | 32 | adaboost_model = adaboost_tool.Adaboost_model(preprocessed = False) 33 | #use other 19 trial as train set 34 | for train_trial_id in range(1, 21): 35 | #can't put the validation trial into train set 36 | if train_trial_id == validation_trial_id: 37 | continue 38 | 39 | #load train data 40 | path = subject_path + 'trial_' + str(train_trial_id) + '/' 41 | adaboost_model.add_one_trial_data(path) 42 | 43 | adaboost_model.train() 44 | 45 | #validation on one trial 46 | path = subject_path + 'trial_' + str(validation_trial_id) + '/' 47 | 48 | #predict one trial 49 | valence_correct, arousal_correct = adaboost_model.predict_one_trial(path) 50 | print (valence_correct, arousal_correct) 51 | if(valence_correct): 52 | acc_valence+=1 53 | if(arousal_correct): 54 | acc_arousal+=1 55 | 56 | print ('subject: ' + str(subject_id)) 57 | print (acc_valence/20., acc_arousal/20.) -------------------------------------------------------------------------------- /adaboost_fusion/process_ONLINE/ONLINE_adaboost_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Oct 2 17:02:42 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | """ 9 | This is an example for processing adaboost fusion algorithm in ONLINE dataset using the function in ../process_adaboost/adaboost_tool script 10 | Note that the dataset's format should be same as the format performed in 'dataset' folder 11 | The ONLINE dataset contains 40 trials for each subject. 12 | In this example, for each subject, 20 trials are for training whereas the other trials are for testing. 13 | """ 14 | 15 | import sys 16 | sys.path.append('../process_adaboost') 17 | import numpy as np 18 | import adaboost_tool 19 | 20 | if __name__ == '__main__': 21 | ROOT_PATH = '../../dataset/ONLINE/' 22 | 23 | for subject_id in range(1, 23): 24 | subject_path = ROOT_PATH + str(subject_id)+'/' 25 | 26 | adaboost_model = adaboost_tool.Adaboost_model(preprocessed=True) 27 | #random select 20 trial for training, the other trials for testing 28 | train_idxs_set = set(np.random.choice(np.arange(1, 41), size = 20, replace = False)) 29 | all_set = set(np.arange(1, 41)) 30 | test_idxs_set = all_set - train_idxs_set 31 | 32 | #training 33 | for trial_id in train_idxs_set: 34 | 35 | trial_path = subject_path + 'trial_' + str(trial_id) + '/' 36 | adaboost_model.add_one_trial_data(trial_path) 37 | 38 | adaboost_model.train() 39 | 40 | #testing 41 | acc_valence, acc_arousal = 0., 0. 42 | for trial_id in test_idxs_set: 43 | trial_path = subject_path + 'trial_' + str(trial_id) + '/' 44 | valence_correct, arousal_correct = adaboost_model.predict_one_trial(trial_path) 45 | acc_valence += 1 if valence_correct else 0 46 | acc_arousal += 1 if arousal_correct else 0 47 | 48 | print (subject_id, acc_valence/20, acc_arousal/20) -------------------------------------------------------------------------------- /adaboost_fusion/process_adaboost/adaboost_tool.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Oct 2 17:01:31 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | ''' 9 | This script describe the implement of using adaboost algorithm to combine 10 | facial expression and EEG. 11 | The key goal of training for this approach is to train face_weight and 12 | eeg_weight in order to obtain results using (1) 13 | result = 1./(e^(-(face_weight*face_score+eeg_weight*eeg_score)) (1) 14 | where result is high when result > 0.5 else low 15 | 16 | In that case, to get face_weight and eeg_weight, we follow these steps: 17 | Step 1: 18 | Initialize the data weights for each data point by (2) 19 | data_weight = 1/M (2) 20 | where M is the number of the data sample 21 | Step 2: 22 | Use subclassifier to predict the training set and calculate the 23 | error rate by (3) 24 | error_rate = sum(data_weight * (y_hat!=y)) (3) 25 | where y_hat is the label predicted by sub-classifier 26 | (i.e. facial expression or eeg) and y is the ground truth label 27 | Step 3: 28 | Update the weight of the sub-classifier by (4) 29 | sub_weight = 1./2*((1-error_rate)/error_rate) (4) 30 | where sub_weight is face_weight or eeg_weight 31 | Step 4: 32 | Update data weight by (5) 33 | data_weight = data_weight * exp(-sub_weight)/sum(data_weight) 34 | if y_hat is y else 35 | data_weight * exp(sub_weight)/sum(data_weight) (5) 36 | 37 | Continue to calculate the weight of the subsequent subclassifier, i.e 38 | return to Step 2 39 | 40 | ''' 41 | 42 | import sys 43 | sys.path.append('../../EEG/process_EEG') 44 | sys.path.append('../../facial_expression/process_facial_expression') 45 | import face_tool 46 | import EEG_tool 47 | import numpy as np 48 | import pandas as pd 49 | 50 | def read_label(trial_path): 51 | ''' 52 | read trial's ground truth from trial path 53 | Parameter: 54 | trial_path: the path of the trial file 55 | Returens: 56 | ground_true_valence: the ground truth value for this trial in valence space 57 | ground_true_arousal: the ground truth value for this trial in arousal space 58 | ''' 59 | 60 | label = pd.read_csv(trial_path + 'label.csv') 61 | ground_true_valence = int(label['valence']) > 5 62 | ground_true_arousal = int(label['arousal']) > 5 63 | 64 | return ground_true_valence, ground_true_arousal 65 | 66 | def init_data_weight(num_sample): 67 | ''' 68 | Step 1: initialize the data weights 69 | Patameters: 70 | num_sample: the number of the sample 71 | Returns: 72 | valence_data_weight: the data weight for sample in valence space 73 | arousal_data_weight: the data weight for sample in arousal space 74 | ''' 75 | valence_data_weight = np.zeros((num_sample,)) + 1./num_sample 76 | arousal_data_weight = np.zeros((num_sample,)) + 1./num_sample 77 | return valence_data_weight, arousal_data_weight 78 | 79 | def cal_error_rate(data_weight, y_hat, y): 80 | ''' 81 | Step 2: calculate the error rate 82 | Parameters: 83 | data_weight: the weights of data, shape (1, ?), array-like numpy 84 | y_hat: the label predicted by sub-classifier, shape (1, ?), array-like numpy 85 | y: the grounp truth label, shape (1, ?), array-like numpy 86 | Returns: 87 | error_rate: a scalar shows the number of the samples that were 88 | mis-classified by classifier in the dataset 89 | In the real program, add a very small number to prevent it divided by zero 90 | ''' 91 | bias = 1e-11 92 | return sum(data_weight * (y_hat!=y)) + bias 93 | 94 | def compute_sub_classifier_weight(error_rate): 95 | ''' 96 | Step 3: 97 | Parameters: 98 | error_rate: a scalar shows the number of the samples that were 99 | mis-classified by classifier in the dataset 100 | Returns: 101 | The weights of the sub-classifier, scalar 102 | ''' 103 | return 1./2*((1-error_rate)/error_rate) 104 | 105 | def update_data_weight(data_weights, y_hat, y, classifier_weight): 106 | ''' 107 | Step 4: 108 | Parameters: 109 | data_weight: the weights of data, shape (1, ?), array-like numpy 110 | y_hat: the label predicted by sub-classifier, shape (1, ?), array-like numpy 111 | y: the grounp truth label, shape (1, ?), array-like numpy 112 | classifier_weight: the weights of the sub-classifier 113 | Returns: 114 | The new data_weight, scalar 115 | 116 | ''' 117 | symbol = (y != y_hat) 118 | idx_0 = np.where(symbol == 0)[0] 119 | symbol[idx_0] = -1 120 | 121 | data_weights *= np.exp(symbol*classifier_weight) 122 | return data_weights 123 | 124 | class Adaboost_model: 125 | 126 | ''' 127 | Attributes: 128 | face_model: model for predicting face data 129 | EEG_model: model for predicting EEG data 130 | face_valence_weight: the linear wight of valence for face model 131 | face_arousal_weight: the linear wight of arousal for face model 132 | eeg_valence_weight: the linear wight of valence for eeg model 133 | eeg_arousal_weight: the linear wight of arousal for eeg model 134 | train_trial_paths: a list stored all train trial's path 135 | 136 | The equation (1) shown as fllowed was used to calculated and updated the adaboost model 137 | 1./(e^(-(face_weight*face_score+eeg_weight*eeg_score)) (1) 138 | where face_weight is face_valence_weight when the model calculate the valence space 139 | and etc... 140 | ''' 141 | 142 | face_model = None 143 | EEG_model = None 144 | face_valence_weight = 0.5 145 | face_arousal_weight = 0.5 146 | eeg_valence_weight = 0.5 147 | eeg_arousal_weight = 0.5 148 | train_trial_paths = None 149 | preprocessed = None 150 | 151 | def __init__(self, preprocessed): 152 | self.preprocessed = preprocessed 153 | self.face_model = face_tool.Face_model() 154 | self.EEG_model = EEG_tool.EEG_model() 155 | self.train_trial_paths = [] 156 | 157 | def train(self): 158 | ''' 159 | train face_model and EEG_model 160 | In the same time, calculate the weights 161 | ''' 162 | #train face_model and EEG_model 163 | self.face_model.train() 164 | self.EEG_model.train() 165 | 166 | #add all train trial results into list 167 | face_valence_results, face_arousal_results, eeg_valence_results, eeg_arousal_results = [], [], [], [] 168 | valences, arousals = [], [] 169 | for trial_path in self.train_trial_paths: 170 | #process face 171 | face_valence_result, face_arousal_result = self.face_model.predict_one_trial_results(trial_path) 172 | face_valence_results.append(face_valence_result) 173 | face_arousal_results.append(face_arousal_result) 174 | 175 | #process EEG 176 | eeg_valence_result, eeg_arousal_result = self.EEG_model.predict_one_trial_results(trial_path, self.preprocessed) 177 | eeg_valence_results.append(eeg_valence_result) 178 | eeg_arousal_results.append(eeg_arousal_result) 179 | 180 | #process label 181 | valence, arousal = read_label(trial_path) 182 | valences.append(valence) 183 | arousals.append(arousal) 184 | 185 | #trun them into numpy array 186 | face_valence_results = np.array(face_valence_results) 187 | face_arousal_results = np.array(face_arousal_results) 188 | eeg_valence_results = np.array(eeg_valence_results) 189 | eeg_arousal_results = np.array(eeg_arousal_results) 190 | valences = np.array(valences) 191 | arousals = np.array(arousals) 192 | 193 | #Step 1: initialize data weight 194 | M = len(face_valence_results) 195 | valence_data_weight, arousal_data_weight = init_data_weight(M) 196 | 197 | #Step 2: compute the error rate 198 | valence_error_rate = cal_error_rate(valence_data_weight, face_valence_results, valences) 199 | arousal_error_rate = cal_error_rate(arousal_data_weight, face_arousal_results, arousals) 200 | 201 | ##Step 3: compute face weight 202 | self.face_valence_weight = compute_sub_classifier_weight(valence_error_rate) 203 | self.face_arousal_weight = compute_sub_classifier_weight(arousal_error_rate) 204 | 205 | #Step 4: update data weight 206 | valence_data_weight = update_data_weight(valence_data_weight, face_valence_results, valences, self.face_valence_weight) 207 | arousal_data_weight = update_data_weight(arousal_data_weight, face_arousal_results, arousals, self.face_arousal_weight) 208 | 209 | #Reuten to Step 2: compute the error rate for valence 210 | valence_error_rate = cal_error_rate(valence_data_weight, eeg_valence_results, valences) 211 | arousal_error_rate = cal_error_rate(arousal_data_weight, eeg_arousal_results, arousals) 212 | 213 | #Step 3: compute eeg weight 214 | self.eeg_valence_weight = compute_sub_classifier_weight(valence_error_rate) 215 | self.eeg_arousal_weight = compute_sub_classifier_weight(arousal_error_rate) 216 | 217 | 218 | def add_one_trial_data(self, trial_path): 219 | ''' 220 | read one-trial data from trial_path and put them into face_model, EEG_model 221 | Parameter: 222 | trial_path: the file path of the trial 223 | preprocessed: whether the EEG data is preprocessed 224 | ''' 225 | self.face_model.add_one_trial_data(trial_path) 226 | self.EEG_model.add_one_trial_data(trial_path, preprocessed = self.preprocessed) 227 | self.train_trial_paths.append(trial_path) 228 | 229 | def predict_one_trial(self, trial_path): 230 | ''' 231 | use model to predict one trial 232 | Parameter: 233 | trial_path: the trial's path 234 | preprocessed: whether the EEG data is preprocessed 235 | Return: 236 | A: whether the valence was correctly predict. (1 stands for correct 0 otherwise) 237 | B: whether the arousal was correctly predict. (1 stands for correct 0 otherwise) 238 | ''' 239 | 240 | #load face data 241 | face_valence_result, face_arousal_result = self.face_model.predict_one_trial_results(trial_path) 242 | 243 | #load EEG data 244 | eeg_valence_result, eeg_arousal_result = self.EEG_model.predict_one_trial_results(trial_path, preprocessed = self.preprocessed) 245 | 246 | #predict result in valence space 247 | face_valence_result = (1 if face_valence_result == 1 else -1) 248 | eeg_valence_result = (1 if eeg_valence_result == 1 else -1) 249 | predict_valence = (self.face_valence_weight * face_valence_result + self.eeg_valence_weight * eeg_valence_result) > 0 250 | 251 | #predict result in arousal space 252 | face_arousal_result = (1 if face_arousal_result == 1 else -1) 253 | eeg_arousal_result = (1 if eeg_arousal_result == 1 else -1) 254 | predict_arousal = (self.face_arousal_weight * face_arousal_result + self.eeg_arousal_weight * eeg_arousal_result) > 0 255 | 256 | #load ground truth result for valence and arousal 257 | valence, arousal = read_label(trial_path) 258 | 259 | return valence == predict_valence, arousal == predict_arousal 260 | -------------------------------------------------------------------------------- /dataset/fer2013/X_mean.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yongruihuang/Eumpy-experiment/d25a6480f013a8a3705d870809352a4ce35d4a10/dataset/fer2013/X_mean.npy -------------------------------------------------------------------------------- /enum_fusion/process_DEAP/DEAP_enum_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 1 16:55:55 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | """ 9 | This is an example for processing enum fusion algorithm in DEAP dataset using the function in ../process_enum/enum_tool 10 | Note that the dataset's format should be same as the format performed in 'dataset' folder 11 | The ONLINE dataset contains 40 trials for each subject. 12 | In this example, for each subject, 20 trials are for training whereas the other trials are for testing. 13 | """ 14 | 15 | import sys 16 | sys.path.append('../process_enum') 17 | import numpy as np 18 | import enum_tool 19 | 20 | if __name__ == '__main__': 21 | ROOT_PATH = '../../dataset/DEAP/' 22 | 23 | for subject_id in range(1, 23): 24 | subject_path = ROOT_PATH + str(subject_id)+'/' 25 | 26 | enum_model = enum_tool.Enum_model(preprocessed=True) 27 | #random select 20 trial for training, the other trials for testing 28 | train_idxs_set = set(np.random.choice(np.arange(1, 41), size = 20, replace = False)) 29 | all_set = set(np.arange(1, 41)) 30 | test_idxs_set = all_set - train_idxs_set 31 | 32 | #training 33 | for trial_id in train_idxs_set: 34 | 35 | trial_path = subject_path + 'trial_' + str(trial_id) + '/' 36 | enum_model.add_one_trial_data(trial_path) 37 | 38 | enum_model.train() 39 | 40 | #testing 41 | acc_valence, acc_arousal = 0., 0. 42 | for trial_id in test_idxs_set: 43 | trial_path = subject_path + 'trial_' + str(trial_id) + '/' 44 | valence_correct, arousal_correct = enum_model.predict_one_trial(trial_path) 45 | acc_valence += 1 if valence_correct else 0 46 | acc_arousal += 1 if arousal_correct else 0 47 | 48 | print (subject_id, acc_valence/20, acc_arousal/20) -------------------------------------------------------------------------------- /enum_fusion/process_MANHOB_HCI/MANHOB_HCI_enum_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Oct 2 16:44:29 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | """ 9 | This is an example for processing enum fusion algorithm in MAHNOB_HCI dataset using the function in ../process_enum/enum_tool script 10 | Note that the dataset's format should be same as the format performed in 'dataset' folder 11 | The MAHNOB_HCI dataset contains 20 trials for each subject. 12 | In this example, a leave-one-out cross validation is performed for each subject. 13 | """ 14 | 15 | import sys 16 | sys.path.append('../process_enum') 17 | import enum_tool 18 | 19 | if __name__ == '__main__': 20 | 21 | root_path = '../../dataset/' + 'MAHNOB_HCI/' 22 | 23 | #for each subject we train 20 model 24 | for subject_id in range(1, 10): 25 | #calculate accuracy 26 | acc_valence, acc_arousal = 0, 0 27 | 28 | subject_path = root_path + str(subject_id) + '/' 29 | #each trial has one change to become validation set. leave-one-trial out 30 | for validation_trial_id in range(1, 21): 31 | 32 | enum_model = enum_tool.Enum_model(preprocessed = False) 33 | #use other 19 trial as train set 34 | for train_trial_id in range(1, 21): 35 | #can't put the validation trial into train set 36 | if train_trial_id == validation_trial_id: 37 | continue 38 | 39 | #load train data 40 | path = subject_path + 'trial_' + str(train_trial_id) + '/' 41 | enum_model.add_one_trial_data(path) 42 | 43 | enum_model.train() 44 | 45 | #validation on one trial 46 | path = subject_path + 'trial_' + str(validation_trial_id) + '/' 47 | 48 | #predict one trial 49 | valence_correct, arousal_correct = enum_model.predict_one_trial(path) 50 | print (valence_correct, arousal_correct) 51 | if(valence_correct): 52 | acc_valence+=1 53 | if(arousal_correct): 54 | acc_arousal+=1 55 | 56 | print ('subject: ' + str(subject_id)) 57 | print (acc_valence/20., acc_arousal/20.) 58 | -------------------------------------------------------------------------------- /enum_fusion/process_ONLINE/ONLINE_enum_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Oct 2 16:37:54 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | """ 9 | This is an example for processing enum fusion algorithm in ONLINE dataset using the function in ../process_enum/enum_tool script 10 | Note that the dataset's format should be same as the format performed in 'dataset' folder 11 | The ONLINE dataset contains 40 trials for each subject. 12 | In this example, for each subject, 20 trials are for training whereas the other trials are for testing. 13 | """ 14 | 15 | import sys 16 | sys.path.append('../process_enum') 17 | import numpy as np 18 | import enum_tool 19 | 20 | if __name__ == '__main__': 21 | ROOT_PATH = '../../dataset/ONLINE/' 22 | 23 | for subject_id in range(1, 23): 24 | subject_path = ROOT_PATH + str(subject_id)+'/' 25 | 26 | enum_model = enum_tool.Enum_model(preprocessed=True) 27 | #random select 20 trial for training, the other trials for testing 28 | train_idxs_set = set(np.random.choice(np.arange(1, 41), size = 20, replace = False)) 29 | all_set = set(np.arange(1, 41)) 30 | test_idxs_set = all_set - train_idxs_set 31 | 32 | #training 33 | for trial_id in train_idxs_set: 34 | 35 | trial_path = subject_path + 'trial_' + str(trial_id) + '/' 36 | enum_model.add_one_trial_data(trial_path) 37 | 38 | enum_model.train() 39 | 40 | #testing 41 | acc_valence, acc_arousal = 0., 0. 42 | for trial_id in test_idxs_set: 43 | trial_path = subject_path + 'trial_' + str(trial_id) + '/' 44 | valence_correct, arousal_correct = enum_model.predict_one_trial(trial_path) 45 | acc_valence += 1 if valence_correct else 0 46 | acc_arousal += 1 if arousal_correct else 0 47 | 48 | print (subject_id, acc_valence/20, acc_arousal/20) -------------------------------------------------------------------------------- /enum_fusion/process_enum/enum_tool.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Oct 2 13:51:12 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | import sys 9 | sys.path.append('../../EEG/process_EEG') 10 | sys.path.append('../../facial_expression/process_facial_expression') 11 | import face_tool 12 | import EEG_tool 13 | import numpy as np 14 | import pandas as pd 15 | 16 | def read_label(trial_path): 17 | ''' 18 | read trial's ground true from trial path 19 | Parameter: 20 | trial_path: the path of the trial file 21 | ''' 22 | label = pd.read_csv(trial_path + 'label.csv') 23 | ground_true_valence = int(label['valence']) > 5 24 | ground_true_arousal = int(label['arousal']) > 5 25 | 26 | return ground_true_valence, ground_true_arousal 27 | 28 | def cal_mistake(predict_valences, predict_arousals, valences, arousals): 29 | ''' 30 | calculate the number of the mistake in target weight 31 | Parameter: 32 | predict_valences: the array stroed predict valence, numpy-array-like 33 | predict_arousals: the array stroed predict arousal, numpy-array-like 34 | valences: the ground truth valence, numpy-array-like 35 | arousals: the ground truth arousal, numpy-array-like 36 | Return: 37 | valence_mistake: the number of mistake model taken in valence space 38 | arousal_mistake: the number of mistake model taken in arousal space 39 | ''' 40 | valence_mistake, arousal_mistake = 0, 0 41 | for (predict_valence, predict_arousal, valence, arousal) in zip(predict_valences, predict_arousals, valences, arousals): 42 | if (predict_valence != valence): 43 | valence_mistake += 1 44 | if (predict_arousal != arousal): 45 | arousal_mistake += 1 46 | 47 | return valence_mistake, arousal_mistake 48 | 49 | class Enum_model(): 50 | ''' 51 | Attributes: 52 | face_model: model for predicting face data 53 | EEG_model: model for predicting EEG data 54 | valence_weight: the linear wight of valence for two model 55 | arousal_weight: the linear wight of arousal for two model 56 | train_trial_paths: a list stored all train trial's path 57 | ''' 58 | 59 | face_model = None 60 | EEG_model = None 61 | valence_weight = 0.5 62 | arousal_weight = 0.5 63 | train_trial_paths = None 64 | preprocessed = None 65 | 66 | def __init__(self, preprocessed): 67 | self.preprocessed = preprocessed 68 | self.face_model = face_tool.Face_model() 69 | self.EEG_model = EEG_tool.EEG_model() 70 | self.train_trial_paths = [] 71 | 72 | 73 | def train(self): 74 | ''' 75 | train face_model and EEG_model 76 | In the same time, get the best weight 77 | ''' 78 | 79 | # train face_model and EEG_model 80 | self.face_model.train() 81 | self.EEG_model.train() 82 | 83 | # find the best weight 84 | 85 | #calculate the scores and read label into list for each trial 86 | face_valence_scores, face_arousal_scores, eeg_valence_scores, eeg_arousal_scores = [], [], [], [] 87 | valences, arousals = [], [] 88 | for train_trial_path in self.train_trial_paths: 89 | 90 | face_valence_score, face_arousal_score = self.face_model.predict_one_trial_scores(train_trial_path) 91 | eeg_valence_score, eeg_arousal_score = self.EEG_model.predict_one_trial_scores(train_trial_path, self.preprocessed) 92 | valence, arousal = read_label(train_trial_path) 93 | 94 | face_valence_scores.append(face_valence_score) 95 | face_arousal_scores.append(face_arousal_score) 96 | eeg_valence_scores.append(eeg_valence_score) 97 | eeg_arousal_scores.append(eeg_arousal_score) 98 | valences.append(valence) 99 | arousals.append(arousal) 100 | 101 | face_valence_scores = np.array(face_valence_scores) 102 | face_arousal_scores = np.array(face_arousal_scores) 103 | eeg_valence_scores = np.array(eeg_valence_scores) 104 | eeg_arousal_scores = np.array(eeg_arousal_scores) 105 | valences = np.array(valences) 106 | arousals = np.array(arousals) 107 | 108 | #find the wight 109 | weights = np.arange(0, 1 + 1./100., 1./100.) 110 | min_valence_mistake, min_arousal_mistake = 20, 20 111 | for weight in weights: 112 | predict_valences_scores = weight*face_valence_scores + (1-weight)*eeg_valence_scores 113 | predict_arousals_scores = weight*face_arousal_scores + (1-weight)*eeg_arousal_scores 114 | 115 | predict_valences = predict_valences_scores > 0.5 116 | predict_arousals = predict_arousals_scores > 0.5 117 | valence_mistake, arousal_mistake = cal_mistake(predict_valences, predict_arousals, valences, arousals) 118 | 119 | #print valence_mistake,arousal_mistake 120 | if valence_mistake <= min_valence_mistake: 121 | min_valence_mistake = valence_mistake 122 | self.valence_weight = weight 123 | if arousal_mistake <= min_arousal_mistake: 124 | min_arousal_mistake = arousal_mistake 125 | self.arousal_weight = weight 126 | 127 | def add_one_trial_data(self, trial_path): 128 | ''' 129 | read one-trial data from trial_path and put them into face_model, EEG_model 130 | Parameter: 131 | trial_path: the file path of the trial 132 | preprocessed: whether the EEG data is preprocessed 133 | ''' 134 | self.face_model.add_one_trial_data(trial_path) 135 | self.EEG_model.add_one_trial_data(trial_path, preprocessed = self.preprocessed) 136 | self.train_trial_paths.append(trial_path) 137 | 138 | 139 | def predict_one_trial(self, trial_path): 140 | ''' 141 | use model to predict one trial 142 | Parameter: 143 | trial_path: the trial's path 144 | preprocessed: whether the EEG data is preprocessed 145 | Return: 146 | A: whether the valence was correctly predict. (1 stands for correct 0 otherwise) 147 | B: whether the arousal was correctly predict. (1 stands for correct 0 otherwise) 148 | ''' 149 | 150 | #load face data 151 | face_valence_score, face_arousal_score = self.face_model.predict_one_trial_scores(trial_path) 152 | 153 | #load EEG data 154 | eeg_valence_score, eeg_arousal_score = self.EEG_model.predict_one_trial_scores(trial_path, preprocessed = self.preprocessed) 155 | 156 | #calculate output 157 | valence_score = self.valence_weight*face_valence_score + (1-self.valence_weight)*eeg_valence_score 158 | arousal_score = self.arousal_weight*face_arousal_score + (1-self.arousal_weight)*eeg_arousal_score 159 | 160 | ground_truth_valence, ground_truth_arousal = read_label(trial_path) 161 | 162 | return (valence_score > 0.5) == ground_truth_valence, (arousal_score > 0.5) == ground_truth_arousal 163 | 164 | 165 | -------------------------------------------------------------------------------- /facial_expression/process_DEAP/DEAP_facial_expression_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Sep 18 16:12:36 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | """ 9 | This is an example for processing facial expression data in DEAP dataset using the function in ../process_facial_expression/face_tool script 10 | Note that the dataset's format should be same as the format performed in 'dataset' folder 11 | The DEAP dataset contains 40 trials for each subject. 12 | In this example, for each subject, 20 trials are for training whereas the other trials are for testing. 13 | """ 14 | 15 | 16 | import sys 17 | sys.path.append('../process_facial_expression') 18 | import face_tool 19 | 20 | if __name__ == '__main__': 21 | 22 | root_path = '../../dataset/DEAP/' 23 | 24 | for subject_id in range(1, 10): 25 | 26 | subject_path = root_path + str(subject_id) + '/' 27 | face_model = face_tool.Face_model() 28 | 29 | #use 20 trial as train set 30 | for train_trial_id in range(1, 21): 31 | 32 | #load train data 33 | path = subject_path + 'trial_' + str(train_trial_id) + '/' 34 | face_model.add_one_trial_data(path) 35 | 36 | face_model.train() 37 | 38 | #calculate accuracy 39 | acc_valence, acc_arousal = 0, 0 40 | 41 | #other 20 for testing 42 | for test_trial_id in range(21, 41): 43 | path = subject_path + 'trial_' + str(test_trial_id) + '/' 44 | valence_correct, arousal_correct = face_model.predict_one_trial(path) 45 | if(valence_correct): 46 | acc_valence+=1 47 | if(arousal_correct): 48 | acc_arousal+=1 49 | 50 | print ('subject: ' + str(subject_id)) 51 | print (acc_valence/20., acc_arousal/20.) 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /facial_expression/process_DEAP/__init__.py: -------------------------------------------------------------------------------- 1 | # -------------------------------------------------------------------------------- /facial_expression/process_MAHNOB_HCI/MAHNOB_HCI_facial_expression_example.py.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Sep 18 13:47:53 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | """ 9 | This is an example for processing facial expression data in MAHNOB_HCI dataset using the function in ../process_facial_expression/face_tool script 10 | Note that the dataset's format should be same as the format performed in 'dataset' folder 11 | The MAHNOB_HCI dataset contains 20 trials for each subject. 12 | In this example, a leave-one-out cross validation is performed for each subject. 13 | """ 14 | 15 | import sys 16 | sys.path.append('../process_facial_expression') 17 | import face_tool 18 | 19 | if __name__ == '__main__': 20 | 21 | root_path = '../../dataset/' + 'MAHNOB_HCI/' 22 | 23 | #for each subject we train 20 model 24 | for subject_id in range(1, 10): 25 | #calculate accuracy 26 | acc_valence, acc_arousal = 0, 0 27 | 28 | subject_path = root_path + str(subject_id) + '/' 29 | #each trial has one change to become validation set. leave-one-trial out 30 | for validation_trial_id in range(1, 21): 31 | 32 | face_model = face_tool.Face_model() 33 | #use other 19 trial as train set 34 | for train_trial_id in range(1, 21): 35 | #can't put the validation trial into train set 36 | if train_trial_id == validation_trial_id: 37 | continue 38 | 39 | #load train data 40 | path = subject_path + 'trial_' + str(train_trial_id) + '/' 41 | face_model.add_one_trial_data(path) 42 | 43 | face_model.train() 44 | 45 | #validation on one trial 46 | path = subject_path + 'trial_' + str(validation_trial_id) + '/' 47 | 48 | #predict one trial 49 | valence_correct, arousal_correct = face_model.predict_one_trial(path) 50 | print (valence_correct, arousal_correct) 51 | if(valence_correct): 52 | acc_valence+=1 53 | if(arousal_correct): 54 | acc_arousal+=1 55 | 56 | print ('subject: ' + str(subject_id)) 57 | print (acc_valence/20., acc_arousal/20.) 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /facial_expression/process_ONLINE/ONLINE_facial_expression_example.py.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Wed Sep 26 14:50:12 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | """ 9 | This is an example for processing facial expression data in ONLINE dataset using the function in ../process_facial_expression/face_tool script 10 | Note that the dataset's format should be same as the format performed in 'dataset' folder 11 | The ONLINE dataset contains 40 trials for each subject. 12 | In this example, for each subject, 20 trials are for training whereas the other trials are for testing. 13 | """ 14 | 15 | import sys 16 | sys.path.append('../process_facial_expression') 17 | import face_tool 18 | 19 | if __name__ == '__main__': 20 | 21 | root_path = '../../dataset/ONLINE/' 22 | 23 | for subject_id in range(1, 10): 24 | 25 | subject_path = root_path + str(subject_id) + '/' 26 | face_model = face_tool.Face_model() 27 | 28 | #use 20 trial as train set 29 | for train_trial_id in range(1, 21): 30 | 31 | #load train data 32 | path = subject_path + 'trial_' + str(train_trial_id) + '/' 33 | face_model.add_one_trial_data(path) 34 | 35 | face_model.train() 36 | 37 | #calculate accuracy 38 | acc_valence, acc_arousal = 0, 0 39 | 40 | #other 20 for testing 41 | for test_trial_id in range(21, 41): 42 | path = subject_path + 'trial_' + str(test_trial_id) + '/' 43 | valence_correct, arousal_correct = face_model.predict_one_trial(path) 44 | if(valence_correct): 45 | acc_valence+=1 46 | if(arousal_correct): 47 | acc_arousal+=1 48 | 49 | print ('subject: ' + str(subject_id)) 50 | print (acc_valence/20., acc_arousal/20.) 51 | -------------------------------------------------------------------------------- /facial_expression/process_facial_expression/face_tool.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Oct 2 13:16:19 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | import numpy as np 9 | import pandas as pd 10 | from keras.optimizers import SGD 11 | import keras 12 | import keras.layers as L 13 | 14 | def format_raw_images_data(imgs): 15 | ''' 16 | conduct normalize and shape the image data in order to feed it directly to keras model 17 | Parameter: 18 | imgs: shape(?, 48, 48), all pixels are range from 0 to 255 19 | Return: 20 | shape(?, 48, 48, 1), image data after normalizing 21 | 22 | ''' 23 | X_mean = np.load('../../dataset/fer2013/X_mean.npy') 24 | imgs = np.array(imgs) - X_mean 25 | return imgs.reshape(imgs.shape[0], 48, 48, 1) 26 | 27 | def setup_to_finetune(model, freeze_num = 0): 28 | ''' 29 | fix the convolution layer and prepare for the training of dense layer 30 | Parameter: 31 | model: the baseline model 32 | freeze_num: the number of the layers need to be fixed 33 | ''' 34 | for layer in model.layers[:freeze_num]: 35 | layer.trainable = False 36 | for layer in model.layers[freeze_num:]: 37 | layer.trainable = True 38 | 39 | model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='binary_crossentropy', \ 40 | metrics=['accuracy']) 41 | 42 | def get_model(): 43 | ''' 44 | load the baseline model and return the new model 45 | Return: 46 | new_model: new model which the convolution layers are fixed. 47 | ''' 48 | 49 | old_model = keras.models.load_model('../../model/CNN_expression_baseline.h5') 50 | feature_layer = old_model.get_layer('bottleneck').output 51 | x = L.Dense(64, activation='relu')(feature_layer) 52 | valence_output = L.Dense(1, activation='sigmoid', name = 'valence')(x) 53 | arousal_output = L.Dense(1, activation='sigmoid', name = 'arousal')(x) 54 | new_model = keras.Model(input = old_model.inputs, outputs = [valence_output, arousal_output]) 55 | setup_to_finetune(new_model, freeze_num=3) 56 | return new_model 57 | 58 | class Face_model: 59 | ''' 60 | Attributes: 61 | model: model for classifying valence and arousal at the same time, built by keras 62 | X_faces: the list that saves all EEGs features 63 | y_valence: the valence label list, ground true 64 | y_arousal: the arousal label list, ground true 65 | ''' 66 | model = None 67 | X = None 68 | y_valence = None 69 | y_arousal = None 70 | 71 | def __init__(self): 72 | self.model = get_model() 73 | self.X = [] 74 | self.y_valence = [] 75 | self.y_arousal = [] 76 | 77 | def train(self): 78 | ''' 79 | train the model 80 | ''' 81 | 82 | #format data 83 | self.X = format_raw_images_data(self.X) 84 | #train model 85 | epochs = 12 86 | batch_size = 16 87 | self.model.fit(self.X, [self.y_valence, self.y_arousal], epochs=epochs, batch_size=batch_size, verbose=0) 88 | 89 | 90 | def add_one_trial_data(self, trial_path): 91 | ''' 92 | read one-trial data from trial_path and put them into X, valence_y, arousal_y 93 | Parameter: 94 | trial_path: the file path of the trial 95 | ''' 96 | 97 | #load trial data 98 | imgs = np.load(trial_path + 'faces.npy') 99 | label = pd.read_csv(trial_path + 'label.csv') 100 | 101 | #random pick 50 image for each trial 102 | idxs = np.random.choice(len(imgs), size = 50 if len(imgs) > 50 else len(imgs), replace = False) 103 | for idx in idxs: 104 | 105 | self.X.append(imgs[idx]) 106 | self.y_valence.append(int(label['valence'] > 5)) 107 | self.y_arousal.append(int(label['arousal'] > 5)) 108 | 109 | def predict_one_trial(self, trial_path): 110 | ''' 111 | use model to predict one trial 112 | Parameter: 113 | trial_path: the trial's path 114 | model: the trained model 115 | Return: 116 | A: whether the valence was correctly predict. (1 stands for correct while 0 otherwise) 117 | B: whether the arousal was correctly predict. (1 stands for correct while 0 otherwise) 118 | ''' 119 | 120 | imgs = np.load(trial_path+'faces.npy') 121 | X = format_raw_images_data(imgs) 122 | label = pd.read_csv(trial_path + 'label.csv') 123 | [predict_valences, predict_arousals] = self.model.predict(X) 124 | predict_valence = np.sum(predict_valences)/float(len(predict_valences)) > 0.5 125 | predict_arousal = np.sum(predict_arousals)/float(len(predict_arousals)) > 0.5 126 | ground_true_valence = int(label['valence']) > 5 127 | ground_true_arousal = int(label['arousal']) > 5 128 | #print (predict_valence, ground_true_valence) 129 | #print (predict_arousal, ground_true_arousal) 130 | 131 | return (predict_valence == ground_true_valence), (predict_arousal == ground_true_arousal) 132 | 133 | def predict_one_trial_scores(self, trial_path): 134 | ''' 135 | use model to predict one trial's scores 136 | Parameter: 137 | trial_path: the trial's path 138 | model: the trained model 139 | Return: 140 | score_valence: the scores of valence predicted by face model 141 | score_arousal: the scores of arousal predicted by EEG model 142 | ''' 143 | 144 | imgs = np.load(trial_path+'faces.npy') 145 | X = format_raw_images_data(imgs) 146 | [predict_valences, predict_arousals] = self.model.predict(X) 147 | score_valence = np.sum(predict_valences)/float(len(predict_valences)) 148 | score_arousal = np.sum(predict_arousals)/float(len(predict_arousals)) 149 | 150 | return score_valence, score_arousal 151 | 152 | def predict_one_trial_results(self, trial_path): 153 | ''' 154 | use model to predict one trial's scores 155 | Parameter: 156 | trial_path: the trial's path 157 | model: the trained model 158 | Return: 159 | result_valence: the results of valence predicted by face model 160 | result_arousal: the results of arousal predicted by EEG model 161 | ''' 162 | 163 | score_valence, score_arousal = self.predict_one_trial_scores(trial_path) 164 | result_valence = score_valence > 0.5 165 | result_arousal = score_arousal > 0.5 166 | 167 | return result_valence, result_arousal -------------------------------------------------------------------------------- /facial_expression/train_baseline_model_inFer2013/base_accuray.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yongruihuang/Eumpy-experiment/d25a6480f013a8a3705d870809352a4ce35d4a10/facial_expression/train_baseline_model_inFer2013/base_accuray.png -------------------------------------------------------------------------------- /facial_expression/train_baseline_model_inFer2013/base_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yongruihuang/Eumpy-experiment/d25a6480f013a8a3705d870809352a4ce35d4a10/facial_expression/train_baseline_model_inFer2013/base_loss.png -------------------------------------------------------------------------------- /facial_expression/train_baseline_model_inFer2013/build_model.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Fri Sep 14 10:05:08 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | import keras.layers as L 9 | import keras 10 | import matplotlib.pyplot as plt 11 | 12 | def get_model(): 13 | 14 | ''' 15 | return: 16 | base model for training 17 | ''' 18 | 19 | input = L.Input(shape = (48, 48, 1)) 20 | x = L.Conv2D(32, (3, 3), activation='relu', padding='same')(input) 21 | x = L.Conv2D(32, (3, 3), activation='relu')(x) 22 | x = L.Conv2D(64, (3, 3), activation='relu')(x) 23 | x = L.Dropout(0.5)(x) 24 | 25 | x = L.MaxPooling2D(pool_size=(3, 3))(x) 26 | 27 | x = L.Flatten(name = 'bottleneck')(x) 28 | x = L.Dense(64, activation='relu')(x) 29 | x = L.Dropout(0.5)(x) 30 | output = L.Dense(7, activation='softmax')(x) 31 | 32 | model = keras.Model(input = input, output = output) 33 | 34 | model.compile(optimizer=keras.optimizers.Adadelta(), loss='categorical_crossentropy', metrics=['accuracy']) 35 | 36 | return model 37 | 38 | def plot_training(history, filename): 39 | ''' 40 | polt the train data image 41 | ''' 42 | 43 | output_acc = history.history['acc'] 44 | val_output_acc = history.history['val_acc'] 45 | 46 | output_loss = history.history['loss'] 47 | val_output_loss = history.history['val_loss'] 48 | 49 | epochs = range(len(val_output_acc)) 50 | 51 | plt.figure() 52 | plt.plot(epochs, output_acc, 'b-', label='train accuracy') 53 | plt.plot(epochs, val_output_acc, 'r-', label='validation accuracy') 54 | plt.legend(loc='best') 55 | plt.title('Training and validation accuracy') 56 | plt.savefig(filename+'_accuray'+'.png') 57 | 58 | plt.figure() 59 | plt.plot(epochs, output_loss, 'b-', label='train loss') 60 | plt.plot(epochs, val_output_loss, 'r-', label='validation loss') 61 | plt.legend(loc='best') 62 | plt.title('Training and validation loss') 63 | plt.savefig(filename+'_loss' + '.png') 64 | 65 | if __name__ == '__main__': 66 | get_model() -------------------------------------------------------------------------------- /facial_expression/train_baseline_model_inFer2013/prepare_data.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Fri Sep 14 08:53:25 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | import pandas as pd 9 | import numpy as np 10 | from numpy import uint8 11 | import matplotlib.pyplot as plt 12 | from sklearn.model_selection._split import train_test_split 13 | 14 | #origin: (0=Angry, 1=Disgust, 2=Fear, 3=Happy, 4=Sad, 5=Surprise, 6=Neutral) 15 | #new_type: 0=Happy, 1=Sad, 2=Surprise, 3=Neutral 16 | def show_demon_image(img): 17 | ''' 18 | img: numpy array respresent an image 19 | ''' 20 | 21 | plt.figure("demon") 22 | plt.imshow(img) 23 | #plt.axis('off') 24 | plt.show() 25 | 26 | def load_data(extend_disgust): 27 | ''' 28 | extract data from 'fer2013.csv' file 29 | 30 | extend_digust: whether to extend disgust class 31 | 32 | return: numpy array -like 33 | train_X: shape(?,48,48) 34 | validation_X: shape(?,48,48) 35 | train_y: shape(?, ) 36 | validation_y: shape(?, ) 37 | ''' 38 | 39 | data = pd.read_csv("../../dataset/fer2013/fer2013.csv") 40 | 41 | X = [] 42 | y = [] 43 | for (pixels, emotion) in zip(data['pixels'], data['emotion']): 44 | #if emotion == 0 or emotion == 1 or emotion == 2: 45 | # continue 46 | img = np.array((pixels.split(' ')), dtype=uint8 ) 47 | img = img.reshape((48, 48)) 48 | #img = cv2.equalizeHist(img) 49 | y.append(emotion) 50 | X.append(img) 51 | 52 | if extend_disgust: 53 | #extend disgust facial expression data, inorder to overcome the problem that class 'digust' has much less sample than other class. 54 | disgust_image = np.load('../../dataset/fer2013/extend_disgust.npy') 55 | X.extend(disgust_image) 56 | y.extend(np.ones((len(disgust_image),))) 57 | 58 | X = np.array(X, dtype=uint8) 59 | y = np.array(y, dtype=uint8) 60 | X = X.astype('float32') 61 | train_X, validation_X, train_y, validation_y = \ 62 | train_test_split(X, y, test_size=0.2, random_state = 0) 63 | 64 | return train_X, validation_X, train_y, validation_y 65 | 66 | 67 | if __name__ == '__main__': 68 | train_X, validation_X, train_y, validation_y = load_data(extend_disgust = True) 69 | 70 | #save data for quicker loading 71 | np.save("../../dataset/fer2013/train_X.npy",train_X) 72 | np.save("../../dataset/fer2013/train_y.npy",train_y) 73 | np.save("../../dataset/fer2013/validation_X.npy",validation_X) 74 | np.save("../../dataset/fer2013/validation_y.npy",validation_y) 75 | 76 | #save mean for normalization 77 | X_mean = np.mean(train_X, axis = 0) 78 | np.save("../../dataset/fer2013/X_mean.npy", X_mean) 79 | 80 | 81 | -------------------------------------------------------------------------------- /facial_expression/train_baseline_model_inFer2013/train.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Fri Sep 14 10:02:31 2018 4 | 5 | @author: Yongrui Huang 6 | """ 7 | 8 | import numpy as np 9 | import build_model 10 | import sys 11 | import keras 12 | 13 | if __name__ == '__main__': 14 | 15 | #record training process file res.log 16 | #saveout = sys.stdout 17 | #fsock = open('res.log', 'w') 18 | 19 | #load dataset set, in this way, the loading process will be very quick 20 | train_X = np.load("../../dataset/fer2013/train_X.npy") 21 | #print (X_train.shape) 22 | train_y = np.load("../../dataset/fer2013/train_y.npy") 23 | #print (y_train.shape) 24 | validation_X = np.load("../../dataset/fer2013/validation_X.npy") 25 | validation_y = np.load("../../dataset/fer2013/validation_y.npy") 26 | 27 | mean_X = np.load("../../dataset/fer2013/X_mean.npy") 28 | train_X -= mean_X 29 | train_X = train_X.reshape(train_X.shape[0], 48, 48, 1) 30 | train_y = keras.utils.to_categorical(train_y, num_classes = 7) 31 | 32 | validation_X -= mean_X 33 | validation_X = validation_X.reshape(validation_X.shape[0], 48, 48, 1) 34 | validation_y = keras.utils.to_categorical(validation_y, num_classes = 7) 35 | 36 | 37 | model = build_model.get_model() 38 | 39 | epochs = 16 40 | batch_size = 128 41 | history = model.fit(train_X, train_y, epochs=epochs, batch_size=batch_size, verbose=2, validation_data = (validation_X, validation_y)) 42 | build_model.plot_training(history, "base") 43 | 44 | #fsock.close() 45 | model.save('../../model/CNN_expression_baseline.h5') 46 | print ("finish") 47 | 48 | 49 | -------------------------------------------------------------------------------- /model/CNN_expression_baseline.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yongruihuang/Eumpy-experiment/d25a6480f013a8a3705d870809352a4ce35d4a10/model/CNN_expression_baseline.h5 --------------------------------------------------------------------------------