├── api ├── data │ ├── __init__.py │ ├── Image_Dataset.py │ └── Seq_Dataset.py ├── models │ ├── __init__.py │ ├── ModelFactory.py │ ├── Multitask_CNN.py │ └── Multitask_CNN_RNN.py ├── utils │ ├── __init__.py │ └── data_utils.py ├── examples │ └── utterance_1.mp4 ├── requirements.txt ├── run_example.py ├── download_pretrained_weights_aff_wild2.sh ├── readme.md ├── config.py └── video_processor.py ├── Multitask-CNN ├── utils │ ├── readme │ └── logging_utils.py ├── torchsampler │ ├── readme │ ├── __init__.py │ ├── imbalanced_sampler.py │ ├── imbalanced_VA.py │ ├── imbalanced_SLML.py │ └── imbalanced_ML.py ├── PATH │ ├── readme │ └── __init__.py ├── readme ├── options │ ├── test_options.py │ ├── train_options.py │ └── base_options.py ├── data │ ├── dataset.py │ ├── test_video_dataset.py │ ├── dataset_Mixed_EXPR.py │ ├── dataset_Mixed_AU.py │ ├── dataset_Mixed_VA.py │ └── custom_dataset_data_loader.py ├── models │ └── models.py └── test.py ├── Multitask-CNN-RNN ├── PATH │ ├── readme │ └── __init__.py ├── data │ ├── readme │ ├── dataset.py │ ├── custom_dataset_data_loader.py │ ├── test_video_dataset.py │ ├── dataset_Mixed_EXPR.py │ ├── dataset_Mixed_AU.py │ └── dataset_Mixed_VA.py ├── models │ ├── readme │ └── models.py ├── options │ ├── readme │ ├── test_options.py │ ├── train_options.py │ └── base_options.py ├── utils │ ├── readme │ └── logging_utils.py ├── torchsampler │ ├── readme │ ├── imbalanced_sampler.py │ ├── imbalanced_VA.py │ ├── imbalanced_SLML.py │ └── imbalanced_ML.py ├── test.py └── val.py ├── AU_pngs ├── AU1.png ├── AU12.png ├── AU15.png ├── AU2.png ├── AU20.png ├── AU25.png ├── AU4.png └── AU6.png ├── imgs ├── algorithm.png ├── thumnail.jpg ├── AU_distribution.png ├── VA_distribution.png └── EXPR_distribution.png ├── create_annotation_file ├── Aff-wild2 │ ├── readme │ ├── create_test_set_file.py │ └── create_train_val_annotation_file.py ├── DISFA │ ├── readme │ ├── read_annotations.py │ └── create_annotation_Mixed_AU.py ├── AFEW-VA │ ├── readme │ ├── create_annotation_files_Mixed_VA.py │ └── read_annotation_and_align_faces.py └── ExpW │ ├── readme │ ├── create_annotations.py │ └── create_annotation_files_Mixed_EXPR.py ├── .gitignore ├── LICENSE ├── download_pretrained_weights_aff_wild2.sh ├── eval_val_set.py └── README.md /api/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /api/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Multitask-CNN/utils/readme: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/PATH/readme: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/data/readme: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/models/readme: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/options/readme: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/utils/readme: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Multitask-CNN/torchsampler/readme: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/torchsampler/readme: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Multitask-CNN/torchsampler/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /api/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .model_utils import * 2 | -------------------------------------------------------------------------------- /Multitask-CNN/PATH/readme: -------------------------------------------------------------------------------- 1 | 2 | Modify __init__.py, change the paths 3 | -------------------------------------------------------------------------------- /Multitask-CNN/readme: -------------------------------------------------------------------------------- 1 | This is the training and testing scripts for Multitask CNN model 2 | -------------------------------------------------------------------------------- /AU_pngs/AU1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels/HEAD/AU_pngs/AU1.png -------------------------------------------------------------------------------- /AU_pngs/AU12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels/HEAD/AU_pngs/AU12.png -------------------------------------------------------------------------------- /AU_pngs/AU15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels/HEAD/AU_pngs/AU15.png -------------------------------------------------------------------------------- /AU_pngs/AU2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels/HEAD/AU_pngs/AU2.png -------------------------------------------------------------------------------- /AU_pngs/AU20.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels/HEAD/AU_pngs/AU20.png -------------------------------------------------------------------------------- /AU_pngs/AU25.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels/HEAD/AU_pngs/AU25.png -------------------------------------------------------------------------------- /AU_pngs/AU4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels/HEAD/AU_pngs/AU4.png -------------------------------------------------------------------------------- /AU_pngs/AU6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels/HEAD/AU_pngs/AU6.png -------------------------------------------------------------------------------- /imgs/algorithm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels/HEAD/imgs/algorithm.png -------------------------------------------------------------------------------- /imgs/thumnail.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels/HEAD/imgs/thumnail.jpg -------------------------------------------------------------------------------- /imgs/AU_distribution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels/HEAD/imgs/AU_distribution.png -------------------------------------------------------------------------------- /imgs/VA_distribution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels/HEAD/imgs/VA_distribution.png -------------------------------------------------------------------------------- /imgs/EXPR_distribution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels/HEAD/imgs/EXPR_distribution.png -------------------------------------------------------------------------------- /api/examples/utterance_1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels/HEAD/api/examples/utterance_1.mp4 -------------------------------------------------------------------------------- /api/requirements.txt: -------------------------------------------------------------------------------- 1 | pillow>=7.1.0 2 | numpy>=1.16 3 | pandas==0.25.3 4 | matplotlib==3.0.3 5 | tqdm==4.40.2 6 | scikit-learn>=0.22 7 | opencv-python>=4.1 -------------------------------------------------------------------------------- /create_annotation_file/Aff-wild2/readme: -------------------------------------------------------------------------------- 1 | Create annotation file which is a dictionary: 2 | first level: ['AU_Set', 'EXPR_Set', 'VA_Set'] 3 | second level: ['Training_Set', 'Validation_Set'] or ['Test_Set'] 4 | third level: ['label', 'path', 'frames_ids'] 5 | -------------------------------------------------------------------------------- /create_annotation_file/DISFA/readme: -------------------------------------------------------------------------------- 1 | Create annotation file which is a dictionary: 2 | second level: ['Training_Set', 'Validation_Set'] or ['Test_Set'] 3 | third level: ['label', 'path', 'frames_ids'] 4 | 5 | DISFA dataset is merged with Aff-wild dataset AU Set. 6 | To create merged dataset, use create_annotation_Mixed_AU.py 7 | -------------------------------------------------------------------------------- /create_annotation_file/AFEW-VA/readme: -------------------------------------------------------------------------------- 1 | Create annotation file which is a dictionary: 2 | second level: ['Training_Set', 'Validation_Set'] or ['Test_Set'] 3 | third level: ['label', 'path', 'frames_ids'] 4 | 5 | AFEW-VA dataset is merged with Aff-wild dataset VA Set. 6 | To create merged dataset, use create_annotation_Mixed_VA.py 7 | -------------------------------------------------------------------------------- /create_annotation_file/ExpW/readme: -------------------------------------------------------------------------------- 1 | 2 | Create annotation file which is a dictionary: 3 | second level: ['Training_Set', 'Validation_Set'] or ['Test_Set'] 4 | third level: ['label', 'path', 'frames_ids'] 5 | 6 | ExpW dataset is merged with Aff-wild dataset EXPR Set. 7 | To create merged dataset, use create_annotation_Mixed_EXPR.py 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Don't track content of these folders 2 | api/OpenFace/ 3 | api/pretrained_models/ 4 | api/pytorch-benchmarks/ 5 | # Compiled source # 6 | ################### 7 | *.com 8 | *.class 9 | *.dll 10 | *.exe 11 | *.o 12 | *.so 13 | *.pyc 14 | *.so 15 | 16 | # shell files 17 | # 18 | *.dat 19 | *.png 20 | *.pkl 21 | # checkpoints 22 | *.pth 23 | *.t7 24 | # Packages # 25 | ############ 26 | # it's better to unpack these files and commit the raw source 27 | # git has its own built in compression methods 28 | *.7z 29 | *.dmg 30 | *.gz 31 | *.iso 32 | *.jar 33 | *.rar 34 | *.tar 35 | *.tar.xz 36 | *.zip 37 | *.h5 38 | 39 | # tensorboard 40 | events.out* 41 | -------------------------------------------------------------------------------- /Multitask-CNN/utils/logging_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pandas as pd 3 | import pickle 4 | import matplotlib.pyplot as plt 5 | plt.rcParams.update({'font.size': 12}) 6 | def draw_plots(df, title, save_path): 7 | plt.figure() 8 | for i, key in enumerate(df.keys()): 9 | if key !='loss': 10 | plt.plot(df.index, df[key], linewidth=2, label = key) 11 | plt.legend() 12 | plt.title(title) 13 | plt.xlabel("Iterations") 14 | plt.savefig(save_path, bbox_inches='tight') 15 | 16 | def save_plots(data_dict, train_save_path, val_save_path): 17 | train_df = data_dict['training'] 18 | val_df = data_dict['validation'] 19 | draw_plots(train_df, 'Training Losses', train_save_path) 20 | draw_plots(val_df, 'Validation Metrics', val_save_path) 21 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/utils/logging_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pandas as pd 3 | import pickle 4 | import matplotlib.pyplot as plt 5 | plt.rcParams.update({'font.size': 12}) 6 | def draw_plots(df, title, save_path): 7 | plt.figure() 8 | for i, key in enumerate(df.keys()): 9 | if key !='loss': 10 | plt.plot(df.index, df[key], linewidth=2, label = key) 11 | plt.legend() 12 | plt.title(title) 13 | plt.xlabel("Iterations") 14 | plt.savefig(save_path, bbox_inches='tight') 15 | plt.clf() 16 | plt.cla() 17 | 18 | 19 | def save_plots(data_dict, train_save_path, val_save_path): 20 | train_df = data_dict['training'] 21 | val_df = data_dict['validation'] 22 | draw_plots(train_df, 'Training Losses', train_save_path) 23 | draw_plots(val_df, 'Validation Metrics', val_save_path) 24 | -------------------------------------------------------------------------------- /Multitask-CNN/torchsampler/imbalanced_sampler.py: -------------------------------------------------------------------------------- 1 | class SamplerFactory: 2 | def __init__(self): 3 | pass 4 | @staticmethod 5 | def get_by_name(dataset_name, dataset): 6 | if dataset_name == 'Mixed_EXPR': 7 | from .imbalanced_SLML import ImbalancedDatasetSampler_SLML 8 | sampler = ImbalancedDatasetSampler_SLML(dataset) 9 | elif dataset_name == 'Mixed_AU': 10 | from .imbalanced_ML import ImbalancedDatasetSampler_ML 11 | sampler = ImbalancedDatasetSampler_ML(dataset) 12 | elif dataset_name == 'Mixed_VA': 13 | from .imbalanced_VA import ImbalancedDatasetSampler_VA 14 | sampler = ImbalancedDatasetSampler_VA(dataset) 15 | else: 16 | raise ValueError("Dataset [%s] not recognized." % dataset_name) 17 | return sampler 18 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/torchsampler/imbalanced_sampler.py: -------------------------------------------------------------------------------- 1 | class SamplerFactory: 2 | def __init__(self): 3 | pass 4 | @staticmethod 5 | def get_by_name(dataset_name, dataset): 6 | if dataset_name == 'Mixed_EXPR': 7 | from .imbalanced_SLML import ImbalancedDatasetSampler_SLML 8 | sampler = ImbalancedDatasetSampler_SLML(dataset) 9 | elif dataset_name == 'Mixed_AU': 10 | from .imbalanced_ML import ImbalancedDatasetSampler_ML 11 | sampler = ImbalancedDatasetSampler_ML(dataset) 12 | elif dataset_name == 'Mixed_VA': 13 | from .imbalanced_VA import ImbalancedDatasetSampler_VA 14 | sampler = ImbalancedDatasetSampler_VA(dataset) 15 | else: 16 | raise ValueError("Dataset [%s] not recognized." % dataset_name) 17 | return sampler 18 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/options/test_options.py: -------------------------------------------------------------------------------- 1 | from .base_options import BaseOptions 2 | 3 | 4 | class TestOptions(BaseOptions): 5 | def initialize(self): 6 | BaseOptions.initialize(self) 7 | self.is_train = False 8 | self._parser.add_argument('--teacher_model_path',default = '', type=str, help='the model to be evaluated') 9 | self._parser.add_argument("--eval_with_teacher", action='store_true' ) 10 | self._parser.add_argument('--mode', type=str, default='Validation', choices=['Validation', 'Test'], help='Whether \ 11 | evaluate it on the validation set or the test set.') 12 | self._parser.add_argument('--ensemble', action='store_true') 13 | self._parser.add_argument("--n_students", type=int, default=5) 14 | self._parser.add_argument("--save_dir", type=str, default = "Predictions") 15 | 16 | self.is_train = False 17 | 18 | -------------------------------------------------------------------------------- /Multitask-CNN/options/test_options.py: -------------------------------------------------------------------------------- 1 | from .base_options import BaseOptions 2 | 3 | 4 | class TestOptions(BaseOptions): 5 | def initialize(self): 6 | BaseOptions.initialize(self) 7 | self.is_train = False 8 | self._parser.add_argument('--teacher_model_path',default = '', type=str, help='the model to be evaluated') 9 | self._parser.add_argument("--eval_with_teacher", action='store_true' ) 10 | self._parser.add_argument('--mode', type=str, default='Validation', choices=['Validation', 'Test'], help='Whether \ 11 | evaluate it on the validation set or the test set.') 12 | self._parser.add_argument('--ensemble', action='store_true') 13 | self._parser.add_argument("--n_students", type=int, default=5) 14 | self._parser.add_argument("--save_dir", type=str, default = "Predictions") 15 | 16 | self.is_train = False 17 | 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Didan Deng 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /api/utils/data_utils.py: -------------------------------------------------------------------------------- 1 | from PIL import Image 2 | import numbers 3 | class RandomCrop(object): 4 | def __init__(self, size, v): 5 | if isinstance(size, numbers.Number): 6 | self.size = (int(size), int(size)) 7 | else: 8 | self.size = size 9 | self.v = v 10 | def __call__(self, img): 11 | 12 | w, h = img.size 13 | th, tw = self.size 14 | x1 = int(( w - tw)*self.v) 15 | y1 = int(( h - th)*self.v) 16 | #print("print x, y:", x1, y1) 17 | assert(img.size[0] == w and img.size[1] == h) 18 | if w == tw and h == th: 19 | out_image = img 20 | else: 21 | out_image = img.crop((x1, y1, x1 + tw, y1 + th)) #same cropping method for all images in the same group 22 | return out_image 23 | 24 | class RandomHorizontalFlip(object): 25 | """Randomly horizontally flips the given PIL.Image with a probability of 0.5 26 | """ 27 | def __init__(self, v): 28 | self.v = v 29 | return 30 | def __call__(self, img): 31 | if self.v < 0.5: 32 | img = img.transpose(Image.FLIP_LEFT_RIGHT) 33 | #print ("horiontal flip: ",self.v) 34 | return img -------------------------------------------------------------------------------- /api/run_example.py: -------------------------------------------------------------------------------- 1 | from Emotion_API import Emotion_API 2 | import os 3 | import torch 4 | import time 5 | example_video = 'examples/utterance_1.mp4' 6 | 7 | # use non-temporal model 8 | api = Emotion_API( 9 | device = torch.device('cpu'), # If GPU is available, change it to torch.device('cuda:0') 10 | use_temporal=False, # When using temporal model, the computation cost is larger than using non-temporal model. 11 | num_students = 1, # One can choose between the integers from 1 to 5. Larger number leads to larger computation cost. 12 | OpenFace_exe = './OpenFace/build/bin/FeatureExtraction', # if OpenFace is installed somewhere else, replace the path here 13 | length = 32, # This length applies when use_temporal=True. 14 | #If the video length is smaller than 32, you can just leave it like this. 15 | #The image sampler will sample some frames repeatedly to meet the length requirement, which can take longer. 16 | #If the video length is smaller than 32 and you want to save time, you can change the length to the video length. 17 | batch_size = 24, # The input tensor is (Batch_size, 3, 112, 112) for non-temporal model and (Batch_size, length, 3, 112, 112) for temporal model. 18 | ) 19 | api.run(example_video, csv_output='examples/multitask_preds.csv') 20 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/options/train_options.py: -------------------------------------------------------------------------------- 1 | from .base_options import BaseOptions 2 | 3 | 4 | class TrainOptions(BaseOptions): 5 | def initialize(self): 6 | BaseOptions.initialize(self) 7 | self._parser.add_argument('--print_freq_s', type=int, default= 5, help='frequency of showing training results on console') 8 | self._parser.add_argument('--lr_policy', type=str, default = 'step', choices = ['lambda', 'step', 'plateau']) 9 | self._parser.add_argument('--lr_decay_epochs', type=int, default= 3, help="learning rate decays by 0.1 after every # epochs (lr_policy is 'step')") 10 | self._parser.add_argument('--teacher_nepochs', type=int, default= 3, help='# of epochs to train') 11 | self._parser.add_argument('--student_nepochs', type=int, default= 3, help='# of epochs for student to train') 12 | self._parser.add_argument('--n_students', type=int, default= 5, help='# of students') 13 | self._parser.add_argument('--lr_F', type=float, default=0.0001, help='initial learning rate for G adam') 14 | self._parser.add_argument('--F_adam_b1', type=float, default=0.5, help='beta1 for G adam') 15 | self._parser.add_argument('--F_adam_b2', type=float, default=0.999, help='beta2 for G adam') 16 | self._parser.add_argument('--optimizer', type=str, default='Adam', choices = ['Adam', 'SGD']) 17 | 18 | 19 | self.is_train = True 20 | -------------------------------------------------------------------------------- /Multitask-CNN/options/train_options.py: -------------------------------------------------------------------------------- 1 | from .base_options import BaseOptions 2 | 3 | 4 | class TrainOptions(BaseOptions): 5 | def initialize(self): 6 | BaseOptions.initialize(self) 7 | self._parser.add_argument('--print_freq_s', type=int, default= 5, help='frequency of showing training results on console') 8 | self._parser.add_argument('--lr_policy', type=str, default = 'step', choices = ['lambda', 'step', 'plateau']) 9 | self._parser.add_argument('--lr_decay_epochs', type=int, default= 3, help="learning rate decays by 0.1 after every # epochs (lr_policy is 'step')") 10 | self._parser.add_argument('--teacher_nepochs', type=int, default= 8, help='# of epochs to train') 11 | self._parser.add_argument('--student_nepochs', type=int, default= 3, help='# of epochs for student to train') 12 | self._parser.add_argument('--n_students', type=int, default= 5, help='# of students') 13 | self._parser.add_argument('--lr_F', type=float, default=0.0001, help='initial learning rate for G adam') 14 | self._parser.add_argument('--F_adam_b1', type=float, default=0.5, help='beta1 for G adam') 15 | self._parser.add_argument('--F_adam_b2', type=float, default=0.999, help='beta2 for G adam') 16 | self._parser.add_argument('--optimizer', type=str, default='Adam', choices = ['Adam', 'SGD']) 17 | 18 | 19 | self.is_train = True 20 | -------------------------------------------------------------------------------- /api/models/ModelFactory.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import os 3 | from copy import deepcopy 4 | from .Multitask_CNN import ResNet50 5 | from .Multitask_CNN_RNN import ResNet50_GRU 6 | import sys 7 | sys.path.append("..") 8 | from config import OPT, PRETRAINED_MODEL_DIR 9 | 10 | 11 | class ModelFactory: 12 | def __init__(self): 13 | pass 14 | @classmethod 15 | def get( 16 | self, 17 | device, 18 | Mtype = 'CNN', 19 | num_models = 1, 20 | ): 21 | assert num_models>=1, "At least 1 model in the ensemble." 22 | if Mtype == 'CNN': 23 | single_model = ResNet50(device) 24 | elif Mtype=='CNN_RNN': 25 | single_model = ResNet50_GRU(device) 26 | else: 27 | raise ValueError("{} not supported".format(Mtype)) 28 | model_paths = [os.path.join(PRETRAINED_MODEL_DIR, Mtype, '{}.pth'.format(i)) for i in range(5)] 29 | model_paths = model_paths[:num_models] 30 | ensemble = [] 31 | val_transforms = None 32 | for model_path in model_paths: 33 | model = deepcopy(single_model) 34 | model.load(model_path) 35 | model.set_eval() 36 | ensemble.append(model) 37 | if Mtype == 'CNN': 38 | val_transforms = model.resnet50.backbone.compose_transforms 39 | else: 40 | val_transforms = model.resnet50_GRU.backbone.backbone.compose_transforms 41 | return ensemble, val_transforms 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/PATH/__init__.py: -------------------------------------------------------------------------------- 1 | import collections 2 | 3 | Dataset_Info = collections.namedtuple("Dataset_Info", 'data_file categories test_data_file') 4 | class PATH(object): 5 | def __init__(self, opt=None): 6 | self.Mixed_EXPR = Dataset_Info(data_file = '/media/Samsung/Aff-wild2-Challenge/exps/single_task/create_new_training_set_EXPR/create_annotation_file/mixed_EXPR_annotations.pkl', 7 | test_data_file = '', 8 | categories = ['Neutral','Anger','Disgust','Fear','Happiness','Sadness','Surprise']) 9 | self.Mixed_AU = Dataset_Info(data_file= '/media/Samsung/Aff-wild2-Challenge/exps/single_task/create_new_training_set_AU/create_annotation_file/mixed_AU_annotations.pkl', 10 | test_data_file = '', 11 | categories = ['AU1', 'AU2', 'AU4', 'AU6', 'AU12', 'AU15', 'AU20', 'AU25']) 12 | self.Mixed_VA = Dataset_Info(data_file= '/media/Samsung/Aff-wild2-Challenge/exps/single_task/create_new_training_set_VA/create_annotation_file/mixed_VA_annotations.pkl', 13 | test_data_file = '', 14 | categories = ['valence', 'arousal']) 15 | self.Aff_wild2 = Dataset_Info(data_file = '/media/Samsung/Aff-wild2-Challenge/annotations/annotations.pkl', 16 | test_data_file = '/media/Samsung/Aff-wild2-Challenge/test_set/test_set.pkl', 17 | categories = {'AU': ['AU1', 'AU2', 'AU4', 'AU6', 'AU12', 'AU15', 'AU20', 'AU25'], 18 | 'EXPR':['Neutral','Anger','Disgust','Fear','Happiness','Sadness','Surprise'], 19 | 'VA':['valence', 'arousal']}) 20 | # pytorch benchmark 21 | self.MODEL_DIR = '/media/Samsung/pytorch-benchmarks/models/' 22 | 23 | 24 | -------------------------------------------------------------------------------- /Multitask-CNN/PATH/__init__.py: -------------------------------------------------------------------------------- 1 | import collections 2 | 3 | Dataset_Info = collections.namedtuple("Dataset_Info", 'data_file categories test_data_file') 4 | class PATH(object): 5 | def __init__(self, opt=None): 6 | self.Mixed_EXPR = Dataset_Info(data_file = '/media/Samsung/Aff-wild2-Challenge/exps/single_task/create_new_training_set_EXPR/create_annotation_file/mixed_EXPR_annotations.pkl', 7 | test_data_file = '', 8 | categories = ['Neutral','Anger','Disgust','Fear','Happiness','Sadness','Surprise']) 9 | self.Mixed_AU = Dataset_Info(data_file= '/media/Samsung/Aff-wild2-Challenge/exps/single_task/create_new_training_set_AU/create_annotation_file/mixed_AU_annotations.pkl', 10 | test_data_file = '', 11 | categories = ['AU1', 'AU2', 'AU4', 'AU6', 'AU12', 'AU15', 'AU20', 'AU25']) 12 | self.Mixed_VA = Dataset_Info(data_file= '/media/Samsung/Aff-wild2-Challenge/exps/single_task/create_new_training_set_VA/create_annotation_file/mixed_VA_annotations.pkl', 13 | test_data_file = '', 14 | categories = ['valence', 'arousal']) 15 | self.Aff_wild2 = Dataset_Info(data_file = '/media/Samsung/Aff-wild2-Challenge/annotations/annotations.pkl', 16 | test_data_file = '/media/Samsung/Aff-wild2-Challenge/test_set/test_set.pkl', 17 | categories = {'AU': ['AU1', 'AU2', 'AU4', 'AU6', 'AU12', 'AU15', 'AU20', 'AU25'], 18 | 'EXPR':['Neutral','Anger','Disgust','Fear','Happiness','Sadness','Surprise'], 19 | 'VA':['valence', 'arousal']}) 20 | # pytorch benchmark 21 | self.MODEL_DIR = '/media/Samsung/pytorch-benchmarks/models/' 22 | 23 | 24 | -------------------------------------------------------------------------------- /api/data/Image_Dataset.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from PIL import Image 3 | import numpy as np 4 | import os 5 | import glob 6 | class Image_Dataset(object): 7 | def __init__(self, 8 | image_dir, 9 | transform = None, 10 | image_ext = ['.jpg', '.bmp', '.png']): 11 | assert transform is not None 12 | self._transform = transform 13 | self.image_dir = image_dir 14 | self.image_ext = image_ext 15 | # read dataset 16 | self._read_dataset() 17 | 18 | def __getitem__(self, index): 19 | assert (index < self._dataset_size) 20 | image = None 21 | label = None 22 | img_path = self._data['path'][index] 23 | image = Image.open( img_path).convert('RGB') 24 | frame_ids = self._data['frames_ids'][index] 25 | # transform data 26 | image = self._transform(image) 27 | # pack data 28 | sample = {'image': image, 29 | 'path': img_path, 30 | 'index': index, 31 | 'frames_ids': frame_ids 32 | } 33 | return sample 34 | def _read_dataset(self): 35 | #sample them 36 | frames_paths = glob.glob(os.path.join(self.image_dir, '*')) 37 | frames_paths = [x for x in frames_paths if any([ext in x for ext in self.image_ext])] 38 | frames_paths = sorted(frames_paths, key=lambda x: int(os.path.basename(x).split('.')[0].split('_')[-1])) 39 | self._data = {'path': frames_paths, 'frames_ids': [int(os.path.basename(p).split('.')[0].split('_')[-1]) for p in frames_paths]} # dataframe are easier for indexing 40 | self._ids = np.arange(len(self._data['path'])) 41 | self._dataset_size = len(self._ids) 42 | 43 | 44 | def __len__(self): 45 | return self._dataset_size 46 | -------------------------------------------------------------------------------- /Multitask-CNN/torchsampler/imbalanced_VA.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.utils.data 3 | import torchvision 4 | from tqdm import tqdm 5 | import numpy as np 6 | import random 7 | 8 | class ImbalancedDatasetSampler_VA(torch.utils.data.sampler.Sampler): 9 | """Samples elements randomly from a given list of indices for imbalanced dataset 10 | Arguments: 11 | indices (list, optional): a list of indices 12 | num_samples (int, optional): number of samples to draw 13 | callback_get_label func: a callback-like function which takes two arguments - dataset and index 14 | """ 15 | 16 | def __init__(self, dataset, indices=None, num_samples=None): 17 | 18 | # if indices is not provided, 19 | # all elements in the dataset will be considered 20 | self.indices = list(range(len(dataset))) \ 21 | if indices is None else indices 22 | 23 | # if num_samples is not provided, 24 | # draw `len(indices)` samples in each iteration 25 | self.num_samples = len(self.indices) \ 26 | if num_samples is None else num_samples 27 | 28 | all_labels = dataset._get_all_label() 29 | N, C = all_labels.shape 30 | assert C == 2 31 | hist, x_edges, y_edges = np.histogram2d(all_labels[:, 0], all_labels[:, 1], bins=[20, 20]) 32 | x_bin_id = np.digitize( all_labels[:, 0], bins = x_edges) - 1 33 | y_bin_id = np.digitize( all_labels[:, 1], bins = y_edges) - 1 34 | # for value beyond the edges, the function returns len(digitize_num), but it needs to be replaced by len(edges)-1 35 | x_bin_id[x_bin_id==20] = 20-1 36 | y_bin_id[y_bin_id==20] = 20-1 37 | weights = [] 38 | for x, y in zip(x_bin_id, y_bin_id): 39 | assert hist[x, y]!=0 40 | weights += [1 / hist[x, y]] 41 | 42 | self.weights = torch.DoubleTensor(weights) 43 | 44 | def __iter__(self): 45 | return (self.indices[i] for i in torch.multinomial( 46 | self.weights, self.num_samples, replacement=True)) 47 | 48 | def __len__(self): 49 | return self.num_samples 50 | 51 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/torchsampler/imbalanced_VA.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.utils.data 3 | import torchvision 4 | from tqdm import tqdm 5 | import numpy as np 6 | import random 7 | 8 | class ImbalancedDatasetSampler_VA(torch.utils.data.sampler.Sampler): 9 | """Samples elements randomly from a given list of indices for imbalanced dataset 10 | Arguments: 11 | indices (list, optional): a list of indices 12 | num_samples (int, optional): number of samples to draw 13 | callback_get_label func: a callback-like function which takes two arguments - dataset and index 14 | """ 15 | 16 | def __init__(self, dataset, indices=None, num_samples=None): 17 | 18 | # if indices is not provided, 19 | # all elements in the dataset will be considered 20 | self.indices = list(range(len(dataset))) \ 21 | if indices is None else indices 22 | 23 | # if num_samples is not provided, 24 | # draw `len(indices)` samples in each iteration 25 | self.num_samples = len(self.indices) \ 26 | if num_samples is None else num_samples 27 | 28 | all_labels = dataset._get_all_label() 29 | N, C = all_labels.shape 30 | assert C == 2 31 | hist, x_edges, y_edges = np.histogram2d(all_labels[:, 0], all_labels[:, 1], bins=[20, 20]) 32 | x_bin_id = np.digitize( all_labels[:, 0], bins = x_edges) - 1 33 | y_bin_id = np.digitize( all_labels[:, 1], bins = y_edges) - 1 34 | # for value beyond the edges, the function returns len(digitize_num), but it needs to be replaced by len(edges)-1 35 | x_bin_id[x_bin_id==20] = 20-1 36 | y_bin_id[y_bin_id==20] = 20-1 37 | weights = [] 38 | for x, y in zip(x_bin_id, y_bin_id): 39 | assert hist[x, y]!=0 40 | weights += [1 / hist[x, y]] 41 | 42 | self.weights = torch.DoubleTensor(weights) 43 | 44 | def __iter__(self): 45 | return (self.indices[i] for i in torch.multinomial( 46 | self.weights, self.num_samples, replacement=True)) 47 | 48 | def __len__(self): 49 | return self.num_samples 50 | 51 | -------------------------------------------------------------------------------- /download_pretrained_weights_aff_wild2.sh: -------------------------------------------------------------------------------- 1 | # download five CNN student models 2 | mkdir pretrained_models/ 3 | mkdir pretrained_models/CNN/ 4 | wget -O pretrained_models/CNN/0.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/ETyqkFqXyIZNiLQykQC66zUBh62Dpz4-SUGzuOlpX-mYqw?e=5YZKHG&download=1" 5 | wget -O pretrained_models/CNN/1.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/EchBCqHWQNhGobaxDjLWd1MBagrKRrhPDOO56nRksBSG1g?e=Vh31dL&download=1" 6 | wget -O pretrained_models/CNN/2.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/EWL4GK5wBERBqHLGzQBKHs8Bf-G8Fu-g9aaiUmB6bUXndA?e=6A4xA7&download=1" 7 | wget -O pretrained_models/CNN/3.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/EbcOJ2aLRSJItKdcevHGzlABtB1f95lzwweueT3h3BHFZg?e=CzVSZK&download=1" 8 | wget -O pretrained_models/CNN/4.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/EQDiC68YD_9PqRFdjYg8oXYBNjAWfdPJMgvVTa1ii6EqSg?e=UxUDNh&download=1" 9 | 10 | # download five CNN-RNN student models (seq_len = 32) 11 | mkdir pretrained_models/CNN_RNN/ 12 | 13 | wget -O pretrained_models/CNN_RNN/0.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/EZ5N9YJjYBdFodTaIgHhvaMBUxFgzcdD5iIx3mrBvpTB2A?e=ZOSbhE&download=1" 14 | wget -O pretrained_models/CNN_RNN/1.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/Ee3j1hcdPHFGgjM4YaOgjBwBRWzXrN1HqrZn7Vpy0Xo24g?e=NvDqjG&download=1" 15 | wget -O pretrained_models/CNN_RNN/2.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/EWrxLLIBBthEkAFhs-7L6jEB53BfVqp_EDL5trh_rn3RkA?e=rHNbO8&download=1" 16 | wget -O pretrained_models/CNN_RNN/3.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/ES0yS8jJluBJnK80_ucAueoB3cianGX5pUPwwnBtLUWQtg?e=z4dILE&download=1" 17 | wget -O pretrained_models/CNN_RNN/4.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/EYBfb9_f5RRPkpBTGmOWSLABisYU--EZcmwk4ALhEWZFVw?e=fLJbwS&download=1" -------------------------------------------------------------------------------- /api/download_pretrained_weights_aff_wild2.sh: -------------------------------------------------------------------------------- 1 | # download five CNN student models 2 | mkdir pretrained_models/ 3 | mkdir pretrained_models/CNN/ 4 | wget -O pretrained_models/CNN/0.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/ETyqkFqXyIZNiLQykQC66zUBh62Dpz4-SUGzuOlpX-mYqw?e=5YZKHG&download=1" 5 | wget -O pretrained_models/CNN/1.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/EchBCqHWQNhGobaxDjLWd1MBagrKRrhPDOO56nRksBSG1g?e=Vh31dL&download=1" 6 | wget -O pretrained_models/CNN/2.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/EWL4GK5wBERBqHLGzQBKHs8Bf-G8Fu-g9aaiUmB6bUXndA?e=6A4xA7&download=1" 7 | wget -O pretrained_models/CNN/3.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/EbcOJ2aLRSJItKdcevHGzlABtB1f95lzwweueT3h3BHFZg?e=CzVSZK&download=1" 8 | wget -O pretrained_models/CNN/4.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/EQDiC68YD_9PqRFdjYg8oXYBNjAWfdPJMgvVTa1ii6EqSg?e=UxUDNh&download=1" 9 | 10 | # download five CNN-RNN student models (seq_len = 32) 11 | mkdir pretrained_models/CNN_RNN/ 12 | 13 | wget -O pretrained_models/CNN_RNN/0.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/EZ5N9YJjYBdFodTaIgHhvaMBUxFgzcdD5iIx3mrBvpTB2A?e=ZOSbhE&download=1" 14 | wget -O pretrained_models/CNN_RNN/1.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/Ee3j1hcdPHFGgjM4YaOgjBwBRWzXrN1HqrZn7Vpy0Xo24g?e=NvDqjG&download=1" 15 | wget -O pretrained_models/CNN_RNN/2.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/EWrxLLIBBthEkAFhs-7L6jEB53BfVqp_EDL5trh_rn3RkA?e=rHNbO8&download=1" 16 | wget -O pretrained_models/CNN_RNN/3.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/ES0yS8jJluBJnK80_ucAueoB3cianGX5pUPwwnBtLUWQtg?e=z4dILE&download=1" 17 | wget -O pretrained_models/CNN_RNN/4.pth "https://hkustconnect-my.sharepoint.com/:u:/g/personal/ddeng_connect_ust_hk/EYBfb9_f5RRPkpBTGmOWSLABisYU--EZcmwk4ALhEWZFVw?e=fLJbwS&download=1" -------------------------------------------------------------------------------- /Multitask-CNN/torchsampler/imbalanced_SLML.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.utils.data 3 | import torchvision 4 | from tqdm import tqdm 5 | 6 | class ImbalancedDatasetSampler_SLML(torch.utils.data.sampler.Sampler): 7 | """Samples elements randomly from a given list of indices for imbalanced dataset 8 | Arguments: 9 | indices (list, optional): a list of indices 10 | num_samples (int, optional): number of samples to draw 11 | callback_get_label func: a callback-like function which takes two arguments - dataset and index 12 | """ 13 | 14 | def __init__(self, dataset, indices=None, num_samples=None, callback_get_label=None): 15 | 16 | # if indices is not provided, 17 | # all elements in the dataset will be considered 18 | self.indices = list(range(len(dataset))) \ 19 | if indices is None else indices 20 | 21 | # define custom callback 22 | self.callback_get_label = callback_get_label 23 | 24 | # if num_samples is not provided, 25 | # draw `len(indices)` samples in each iteration 26 | self.num_samples = len(self.indices) \ 27 | if num_samples is None else num_samples 28 | 29 | # distribution of classes in the dataset 30 | label_to_count = {} 31 | for idx in tqdm(self.indices, total = len(self.indices)): 32 | label = self._get_label(dataset, idx) 33 | if label in label_to_count: 34 | label_to_count[label] += 1 35 | else: 36 | label_to_count[label] = 1 37 | 38 | # weight for each sample 39 | weights = [1.0 / label_to_count[self._get_label(dataset, idx)] 40 | for idx in self.indices] 41 | self.weights = torch.DoubleTensor(weights) 42 | 43 | def _get_label(self, dataset, idx): 44 | if self.callback_get_label: 45 | return self.callback_get_label(dataset, idx) 46 | else: 47 | return dataset._data['label'][idx] 48 | 49 | def __iter__(self): 50 | return (self.indices[i] for i in torch.multinomial( 51 | self.weights, self.num_samples, replacement=True)) 52 | 53 | def __len__(self): 54 | return self.num_samples 55 | 56 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/torchsampler/imbalanced_SLML.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.utils.data 3 | import torchvision 4 | from tqdm import tqdm 5 | 6 | class ImbalancedDatasetSampler_SLML(torch.utils.data.sampler.Sampler): 7 | """Samples elements randomly from a given list of indices for imbalanced dataset 8 | Arguments: 9 | indices (list, optional): a list of indices 10 | num_samples (int, optional): number of samples to draw 11 | callback_get_label func: a callback-like function which takes two arguments - dataset and index 12 | """ 13 | 14 | def __init__(self, dataset, indices=None, num_samples=None, callback_get_label=None): 15 | 16 | # if indices is not provided, 17 | # all elements in the dataset will be considered 18 | self.indices = list(range(len(dataset))) \ 19 | if indices is None else indices 20 | 21 | # define custom callback 22 | self.callback_get_label = callback_get_label 23 | 24 | # if num_samples is not provided, 25 | # draw `len(indices)` samples in each iteration 26 | self.num_samples = len(self.indices) \ 27 | if num_samples is None else num_samples 28 | 29 | # distribution of classes in the dataset 30 | label_to_count = {} 31 | for idx in tqdm(self.indices, total = len(self.indices)): 32 | label = self._get_label(dataset, idx) 33 | if label in label_to_count: 34 | label_to_count[label] += 1 35 | else: 36 | label_to_count[label] = 1 37 | 38 | # weight for each sample 39 | weights = [1.0 / label_to_count[self._get_label(dataset, idx)] 40 | for idx in self.indices] 41 | self.weights = torch.DoubleTensor(weights) 42 | 43 | def _get_label(self, dataset, idx): 44 | if self.callback_get_label: 45 | return self.callback_get_label(dataset, idx) 46 | else: 47 | return dataset._data['label'][idx] 48 | 49 | def __iter__(self): 50 | return (self.indices[i] for i in torch.multinomial( 51 | self.weights, self.num_samples, replacement=True)) 52 | 53 | def __len__(self): 54 | return self.num_samples 55 | 56 | -------------------------------------------------------------------------------- /Multitask-CNN/data/dataset.py: -------------------------------------------------------------------------------- 1 | import torch.utils.data as data 2 | from PIL import Image 3 | import torchvision.transforms as transforms 4 | import os 5 | import os.path 6 | 7 | 8 | class DatasetFactory: 9 | def __init__(self): 10 | pass 11 | @staticmethod 12 | def get_by_name(dataset_name, opt, train_mode='Train', transform = None): 13 | if dataset_name == 'Mixed_EXPR': 14 | from data.dataset_Mixed_EXPR import dataset_Mixed_EXPR 15 | dataset = dataset_Mixed_EXPR(opt, train_mode, transform) 16 | elif dataset_name == 'Mixed_AU': 17 | from data.dataset_Mixed_AU import dataset_Mixed_AU 18 | dataset = dataset_Mixed_AU(opt, train_mode, transform) 19 | elif dataset_name == 'Mixed_VA': 20 | from data.dataset_Mixed_VA import dataset_Mixed_VA 21 | dataset = dataset_Mixed_VA(opt, train_mode, transform) 22 | else: 23 | raise ValueError("Dataset [%s] not recognized." % dataset_name) 24 | 25 | print('Dataset {} was created'.format(dataset.name)) 26 | return dataset 27 | 28 | 29 | class DatasetBase(data.Dataset): 30 | def __init__(self, opt, train_mode='Train', transform=None): 31 | super(DatasetBase, self).__init__() 32 | self._name = 'BaseDataset' 33 | self._root = None 34 | self._opt = opt 35 | self._transform = None 36 | self._train_mode = None 37 | self._create_transform() 38 | 39 | self._IMG_EXTENSIONS = [ 40 | '.jpg', '.JPG', '.jpeg', '.JPEG', 41 | '.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP', 42 | ] 43 | 44 | @property 45 | def name(self): 46 | return self._name 47 | 48 | @property 49 | def path(self): 50 | return self._root 51 | 52 | def _create_transform(self): 53 | self._transform = transforms.Compose([]) 54 | 55 | def get_transform(self): 56 | return self._transform 57 | 58 | def _is_image_file(self, filename): 59 | return any(filename.endswith(extension) for extension in self._IMG_EXTENSIONS) 60 | 61 | def _is_csv_file(self, filename): 62 | return filename.endswith('.csv') 63 | 64 | def _get_all_files_in_subfolders(self, dir, is_file): 65 | images = [] 66 | assert os.path.isdir(dir), '%s is not a valid directory' % dir 67 | 68 | for root, _, fnames in sorted(os.walk(dir)): 69 | for fname in fnames: 70 | if is_file(fname): 71 | path = os.path.join(root, fname) 72 | images.append(path) 73 | 74 | return images 75 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/data/dataset.py: -------------------------------------------------------------------------------- 1 | import torch.utils.data as data 2 | from PIL import Image 3 | import torchvision.transforms as transforms 4 | import os 5 | import os.path 6 | 7 | 8 | class DatasetFactory: 9 | def __init__(self): 10 | pass 11 | @staticmethod 12 | def get_by_name(dataset_name, opt, train_mode='Train', transform = None): 13 | if dataset_name == 'Mixed_EXPR': 14 | from data.dataset_Mixed_EXPR import dataset_Mixed_EXPR 15 | dataset = dataset_Mixed_EXPR(opt, train_mode, transform) 16 | elif dataset_name == 'Mixed_AU': 17 | from data.dataset_Mixed_AU import dataset_Mixed_AU 18 | dataset = dataset_Mixed_AU(opt, train_mode, transform) 19 | elif dataset_name == 'Mixed_VA': 20 | from data.dataset_Mixed_VA import dataset_Mixed_VA 21 | dataset = dataset_Mixed_VA(opt, train_mode, transform) 22 | else: 23 | raise ValueError("Dataset [%s] not recognized." % dataset_name) 24 | 25 | print('Dataset {} was created'.format(dataset.name)) 26 | return dataset 27 | 28 | 29 | class DatasetBase(data.Dataset): 30 | def __init__(self, opt, train_mode='Train', transform=None): 31 | super(DatasetBase, self).__init__() 32 | self._name = 'BaseDataset' 33 | self._root = None 34 | self._opt = opt 35 | self._transform = None 36 | self._train_mode = None 37 | self._create_transform() 38 | 39 | self._IMG_EXTENSIONS = [ 40 | '.jpg', '.JPG', '.jpeg', '.JPEG', 41 | '.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP', 42 | ] 43 | 44 | @property 45 | def name(self): 46 | return self._name 47 | 48 | @property 49 | def path(self): 50 | return self._root 51 | 52 | def _create_transform(self): 53 | self._transform = transforms.Compose([]) 54 | 55 | def get_transform(self): 56 | return self._transform 57 | 58 | def _is_image_file(self, filename): 59 | return any(filename.endswith(extension) for extension in self._IMG_EXTENSIONS) 60 | 61 | def _is_csv_file(self, filename): 62 | return filename.endswith('.csv') 63 | 64 | def _get_all_files_in_subfolders(self, dir, is_file): 65 | images = [] 66 | assert os.path.isdir(dir), '%s is not a valid directory' % dir 67 | 68 | for root, _, fnames in sorted(os.walk(dir)): 69 | for fname in fnames: 70 | if is_file(fname): 71 | path = os.path.join(root, fname) 72 | images.append(path) 73 | 74 | return images 75 | -------------------------------------------------------------------------------- /api/data/Seq_Dataset.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from PIL import Image 3 | import os 4 | import glob 5 | import numpy as np 6 | import torch 7 | 8 | class Seq_Dataset(object): 9 | def __init__(self, 10 | image_dir, 11 | seq_len = 32, 12 | transform = None, 13 | image_ext = ['.jpg', '.bmp', '.png']): 14 | 15 | assert transform is not None 16 | self._transform = transform 17 | self.image_ext = image_ext 18 | self.image_dir = image_dir 19 | self.seq_len = seq_len 20 | # read dataset 21 | self._read_dataset() 22 | 23 | def __getitem__(self, index): 24 | assert (index < self._dataset_size) 25 | images = [] 26 | labels = [] 27 | img_paths = [] 28 | frames_ids = [] 29 | df = self.sample_seqs[index] 30 | for i, row in df.iterrows(): 31 | img_path = row['path'] 32 | image = Image.open(img_path).convert('RGB') 33 | image = self._transform(image) 34 | frame_id = row['frames_ids'] 35 | images.append(image) 36 | img_paths.append(img_path) 37 | frames_ids.append(frame_id) 38 | # pack data 39 | sample = {'image': torch.stack(images,dim=0), 40 | 'path': img_paths, 41 | 'index': index, 42 | 'frames_ids':frames_ids 43 | } 44 | return sample 45 | def _read_dataset(self): 46 | #sample them 47 | seq_len = self.seq_len 48 | frames_paths = glob.glob(os.path.join(self.image_dir, '*')) 49 | frames_paths = [x for x in frames_paths if any([ext in x for ext in self.image_ext])] 50 | frames_paths = sorted(frames_paths, key=lambda x: int(os.path.basename(x).split('.')[0].split('_')[-1])) 51 | self._data = {'path': frames_paths, 'frames_ids': [int(os.path.basename(p).split('.')[0].split('_')[-1]) for p in frames_paths]} # dataframe are easier for indexing 52 | self._data = pd.DataFrame.from_dict(self._data) 53 | self.sample_seqs = [] 54 | N = seq_len 55 | for i in range(len(self._data['path'])//N + 1): 56 | start, end = i*N, i*N + seq_len 57 | if end >= len(self._data): 58 | start, end = len(self._data) - seq_len, len(self._data) 59 | new_df = self._data.iloc[start:end] 60 | assert len(new_df) <= seq_len 61 | self.sample_seqs.append(new_df) 62 | self._ids = np.arange(len(self.sample_seqs)) 63 | self._dataset_size = len(self._ids) 64 | 65 | def __len__(self): 66 | return self._dataset_size 67 | -------------------------------------------------------------------------------- /Multitask-CNN/data/test_video_dataset.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import torchvision.transforms as transforms 3 | from .dataset import DatasetBase 4 | from PIL import Image 5 | import random 6 | import numpy as np 7 | import pickle 8 | import pandas as pd 9 | from PATH import PATH 10 | PRESET_VARS = PATH() 11 | import pickle 12 | class Test_dataset(object): 13 | def __init__(self, opt, video_data, train_mode = 'Test', transform = None): 14 | self._name = 'Test_dataset' 15 | self._train_mode = train_mode 16 | if transform is not None: 17 | self._transform = transform 18 | else: 19 | self._transform = self._create_transform() 20 | # read dataset 21 | self._data = video_data 22 | self._read_dataset() 23 | def __getitem__(self, index): 24 | assert (index < self._dataset_size) 25 | image = None 26 | label = None 27 | img_path = self._data['path'][index] 28 | image = Image.open( img_path).convert('RGB') 29 | label = self._data['label'][index] 30 | frame_id = self._data['frames_ids'][index] 31 | 32 | # transform data 33 | image = self._transform(image) 34 | # pack data 35 | sample = {'image': image, 36 | 'label': label, 37 | 'path': img_path, 38 | 'index': index, 39 | 'frames_ids': frame_id 40 | } 41 | 42 | return sample 43 | 44 | def _read_dataset(self): 45 | self._ids = np.arange(len(self._data['path'])) 46 | self._dataset_size = len(self._ids) 47 | 48 | def __len__(self): 49 | return self._dataset_size 50 | def _create_transform(self): 51 | if self._train_mode == 'Train': 52 | img_size = self._opt.image_size 53 | resize = int(img_size * 1.2) 54 | transform_list = [transforms.Resize(resize), 55 | transforms.RandomCrop(img_size), 56 | transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3), 57 | transforms.RandomHorizontalFlip(), 58 | transforms.ToTensor(), 59 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 60 | std=[0.229, 0.224, 0.225]), 61 | ] 62 | else: 63 | img_size = self._opt.image_size 64 | transform_list = [transforms.Resize(img_size), 65 | transforms.ToTensor(), 66 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 67 | std=[0.229, 0.224, 0.225]), 68 | ] 69 | self._transform = transforms.Compose(transform_list) 70 | 71 | 72 | -------------------------------------------------------------------------------- /create_annotation_file/ExpW/create_annotations.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import os 3 | import numpy as np 4 | import argparse 5 | from matplotlib import pyplot as plt 6 | import matplotlib 7 | import glob 8 | import pandas as pd 9 | from PIL import Image 10 | from tqdm import tqdm 11 | parser = argparse.ArgumentParser(description='save annotations') 12 | parser.add_argument('--vis', action = 'store_true', 13 | help='whether to visualize the distribution') 14 | parser.add_argument('--image_dir', type=str, default = '/media/Samsung/ExpW_dataset/image/origin', 15 | help='image dir') 16 | parser.add_argument('--lst_file', type=str, default='/media/Samsung/ExpW_dataset/label/label.lst') 17 | parser.add_argument('--output_dir', type=str, default= '/media/Samsung/ExpW_dataset/image/cropped_faces') 18 | parser.add_argument('--save_path', type=str, default='/media/Samsung/ExpW_dataset/annotations.pkl') 19 | args = parser.parse_args() 20 | if not os.path.exists(args.output_dir): 21 | os.makedirs(args.output_dir) 22 | Expr_list = ['angry','disgust','fear','happy','sad','surprise','neutral'] 23 | plot_Expr_list = [6, 0, 1, 2, 3, 4, 5] 24 | def read_lst(lst_file): 25 | with open(lst_file, 'r') as f: 26 | lines = f.readlines() 27 | lines = [l.strip() for l in lines] 28 | data = {'name': [] , 'face_id': [] , 'ymin':[] , 'xmin':[], 'xmax': [] , 'ymax':[], 'confidence':[], 'emotion':[]} 29 | for l in lines: 30 | l = l.split(" ") 31 | data['name'].append(l[0]) 32 | data['face_id'].append(int(l[1])) 33 | data['ymin'].append(int(l[2])) 34 | data['xmin'].append(int(l[3])) 35 | data['xmax'].append(int(l[4])) 36 | data['ymax'].append(int(l[5])) 37 | data['confidence'].append(float(l[6])) 38 | data['emotion'].append(int(l[7])) 39 | df = pd.DataFrame.from_dict(data) 40 | return df 41 | def read_all_image(): 42 | data_file = {} 43 | df = read_lst(args.lst_file) 44 | paths = [] 45 | labels = [] 46 | for i in tqdm(range(len(df)), total = len(df)): 47 | line = df.iloc[i] 48 | path = os.path.join(args.image_dir, line['name']) 49 | if os.path.exists(path): 50 | bbox = line[['xmin', 'ymin', 'xmax', 'ymax']].values 51 | img = Image.open(path).convert("RGB") 52 | face = img.crop(tuple(bbox)) 53 | save_path = os.path.join(args.output_dir, line['name']) 54 | face.save(save_path) 55 | paths.append(save_path) 56 | labels.append(line['emotion']) 57 | data_file['label'] = np.array(labels) 58 | data_file['path'] = paths 59 | pickle.dump(data_file, open(args.save_path, 'wb')) 60 | return data_file 61 | def plot_distribution(data_file): 62 | histogram = np.zeros(len(plot_Expr_list)) 63 | all_samples = data_file['label'] 64 | for i in range(7): 65 | find_true = sum(all_samples==i) 66 | histogram[i] =find_true/all_samples.shape[0] 67 | histogram = histogram[plot_Expr_list] 68 | Expr_list_new = [Expr_list[ii] for ii in plot_Expr_list] 69 | plt.bar(np.arange(len(Expr_list)), histogram) 70 | plt.xticks(np.arange(len(Expr_list)), Expr_list_new) 71 | plt.show() 72 | 73 | if __name__== '__main__': 74 | #data_file = read_all_image() 75 | data_file = pickle.load(open(args.save_path, 'rb')) 76 | plot_distribution(data_file) 77 | 78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /api/models/Multitask_CNN.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import sys 3 | sys.path.append("..") 4 | from config import OPT 5 | import torch 6 | from config import MODEL_DIR 7 | from utils import Head,Model, BackBone 8 | from torch.autograd import Variable 9 | import torch.nn.functional as F 10 | import copy 11 | import numpy as np 12 | ################################################## Model: ResNet50 ############################################ 13 | 14 | class ResNet50(): 15 | def __init__(self, device): 16 | self._opt = copy.copy( OPT) 17 | self.device = device 18 | self._name = 'ResNet50' 19 | self._output_size_per_task = {'AU': self._opt.AU_label_size, 'EXPR': self._opt.EXPR_label_size, 'VA': self._opt.VA_label_size * self._opt.digitize_num} 20 | # create networks 21 | self._init_create_networks() 22 | 23 | def _init_create_networks(self): 24 | """ 25 | init current model according to sofar tasks 26 | """ 27 | backbone = BackBone(self._opt) 28 | output_sizes = [self._output_size_per_task[x] for x in self._opt.tasks] 29 | output_feature_dim = backbone.output_feature_dim 30 | classifiers = [Head(output_feature_dim, self._opt.hidden_size, output_sizes[i]) for i in range(len(self._opt.tasks))] 31 | classifiers = nn.ModuleList(classifiers) 32 | self.resnet50 = Model(backbone, classifiers, self._opt.tasks) 33 | self.to_device() 34 | def to_device(self): 35 | self.resnet50.to(self.device) 36 | def load(self, model_path): 37 | self.resnet50.load_state_dict(torch.load(model_path, map_location=self.device)) 38 | 39 | def set_eval(self): 40 | self.resnet50.eval() 41 | self._is_train = False 42 | 43 | def forward(self, input_image = None): 44 | assert self._is_train is False, "Model must be in eval mode" 45 | with torch.no_grad(): 46 | input_image = Variable(input_image) 47 | input_image = input_image.to(self.device) 48 | output = self.resnet50(input_image) 49 | out_dict = self._format_estimates(output['output']) 50 | out_dict_raw = dict([(key,output['output'][key]) for key in output['output'].keys()]) 51 | return out_dict, out_dict_raw 52 | def _format_estimates(self, output): 53 | estimates = {} 54 | for task in output.keys(): 55 | if task == 'AU': 56 | o = (torch.sigmoid(output['AU'].cpu())>0.5).type(torch.LongTensor) 57 | estimates['AU'] = o.numpy() 58 | elif task == 'EXPR': 59 | o = F.softmax(output['EXPR'].cpu(), dim=-1).argmax(-1).type(torch.LongTensor) 60 | estimates['EXPR'] = o.numpy() 61 | elif task == 'VA': 62 | N = self._opt.digitize_num 63 | v = F.softmax(output['VA'][:, :N].cpu(), dim=-1).numpy() 64 | a = F.softmax(output['VA'][:, N:].cpu(), dim=-1).numpy() 65 | bins = np.linspace(-1, 1, num=self._opt.digitize_num) 66 | v = (bins * v).sum(-1) 67 | a = (bins * a).sum(-1) 68 | estimates['VA'] = np.stack([v, a], axis = 1) 69 | return estimates 70 | -------------------------------------------------------------------------------- /Multitask-CNN/torchsampler/imbalanced_ML.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.utils.data 3 | import torchvision 4 | from tqdm import tqdm 5 | import numpy as np 6 | import random 7 | def IRLbl(labels): 8 | # imbalance ratio per label 9 | # Args: 10 | # labels is a 2d numpy array, each row is one instance, each column is one class; the array contains (0, 1) only 11 | N, C = labels.shape 12 | pos_nums_per_label = np.sum(labels, axis=0) 13 | max_pos_nums = np.max(pos_nums_per_label) 14 | return max_pos_nums/pos_nums_per_label 15 | 16 | def MeanIR(labels): 17 | IRLbl_VALUE = IRLbl(labels) 18 | return np.mean(IRLbl_VALUE) 19 | 20 | class ImbalancedDatasetSampler_ML(torch.utils.data.sampler.Sampler): 21 | """Samples elements randomly from a given list of indices for imbalanced dataset 22 | Arguments: 23 | indices (list, optional): a list of indices 24 | num_samples (int, optional): number of samples to draw 25 | callback_get_label func: a callback-like function which takes two arguments - dataset and index 26 | """ 27 | 28 | def __init__(self, dataset, indices=None, num_samples=None, Preset_MeanIR_value= 2., 29 | max_clone_percentage=50, sample_size=32): 30 | 31 | # if indices is not provided, 32 | # all elements in the dataset will be considered 33 | self.indices = list(range(len(dataset))) \ 34 | if indices is None else indices 35 | 36 | # if num_samples is not provided, 37 | # draw `len(indices)` samples in each iteration 38 | self.num_samples = len(self.indices) \ 39 | if num_samples is None else num_samples 40 | 41 | all_labels = dataset._get_all_label() 42 | MeanIR_value = MeanIR(all_labels) if Preset_MeanIR_value ==0 else Preset_MeanIR_value 43 | IRLbl_value = IRLbl(all_labels) 44 | N, C = all_labels.shape 45 | indices_per_class = {} 46 | minority_classes = [] 47 | maxSamplesToClone = N / 100 * max_clone_percentage 48 | for i in range(C): 49 | ids = all_labels[:,i] == 1 50 | indices_per_class[i] = [ii for ii, x in enumerate(ids) if x ] 51 | if IRLbl_value[i] > MeanIR_value: 52 | minority_classes.append(i) 53 | new_all_labels = all_labels 54 | oversampled_ids = [] 55 | for i in minority_classes: 56 | while True: 57 | pick_id = list(np.random.choice(indices_per_class[i], sample_size)) 58 | indices_per_class[i].extend(pick_id) 59 | # recalculate the IRLbl_value 60 | new_all_labels = np.concatenate([new_all_labels, all_labels[pick_id]], axis=0) 61 | oversampled_ids.extend(pick_id) 62 | if IRLbl(new_all_labels)[i] <= MeanIR_value or len(oversampled_ids)>=maxSamplesToClone : 63 | break 64 | print("oversample length:{}".format(len(oversampled_ids)), end='\r') 65 | if len(oversampled_ids) >=maxSamplesToClone: 66 | break 67 | oversampled_ids = np.array(oversampled_ids) 68 | weights = np.array([1.0/len(self.indices)] * len(self.indices)) 69 | unique, counts = np.unique(oversampled_ids, return_counts=True) 70 | for i, n in zip(unique, counts): 71 | weights[i] = weights[i]*n 72 | self.weights = torch.DoubleTensor(weights) 73 | 74 | def __iter__(self): 75 | return (self.indices[i] for i in torch.multinomial( 76 | self.weights, self.num_samples, replacement=True)) 77 | 78 | def __len__(self): 79 | return self.num_samples 80 | 81 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/torchsampler/imbalanced_ML.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.utils.data 3 | import torchvision 4 | from tqdm import tqdm 5 | import numpy as np 6 | import random 7 | def IRLbl(labels): 8 | # imbalance ratio per label 9 | # Args: 10 | # labels is a 2d numpy array, each row is one instance, each column is one class; the array contains (0, 1) only 11 | N, C = labels.shape 12 | pos_nums_per_label = np.sum(labels, axis=0) 13 | max_pos_nums = np.max(pos_nums_per_label) 14 | return max_pos_nums/pos_nums_per_label 15 | 16 | def MeanIR(labels): 17 | IRLbl_VALUE = IRLbl(labels) 18 | return np.mean(IRLbl_VALUE) 19 | 20 | class ImbalancedDatasetSampler_ML(torch.utils.data.sampler.Sampler): 21 | """Samples elements randomly from a given list of indices for imbalanced dataset 22 | Arguments: 23 | indices (list, optional): a list of indices 24 | num_samples (int, optional): number of samples to draw 25 | callback_get_label func: a callback-like function which takes two arguments - dataset and index 26 | """ 27 | 28 | def __init__(self, dataset, indices=None, num_samples=None, Preset_MeanIR_value= 2., 29 | max_clone_percentage=50, sample_size=32): 30 | 31 | # if indices is not provided, 32 | # all elements in the dataset will be considered 33 | self.indices = list(range(len(dataset))) \ 34 | if indices is None else indices 35 | 36 | # if num_samples is not provided, 37 | # draw `len(indices)` samples in each iteration 38 | self.num_samples = len(self.indices) \ 39 | if num_samples is None else num_samples 40 | 41 | all_labels = dataset._get_all_label() 42 | MeanIR_value = MeanIR(all_labels) if Preset_MeanIR_value ==0 else Preset_MeanIR_value 43 | IRLbl_value = IRLbl(all_labels) 44 | N, C = all_labels.shape 45 | indices_per_class = {} 46 | minority_classes = [] 47 | maxSamplesToClone = N / 100 * max_clone_percentage 48 | for i in range(C): 49 | ids = all_labels[:,i] == 1 50 | indices_per_class[i] = [ii for ii, x in enumerate(ids) if x ] 51 | if IRLbl_value[i] > MeanIR_value: 52 | minority_classes.append(i) 53 | new_all_labels = all_labels 54 | oversampled_ids = [] 55 | for i in minority_classes: 56 | while True: 57 | pick_id = list(np.random.choice(indices_per_class[i], sample_size)) 58 | indices_per_class[i].extend(pick_id) 59 | # recalculate the IRLbl_value 60 | new_all_labels = np.concatenate([new_all_labels, all_labels[pick_id]], axis=0) 61 | oversampled_ids.extend(pick_id) 62 | if IRLbl(new_all_labels)[i] <= MeanIR_value or len(oversampled_ids)>=maxSamplesToClone : 63 | break 64 | print("oversample length:{}".format(len(oversampled_ids)), end='\r') 65 | if len(oversampled_ids) >=maxSamplesToClone: 66 | break 67 | oversampled_ids = np.array(oversampled_ids) 68 | weights = np.array([1.0/len(self.indices)] * len(self.indices)) 69 | unique, counts = np.unique(oversampled_ids, return_counts=True) 70 | for i, n in zip(unique, counts): 71 | weights[i] = weights[i]*n 72 | self.weights = torch.DoubleTensor(weights) 73 | 74 | def __iter__(self): 75 | return (self.indices[i] for i in torch.multinomial( 76 | self.weights, self.num_samples, replacement=True)) 77 | 78 | def __len__(self): 79 | return self.num_samples 80 | 81 | -------------------------------------------------------------------------------- /create_annotation_file/Aff-wild2/create_test_set_file.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import os 3 | import numpy as np 4 | import argparse 5 | from matplotlib import pyplot as plt 6 | import matplotlib 7 | import glob 8 | import pandas as pd 9 | from tqdm import tqdm 10 | import cv2 11 | parser = argparse.ArgumentParser(description='save annotations') 12 | parser.add_argument('--au_txt', type=str, default = 'aus_test_set.txt') 13 | parser.add_argument('--expr_txt', type=str, default = 'expression_test_set.txt') 14 | parser.add_argument('--va_txt', type=str, default = 'va_test_set.txt') 15 | parser.add_argument('--data_dir', type=str, default= '../cropped_aligned') 16 | parser.add_argument('--video_dir', type=str, default = '../videos') 17 | 18 | args = parser.parse_args() 19 | 20 | def read_txt(txt_file): 21 | with open(txt_file, 'r') as f: 22 | videos = f.readlines() 23 | videos = [x.strip() for x in videos] 24 | return videos 25 | 26 | def refine_frames_paths(frames, length): 27 | if (len(frames) > length) and np.abs(len(frames) - length) ==1: 28 | length = len(frames) 29 | frames_ids = [int(frame.split('/')[-1].split('.')[0]) - 1 for frame in frames] 30 | if len(frames) == length: 31 | return frames 32 | else: 33 | extra_frame_ids = [] 34 | prev_frame_id = frames_ids[0] 35 | for i in range(length): 36 | if i not in frames_ids: 37 | extra_frame_ids.append(prev_frame_id) 38 | else: 39 | prev_frame_id = i 40 | frames_ids.extend(extra_frame_ids) 41 | frames_ids = sorted(frames_ids) 42 | prefix = '/'.join(frames[0].split('/')[:-1]) 43 | return_frames = [prefix+'/{0:05d}.jpg'.format(id+1) for id in frames_ids] 44 | return return_frames 45 | def main(): 46 | tasks = ['AU_Set', 'EXPR_Set', 'VA_Set'] 47 | data_file = {} 48 | for task in tasks: 49 | data_file[task] = {} 50 | mode = 'Test_Set' 51 | txt_file = {'AU_Set':args.au_txt, "EXPR_Set": args.expr_txt, 'VA_Set':args.va_txt}[task] 52 | videos = read_txt(txt_file) 53 | data_file[task][mode] = {} 54 | for video in tqdm(videos): 55 | name = video 56 | frames_paths = sorted(glob.glob(os.path.join(args.data_dir, name, '*.jpg'))) 57 | if '_left' in name: 58 | video_name = name[:-5] 59 | elif '_right' in name: 60 | video_name = name[:-6] 61 | else: 62 | video_name = name 63 | video_path = glob.glob(os.path.join(args.video_dir, 'batch*', video_name+".*"))[0] 64 | cap = cv2.VideoCapture(video_path) 65 | length = int(cap.get(7)) + 1 66 | frames_paths = refine_frames_paths(frames_paths, length) 67 | length = len(frames_paths) 68 | if task == 'AU_Set': 69 | AU_list = ['AU1','AU2','AU4','AU6','AU12','AU15','AU20','AU25'] 70 | au_array = np.zeros((length, len(AU_list))) 71 | data_dict = {'label': au_array} 72 | data_dict.update({'path': frames_paths, 'frames_ids': np.arange(length)}) 73 | elif task == 'EXPR_Set': 74 | Expr_list = ['Neutral','Anger','Disgust','Fear','Happiness','Sadness','Surprise'] 75 | data_dict = {'label': np.zeros(length), 'path':frames_paths, 'frames_ids': np.arange(length)} 76 | elif task == 'VA_Set': 77 | va_label = np.zeros((length, 2)) 78 | data_dict = {'label':va_label, 'path':frames_paths, 'frames_ids': np.arange(length)} 79 | data_file[task][mode][name] = data_dict 80 | save_path = os.path.join('.', 'test_set.pkl') 81 | pickle.dump(data_file, open(save_path, 'wb')) 82 | if __name__== '__main__': 83 | main() 84 | -------------------------------------------------------------------------------- /Multitask-CNN/data/dataset_Mixed_EXPR.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import torchvision.transforms as transforms 3 | from .dataset import DatasetBase 4 | from PIL import Image 5 | import random 6 | import numpy as np 7 | import pickle 8 | import pandas as pd 9 | from PATH import PATH 10 | PRESET_VARS = PATH() 11 | 12 | class dataset_Mixed_EXPR(DatasetBase): 13 | def __init__(self, opt, train_mode='Train', transform = None): 14 | super(dataset_Mixed_EXPR, self).__init__(opt, train_mode, transform) 15 | self._name = 'dataset_Mixed_EXPR' 16 | self._train_mode = train_mode 17 | if transform is not None: 18 | self._transform = transform 19 | # read dataset 20 | self._read_dataset_paths() 21 | def __getitem__(self, index): 22 | assert (index < self._dataset_size) 23 | # start_time = time.time() 24 | image = None 25 | label = None 26 | img_path = self._data['path'][index] 27 | image = Image.open(img_path).convert('RGB') 28 | label = self._data['label'][index] 29 | image = self._transform(image) 30 | # pack data 31 | sample = {'image': image, 32 | 'label': label, 33 | 'path': img_path, 34 | 'index': index 35 | } 36 | # print (time.time() - start_time) 37 | return sample 38 | def _read_dataset_paths(self): 39 | if not self._train_mode == 'Test': 40 | self._data = self._read_path_label(PRESET_VARS.Mixed_EXPR.data_file) 41 | else: 42 | self._data = self._read_path_label(PRESET_VARS.Aff_wild2.test_data_file) 43 | self._ids = np.arange(len(self._data['label'])) 44 | self._dataset_size = len(self._ids) 45 | def __len__(self): 46 | return self._dataset_size 47 | def _read_path_label(self, file_path): 48 | data = pickle.load(open(file_path, 'rb')) 49 | # read frames ids 50 | if self._train_mode == 'Train': 51 | data = data['Training_Set'] 52 | elif self._train_mode == 'Validation': 53 | data = data['Validation_Set'] 54 | elif self._train_mode == 'Test': 55 | data = data['Test_Set'] 56 | else: 57 | raise ValueError("train mode must be in : Train, Validation, Test") 58 | return data 59 | 60 | def _create_transform(self): 61 | if self._train_mode == 'Train': 62 | img_size = self._opt.image_size 63 | resize = int(img_size * 1.2) 64 | transform_list = [transforms.Resize(resize), 65 | transforms.RandomCrop(img_size), 66 | transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3), 67 | transforms.RandomHorizontalFlip(), 68 | transforms.ToTensor(), 69 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 70 | std=[0.229, 0.224, 0.225]), 71 | ] 72 | else: 73 | img_size = self._opt.image_size 74 | transform_list = [transforms.Resize(img_size), 75 | transforms.ToTensor(), 76 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 77 | std=[0.229, 0.224, 0.225]), 78 | ] 79 | self._transform = transforms.Compose(transform_list) 80 | -------------------------------------------------------------------------------- /api/models/Multitask_CNN_RNN.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch 3 | import sys 4 | sys.path.append("..") 5 | from config import OPT 6 | from config import MODEL_DIR 7 | from utils import Head, GRU_Head, Seq_Model, BackBone,Model 8 | from torch.autograd import Variable 9 | import torch.nn.functional as F 10 | import copy 11 | import numpy as np 12 | ############################################ CNN-RNN ########################################33 13 | 14 | class ResNet50_GRU(): 15 | def __init__(self, device): 16 | self._opt = copy.copy( OPT) 17 | self.device = device 18 | self._name = 'ResNet50_GRU' 19 | self._output_size_per_task = {'AU': self._opt.AU_label_size, 'EXPR': self._opt.EXPR_label_size, 'VA': self._opt.VA_label_size * self._opt.digitize_num} 20 | # create networks 21 | self._init_create_networks() 22 | def _init_create_networks(self): 23 | """ 24 | init current model according to sofar tasks 25 | """ 26 | backbone = BackBone(self._opt) 27 | output_sizes = [self._output_size_per_task[x] for x in self._opt.tasks] 28 | output_feature_dim = backbone.output_feature_dim 29 | classifiers = [Head(output_feature_dim, self._opt.hidden_size, output_sizes[i]) for i in range(len(self._opt.tasks))] 30 | classifiers = nn.ModuleList(classifiers) 31 | resnet50 = Model(backbone, classifiers, self._opt.tasks) 32 | # create GRUs 33 | GRU_classifiers = [GRU_Head(self._opt.hidden_size, self._opt.hidden_size//2, output_sizes[i]) for i in range(len(self._opt.tasks))] 34 | GRU_classifiers = nn.ModuleList(GRU_classifiers) 35 | self.resnet50_GRU = Seq_Model(resnet50, GRU_classifiers, self._opt.tasks) 36 | self.to_device() 37 | def to_device(self): 38 | self.resnet50_GRU.to(self.device) 39 | def load(self, model_path): 40 | self.resnet50_GRU.load_state_dict(torch.load(model_path, map_location=self.device)) 41 | def set_eval(self): 42 | self.resnet50_GRU.eval() 43 | self._is_train = False 44 | 45 | def forward(self, input_image = None): 46 | assert self._is_train is False, "Model must be in eval mode" 47 | with torch.no_grad(): 48 | input_image = Variable(input_image) 49 | input_image = input_image.to(self.device) 50 | output = self.resnet50_GRU(input_image) 51 | out_dict = self._format_estimates(output['output']) 52 | out_dict_raw = dict([(key, output['output'][key]) for key in output['output'].keys()]) 53 | return out_dict, out_dict_raw 54 | 55 | def _format_estimates(self, output): 56 | estimates = {} 57 | for task in output.keys(): 58 | if task == 'AU': 59 | o = (torch.sigmoid(output['AU'].cpu())>0.5).type(torch.LongTensor) 60 | estimates['AU'] = o.numpy() 61 | elif task == 'EXPR': 62 | o = F.softmax(output['EXPR'].cpu(), dim=-1).argmax(-1).type(torch.LongTensor) 63 | estimates['EXPR'] = o.numpy() 64 | elif task == 'VA': 65 | N = self._opt.digitize_num 66 | v = F.softmax(output['VA'][:,:, :N].cpu(), dim=-1).numpy() 67 | a = F.softmax(output['VA'][:,:, N:].cpu(), dim=-1).numpy() 68 | bins = np.linspace(-1, 1, num=self._opt.digitize_num) 69 | v = (bins * v).sum(-1) 70 | a = (bins * a).sum(-1) 71 | estimates['VA'] = np.stack([v, a], axis = -1) 72 | return estimates 73 | -------------------------------------------------------------------------------- /Multitask-CNN/data/dataset_Mixed_AU.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import torchvision.transforms as transforms 3 | from .dataset import DatasetBase 4 | from PIL import Image 5 | import random 6 | import numpy as np 7 | import pickle 8 | import pandas as pd 9 | from PATH import PATH 10 | PRESET_VARS = PATH() 11 | 12 | class dataset_Mixed_AU(DatasetBase): 13 | def __init__(self, opt, train_mode='Train', transform = None): 14 | super(dataset_Mixed_AU, self).__init__(opt, train_mode, transform) 15 | self._name = 'dataset_Mixed_AU' 16 | self._train_mode = train_mode 17 | if transform is not None: 18 | self._transform = transform 19 | # read dataset 20 | self._read_dataset_paths() 21 | def _get_all_label(self): 22 | return self._data['label'] 23 | def __getitem__(self, index): 24 | assert (index < self._dataset_size) 25 | # start_time = time.time() 26 | image = None 27 | label = None 28 | img_path = self._data['path'][index] 29 | image = Image.open(img_path).convert('RGB') 30 | label = self._data['label'][index] 31 | image = self._transform(image) 32 | # pack data 33 | sample = {'image': image, 34 | 'label': label, 35 | 'path': img_path, 36 | 'index': index 37 | } 38 | # print (time.time() - start_time) 39 | return sample 40 | def _read_dataset_paths(self): 41 | if not self._train_mode == 'Test': 42 | self._data = self._read_path_label(PRESET_VARS.Mixed_AU.data_file) 43 | else: 44 | self._data = self._read_path_label(PRESET_VARS.Aff_wild2.test_data_file) 45 | self._ids = np.arange(len(self._data['label'])) 46 | self._dataset_size = len(self._ids) 47 | def __len__(self): 48 | return self._dataset_size 49 | def _read_path_label(self, file_path): 50 | data = pickle.load(open(file_path, 'rb')) 51 | # read frames ids 52 | if self._train_mode == 'Train': 53 | data = data['Training_Set'] 54 | elif self._train_mode == 'Validation': 55 | data = data['Validation_Set'] 56 | elif self._train_mode == 'Test': 57 | data = data['Test_Set'] 58 | else: 59 | raise ValueError("train mode must be in : Train, Validation, Test") 60 | return data 61 | 62 | def _create_transform(self): 63 | if self._train_mode == 'Train': 64 | img_size = self._opt.image_size 65 | resize = int(img_size * 1.2) 66 | transform_list = [transforms.Resize(resize), 67 | transforms.RandomCrop(img_size), 68 | transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3), 69 | transforms.RandomHorizontalFlip(), 70 | transforms.ToTensor(), 71 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 72 | std=[0.229, 0.224, 0.225]), 73 | ] 74 | else: 75 | img_size = self._opt.image_size 76 | transform_list = [transforms.Resize(img_size), 77 | transforms.ToTensor(), 78 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 79 | std=[0.229, 0.224, 0.225]), 80 | ] 81 | self._transform = transforms.Compose(transform_list) -------------------------------------------------------------------------------- /create_annotation_file/DISFA/read_annotations.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import numpy as np 4 | import argparse 5 | from matplotlib import pyplot as plt 6 | import matplotlib 7 | import glob 8 | np.random.seed(0) 9 | parser = argparse.ArgumentParser(description='save annotations') 10 | parser.add_argument('--vis', action = 'store_true', 11 | help='whether to visualize the distribution') 12 | parser.add_argument('--annot_dir', type=str, default = 'ActionUnit_Labels', 13 | help='annotation dir') 14 | parser.add_argument("--image_dir", type=str, default= '/media/Samsung/DISFA/Videos_LeftCamera_OpenFace_Aligned') 15 | 16 | args = parser.parse_args() 17 | def read_au(txt_prefix): 18 | au_list = [1, 2, 4, 6, 12, 15, 20, 25] 19 | aus = [] 20 | for i in au_list: 21 | txt_file = txt_prefix+"{}.txt".format(i) 22 | with open(txt_file, 'r') as f: 23 | lines = f.readlines() 24 | lines = [x.strip() for x in lines] 25 | lines = [int(x.split(',')[1]) for x in lines] 26 | lines = [0 if x<2 else 1 for x in lines ] # if intensity is equal to or greater than 2, it is positive sample 27 | aus.append(lines) 28 | aus = np.stack(aus, axis=1) 29 | return aus 30 | 31 | def plot_pie(AU_list, pos_freq, neg_freq): 32 | ploting_labels = [x+'+ {0:.2f}'.format(y) for x, y in zip(AU_list, pos_freq)] + [x+'- {0:.2f}'.format(y) for x, y in zip(AU_list, neg_freq)] 33 | cmap = matplotlib.cm.get_cmap('coolwarm') 34 | colors = [cmap(x) for x in pos_freq] + [cmap(x) for x in neg_freq] 35 | fracs = np.ones(len(AU_list)*2) 36 | plt.pie(fracs, labels=ploting_labels, autopct=None, shadow=False, colors=colors,startangle =78.75) 37 | plt.title("AUs distribution") 38 | plt.show() 39 | AU_list = ['AU1','AU2','AU4','AU6','AU12','AU15','AU20','AU25'] 40 | annot_dir = args.annot_dir 41 | data_file = {} 42 | videos = sorted(os.listdir(annot_dir)) 43 | # in total 27 videos 44 | ids = np.random.permutation(len(videos)) 45 | videos = [videos[i] for i in ids] 46 | train_videos = videos[:21] 47 | val_videos = videos[21:] 48 | data_file['Training_Set'] = {} 49 | data_file['Validation_Set'] = {} 50 | for video in train_videos: 51 | aus = read_au(annot_dir+'/{}/{}_au'.format(video, video)) 52 | frames = sorted(glob.glob(os.path.join(args.image_dir, "LeftVideo"+video+"_comp", "LeftVideo"+video+"_comp_aligned", '*.bmp'))) 53 | frames_id = [int(x.split("/")[-1].split(".")[0].split("_")[-1]) -1 for x in frames] 54 | assert len(aus)>=len(frames) 55 | frames = [frames[id] for id in frames_id] 56 | aus = aus[frames_id] 57 | data_file['Training_Set'][video] = {'label':aus, 'path':frames} 58 | for video in val_videos: 59 | aus = read_au(annot_dir+'/{}/{}_au'.format(video, video)) 60 | frames = sorted(glob.glob(os.path.join(args.image_dir, "LeftVideo"+video+"_comp", "LeftVideo"+video+"_comp_aligned", '*.bmp'))) 61 | frames_id = [int(x.split("/")[-1].split(".")[0].split("_")[-1]) -1 for x in frames] 62 | assert len(aus)>= len(frames) 63 | frames = [frames[id] for id in frames_id] 64 | aus = aus[frames_id] 65 | data_file['Validation_Set'][video] = {'label':aus, 'path':frames} 66 | 67 | if args.vis: 68 | total_dict = {**data_file['Training_Set'], **data_file['Validation_Set']} 69 | all_samples = np.concatenate([total_dict[x]['label'] for x in total_dict.keys()], axis=0) 70 | pos_freq = np.sum(all_samples, axis=0)/all_samples.shape[0] 71 | neg_freq = -np.sum(all_samples-1, axis=0)/all_samples.shape[0] 72 | print("pos_weight:", neg_freq/pos_freq) 73 | plot_pie(AU_list, pos_freq, neg_freq) 74 | save_path = 'annotations.pkl' 75 | pickle.dump(data_file, open(save_path, 'wb')) 76 | 77 | -------------------------------------------------------------------------------- /Multitask-CNN/data/dataset_Mixed_VA.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import torchvision.transforms as transforms 3 | from .dataset import DatasetBase 4 | from PIL import Image 5 | import random 6 | import numpy as np 7 | import pickle 8 | import pandas as pd 9 | from PATH import PATH 10 | PRESET_VARS = PATH() 11 | 12 | class dataset_Mixed_VA(DatasetBase): 13 | def __init__(self, opt, train_mode='Train', transform = None): 14 | super(dataset_Mixed_VA, self).__init__(opt, train_mode, transform) 15 | self._name = 'dataset_Mixed_VA' 16 | self._train_mode = train_mode 17 | if transform is not None: 18 | self._transform = transform 19 | # read dataset 20 | self._read_dataset_paths() 21 | def _get_all_label(self): 22 | return self._data['label'] 23 | def __getitem__(self, index): 24 | assert (index < self._dataset_size) 25 | # start_time = time.time() 26 | image = None 27 | label = None 28 | img_path = self._data['path'][index] 29 | image = Image.open(img_path).convert('RGB') 30 | label = self._data['label'][index] 31 | image = self._transform(image) 32 | # pack data 33 | sample = {'image': image, 34 | 'label': label, 35 | 'path': img_path, 36 | 'index': index 37 | } 38 | # print (time.time() - start_time) 39 | return sample 40 | def _read_dataset_paths(self): 41 | if not self._train_mode == 'Test': 42 | self._data = self._read_path_label(PRESET_VARS.Mixed_VA.data_file) 43 | else: 44 | self._data = self._read_path_label(PRESET_VARS.Aff_wild2.test_data_file) 45 | self._ids = np.arange(len(self._data['label'])) 46 | self._dataset_size = len(self._ids) 47 | def __len__(self): 48 | return self._dataset_size 49 | def _read_path_label(self, file_path): 50 | data = pickle.load(open(file_path, 'rb')) 51 | # read frames ids 52 | if self._train_mode == 'Train': 53 | data = data['Training_Set'] 54 | elif self._train_mode == 'Validation': 55 | data = data['Validation_Set'] 56 | elif self._train_mode == 'Test': 57 | data = data['Test_Set'] 58 | else: 59 | raise ValueError("train mode must be in : Train, Validation, Test") 60 | return data 61 | 62 | def _create_transform(self): 63 | if self._train_mode == 'Train': 64 | img_size = self._opt.image_size 65 | resize = int(img_size * 1.2) 66 | transform_list = [transforms.Resize(resize), 67 | transforms.RandomCrop(img_size), 68 | transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3), 69 | transforms.RandomHorizontalFlip(), 70 | transforms.ToTensor(), 71 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 72 | std=[0.229, 0.224, 0.225]), 73 | ] 74 | else: 75 | img_size = self._opt.image_size 76 | transform_list = [transforms.Resize(img_size), 77 | transforms.ToTensor(), 78 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 79 | std=[0.229, 0.224, 0.225]), 80 | ] 81 | self._transform = transforms.Compose(transform_list) 82 | -------------------------------------------------------------------------------- /create_annotation_file/AFEW-VA/create_annotation_files_Mixed_VA.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import os 3 | import numpy as np 4 | import argparse 5 | from matplotlib import pyplot as plt 6 | import matplotlib 7 | import glob 8 | import pandas as pd 9 | from PIL import Image 10 | from tqdm import tqdm 11 | parser = argparse.ArgumentParser(description='read two annotations files') 12 | parser.add_argument('--aff_wild2_pkl', type=str, default = '/media/Samsung/Aff-wild2-Challenge/annotations/annotations.pkl') 13 | parser.add_argument('--VA_pkl', type=str, default = '/media/Samsung/AFEW_VA/annotations.pkl') 14 | parser.add_argument('--save_path', type=str, default='/media/Samsung/Aff-wild2-Challenge/exps/create_new_training_set_VA/create_annotation_file/mixed_VA_annotations.pkl') 15 | args = parser.parse_args() 16 | VA_list = ['valence', 'arousal'] 17 | def read_aff_wild2(): 18 | total_data = pickle.load(open(args.aff_wild2_pkl, 'rb')) 19 | # training set 20 | train_data = total_data['VA_Set']['Training_Set'] 21 | paths = [] 22 | labels = [] 23 | for video in train_data.keys(): 24 | data = train_data[video] 25 | labels.append(np.stack([data['valence'], data['arousal']], axis=1)) 26 | paths.append(data['path'].values) 27 | paths = np.concatenate(paths, axis=0) 28 | labels = np.concatenate(labels, axis=0) 29 | train_data = {'label': labels, 'path': paths} 30 | # validation set 31 | val_data = total_data['VA_Set']['Validation_Set'] 32 | paths = [] 33 | labels = [] 34 | for video in val_data.keys(): 35 | data = val_data[video] 36 | labels.append(np.stack([data['valence'], data['arousal']], axis=1)) 37 | paths.append(data['path'].values) 38 | paths = np.concatenate(paths, axis=0) 39 | labels = np.concatenate(labels, axis=0) 40 | val_data = {'label':labels, 'path':paths} 41 | return train_data, val_data 42 | def merge_two_datasets(): 43 | data_aff_wild2, data_aff_wild2_val = read_aff_wild2() 44 | # downsample x 5 the training set in aff_wild training set 45 | aff_wild_train_labels = data_aff_wild2['label'] 46 | aff_wild_train_paths = data_aff_wild2['path'] 47 | length = len(aff_wild_train_labels) 48 | index = [True if i%5 ==0 else False for i in range(length)] 49 | aff_wild_train_labels = aff_wild_train_labels[index] 50 | aff_wild_train_paths = aff_wild_train_paths[index] 51 | data_aff_wild2 = {'label':aff_wild_train_labels, 'path':aff_wild_train_paths} 52 | # downsample x 5 the training set in aff_wild 53 | data_VA = pickle.load(open(args.VA_pkl, 'rb')) 54 | data_VA = {**data_VA['Training_Set'], **data_VA['Validation_Set']} 55 | labels =[] 56 | paths = [] 57 | for video in data_VA.keys(): 58 | data = data_VA[video] 59 | labels.append(np.stack([data['valence'], data['arousal']], axis=1)) 60 | paths.append(data['path']) 61 | paths = np.concatenate(paths, axis=0) 62 | labels = np.concatenate(labels, axis=0) 63 | data_VA = {'label':labels, 'path':paths} 64 | data_merged = {'label': np.concatenate((data_aff_wild2['label'], data_VA['label']), axis=0), 65 | 'path': list(data_aff_wild2['path']) + list(data_VA['path'])} 66 | print("Aff-wild2 :{}".format(len(data_aff_wild2['label']))) 67 | print("AFEW_VA:{}".format(len(data_VA['label']))) 68 | return {'Training_Set': data_merged, 'Validation_Set': data_aff_wild2_val} 69 | 70 | def plot_distribution(data): 71 | all_samples = data['label'] 72 | plt.hist2d(all_samples[:, 0] , all_samples[:, 1] , bins=(20, 20), cmap=plt.cm.jet) 73 | plt.xlabel("Valence") 74 | plt.ylabel('Arousal') 75 | plt.colorbar() 76 | plt.show() 77 | if __name__== '__main__': 78 | data_file = merge_two_datasets() 79 | pickle.dump(data_file, open(args.save_path, 'wb')) 80 | plot_distribution(data_file['Training_Set']) -------------------------------------------------------------------------------- /create_annotation_file/ExpW/create_annotation_files_Mixed_EXPR.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import os 3 | import numpy as np 4 | import argparse 5 | from matplotlib import pyplot as plt 6 | import matplotlib 7 | import glob 8 | import pandas as pd 9 | from PIL import Image 10 | from tqdm import tqdm 11 | parser = argparse.ArgumentParser(description='read two annotations files') 12 | parser.add_argument('--aff_wild2_pkl', type=str, default = '/media/Samsung/Aff-wild2-Challenge/annotations/annotations.pkl') 13 | parser.add_argument('--ExpW_pkl', type=str, default = '/media/Samsung/ExpW_dataset/annotations.pkl') 14 | parser.add_argument('--save_path', type=str, default='/media/Samsung/Aff-wild2-Challenge/exps/create_new_training_set_EXPR/create_annotation_file/mixed_EXPR_annotations.pkl') 15 | args = parser.parse_args() 16 | Expr_list = ['Neutral','Anger','Disgust','Fear','Happiness','Sadness','Surprise'] 17 | def read_aff_wild2(): 18 | total_data = pickle.load(open(args.aff_wild2_pkl, 'rb')) 19 | # training set 20 | data = total_data['EXPR_Set']['Training_Set'] 21 | paths = [] 22 | labels = [] 23 | for video in data.keys(): 24 | df = data[video] 25 | labels.append(df['label'].values.astype(np.float32)) 26 | paths.append(df['path'].values) 27 | # undersample the neutral samples by 10 28 | # undersample the happy and sad samples by 20 29 | paths = np.concatenate(paths, axis=0) 30 | labels = np.concatenate(labels, axis=0) 31 | # neutral 32 | keep_10 = np.array([True if i%10==0 else False for i in range(len(labels))]) 33 | to_drop = labels == 0 34 | to_drop = to_drop * (~keep_10) 35 | labels = labels[~to_drop] 36 | paths = paths[~to_drop] 37 | # happy 38 | keep_2 = np.array([True if i%2==0 else False for i in range(len(labels))]) 39 | to_drop = labels == 4 40 | to_drop = to_drop * (~keep_2) 41 | labels = labels[~to_drop] 42 | paths = paths[~to_drop] 43 | # sadness 44 | keep_2 = np.array([True if i%2==0 else False for i in range(len(labels))]) 45 | to_drop = labels == 5 46 | to_drop = to_drop * (~keep_2) 47 | labels = labels[~to_drop] 48 | paths = paths[~to_drop] 49 | data = {'label': labels, 'path': paths} 50 | # validation set 51 | val_data = total_data['EXPR_Set']['Validation_Set'] 52 | paths = [] 53 | labels = [] 54 | for video in val_data.keys(): 55 | df = val_data[video] 56 | labels.append(df['label'].values.astype(np.float32)) 57 | paths.append(df['path'].values) 58 | # undersample the neutral samples by 10 59 | # undersample the happy and sad samples by 20 60 | paths = np.concatenate(paths, axis=0) 61 | labels = np.concatenate(labels, axis=0) 62 | val_data = {'label':labels, 'path':paths} 63 | return data, val_data 64 | def merge_two_datasets(): 65 | data_aff_wild2, data_aff_wild2_val = read_aff_wild2() 66 | data_ExpW = pickle.load(open(args.ExpW_pkl, 'rb')) 67 | # change the label integer, because of the different labelling in two datasets 68 | ExpW_to_aff_wild2 = [1, 2, 3, 4, 5, 6, 0] 69 | data_ExpW['label'] = np.array([ExpW_to_aff_wild2[x] for x in data_ExpW['label']]) 70 | data_merged = {'label': np.concatenate((data_aff_wild2['label'], data_ExpW['label']), axis=0), 71 | 'path': list(data_aff_wild2['path']) +data_ExpW['path']} 72 | print("Dataset\t"+"\t".join(Expr_list)) 73 | print("Aff_wild2 dataset:\t" +"\t".join([str(sum(data_aff_wild2['label']==i)) for i in range(len(Expr_list))])) 74 | print("ExpW dataset:\t" +"\t".join([str(sum(data_ExpW['label']==i)) for i in range(len(Expr_list))])) 75 | return {'Training_Set': data_merged, 'Validation_Set': data_aff_wild2_val} 76 | 77 | def plot_distribution(data): 78 | all_samples = data['label'] 79 | histogram = np.zeros(len(Expr_list)) 80 | for i in range(len(Expr_list)): 81 | find_true = sum(all_samples==i) 82 | histogram[i] =find_true/all_samples.shape[0] 83 | plt.bar(np.arange(len(Expr_list)), histogram) 84 | plt.xticks(np.arange(len(Expr_list)), Expr_list) 85 | plt.show() 86 | if __name__== '__main__': 87 | #data_file = read_all_image() 88 | data_file = merge_two_datasets() 89 | pickle.dump(data_file, open(args.save_path, 'wb')) 90 | plot_distribution(data_file['Training_Set']) -------------------------------------------------------------------------------- /api/readme.md: -------------------------------------------------------------------------------- 1 | # Multitask Emotion Recognition api 2 | --- 3 | This repository contain all scripts that needed for running Multi-task Emotion Recognition api for video files. Using this this api, you are able to get three types of emotion predictions, including 8 FAUs(Facial Action Units), 7 basic facial expressions and VA (Valence and Arousal) of each frame of facial images in the input video. 4 | 5 | # Get Started 6 | 7 | These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system. 8 | 9 | ## Prerequisites 10 | My recommendation is to use Anaconda to create an virtual environment. For example: 11 | ``` 12 | conda create --name myenv python=3.6 13 | ``` 14 | 15 | And then activate the virtual environment by: 16 | ``` 17 | conda activate myenv 18 | ``` 19 | 20 | - CUDA 21 | My recommendation is to used Anaconda to install cudatoolkit 10.1, torch and torchvision: 22 | ``` 23 | conda install pytorch torchvision cudatoolkit=10.1 24 | ``` 25 | 26 | - OpenFace 27 | 28 | If you have already installed OpenFace, you can skip the installation below. You only need to replace pass the executable file path `FeatureExtraction` to `EmotionAPI()`. 29 | 30 | To install OpenFace in the root directory of this project: 31 | ``` 32 | git clone https://github.com/TadasBaltrusaitis/OpenFace.git 33 | cd OpenFace 34 | ``` 35 | Then download the needed models by: 36 | ``` 37 | bash ./download_models.sh 38 | ``` 39 | It's better to install some dependencies required by OpenCV before you run 'install.sh': 40 | ``` 41 | sudo apt-get update 42 | sudo apt-get upgrade 43 | sudo apt-get install build-essential cmake pkg-config # Install developer tools used to compile OpenCV 44 | sudo apt-get install libjpeg8-dev libtiff5-dev libjasper-dev libpng12-dev # Install libraries and packages used to read various image formats from disk 45 | sudo apt-get install libavcodec-dev libavformat-dev libswscale-dev libv4l-dev 46 | sudo apt-get install libxvidcore-dev libx264-dev # Install a few libraries used to read video formats from disk 47 | ``` 48 | And then install OpenFace by: 49 | ``` 50 | sudo bash ./install.sh 51 | ``` 52 | If this shell script fails to install it correctly, you can follow the instruction [here](https://github.com/TadasBaltrusaitis/OpenFace/wiki/Unix-Installation#advanced-ubuntu-installation-if-not-using-installsh-or-if-it-fails). 53 | 54 | - pytorch-benchmarks 55 | Download pytorch-benchmarks in the root directory of your project: 56 | ``` 57 | git clone https://github.com/albanie/pytorch-benchmarks.git 58 | ``` 59 | Download the three folders in this [link](https://hkustconnect-my.sharepoint.com/:f:/g/personal/ddeng_connect_ust_hk/ElE4ifQSxLBCgtfQLvgE5Z8B8wGo-SpoUJc-So_3FruMcg?e=LL3u6V) and save the three folders under `pytorch-benchmarks/models`. 60 | 61 | - Other requirements 62 | 63 | Install other requirements by: 64 | ``` 65 | pip install -r requirements.txt 66 | ``` 67 | 68 | - Download pretrained models 69 | 70 | Download pretrained Multitask CNN models and Multitask CNN-RNN models by: 71 | 72 | ``` 73 | bash download_pretrained_weights_aff_wild2.sh 74 | ``` 75 | 76 | # How to use 77 | There is an example script `run_example.py`. 78 | 79 | Run 80 | ``` 81 | python run_example.py 82 | ``` 83 | The prediction results will be saved to `predictions.csv`. 84 | 85 | Example: 86 | ``` 87 | frames_ids AU1 AU2 AU4 AU6 ... AU20 AU25 EXPR valence arousal 88 | 0 1 0 0 1 1 ... 0 0 4 0.499485 0.522120 89 | 1 2 0 0 1 1 ... 0 0 4 0.488518 0.504697 90 | 2 3 0 0 1 1 ... 0 0 4 0.445970 0.469693 91 | 3 4 0 0 1 1 ... 0 0 4 0.421242 0.448395 92 | 4 5 0 0 1 1 ... 0 0 4 0.430781 0.450103 93 | 94 | [5 rows x 12 columns] 95 | 96 | ``` 97 | 98 | # TODO 99 | - To generate a video will where three types of emotion predictions are drawn. 100 | -------------------------------------------------------------------------------- /api/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import matplotlib.colors as mcolors 3 | import argparse 4 | import matplotlib 5 | matplotlib.use('TkAgg') 6 | font = {'family' : 'normal', 7 | 'size' : 24} 8 | matplotlib.rc('font', **font) 9 | MODEL_DIR = os.path.abspath('./pytorch-benchmarks/models/') 10 | PRETRAINED_MODEL_DIR = os.path.abspath('./pretrained_models/') 11 | CATEGORIES = {'AU': ['AU1', 'AU2', 'AU4', 'AU6', 'AU12', 'AU15', 'AU20', 'AU25'], 12 | 'EXPR':['Neutral','Anger','Disgust','Fear','Happiness','Sadness','Surprise'], 13 | 'VA':['valence', 'arousal']} 14 | Best_AU_Thresholds = {'CNN': [0.1448537, 0.03918985, 0.13766725, 0.02652811, 0.40589422, 0.15572545,0.04808964, 0.10848708], 15 | 'CNN_RNN': {32: [0.4253935, 0.02641966, 0.1119782, 0.02978198, 0.17256933, 0.06369855, 0.07433069, 0.13828614], 16 | 16: [0.30485213, 0.09509478, 0.59577084, 0.4417419, 0.4396544, 0.0452404,0.05204154, 0.0633798 ], 17 | 8: [0.4365209 ,0.10177602, 0.2649502, 0.22586018, 0.3772219, 0.07532539, 0.07667687, 0.04306327]}} 18 | tasks = ['AU','EXPR','VA'] 19 | 20 | __OPT = {"AU_label_size":8 , 21 | "EXPR_label_size":7, 22 | "VA_label_size":2, 23 | "digitize_num": 20, 24 | "hidden_size": 128, 25 | "image_size": 112, 26 | "tasks": ['EXPR','AU','VA'], 27 | "pretrained_dataset": 'ferplus'} 28 | class AttrDict(dict): 29 | def __init__(self, *args, **kwargs): 30 | super(AttrDict, self).__init__(*args, **kwargs) 31 | self.__dict__ = self 32 | OPT = AttrDict(__OPT) 33 | # parser = argparse.ArgumentParser(description='Implement the pretrained model on your own data') 34 | # parser.add_argument('--image_dir', type=str, 35 | # help='a directory containing a sequence of cropped and aligned face images') 36 | # parser.add_argument('--model_type', type=str, default='CNN', choices= ['CNN', 'CNN-RNN'], 37 | # help='By default, the CNN pretrained models are stored in "Multitask-CNN", and the CNN-RNN \ 38 | # pretrained models are stored in "Multitask-CNN-RNN"') 39 | # parser.add_argument('--seq_len', type=int, default = 32, choices = [32, 16, 8], help='sequence length when the model type is CNN-RNN') 40 | # parser.add_argument('--image_ext', default = ['.jpg', '.bmp', '.png'], help='image extentions') 41 | # parser.add_argument('--eval_with_teacher', action='store_true', help='whether to predict with teacher model') 42 | # parser.add_argument('--eval_with_students', action='store_true', help='whether to predict with student models') 43 | # parser.add_argument('--ensemble', action='store_true', help='whether to merge the student predictions') 44 | # parser.add_argument('--AU_label_size', type=int, default = 8, help='# of AUs') 45 | # parser.add_argument('--EXPR_label_size', type=int, default = 7, help='# of EXpressions') 46 | # parser.add_argument('--VA_label_size', type=int, default = 2, help='# of VA ') 47 | # parser.add_argument('--digitize_num', type=int, default= 20, choices = [1, 20], help='1 means no digitization,\ 48 | # 20 means to digitize continuous label to 20 one hot vector ') 49 | # parser.add_argument('--hidden_size', type=int, default = 128, help='the embedding size of each output head' ) 50 | # parser.add_argument('--image_size', type=int, default= 112, help='input image size') 51 | # parser.add_argument('--batch_size', type=int, default= 20, help='input batch size per task') 52 | # parser.add_argument('--gpu_ids', type=str, default='0', help='gpu ids: e.g. 0 0,1,2, 0,2. use -1 for CPU') 53 | # parser.add_argument('--workers', type=int, default=0, help='number of workers') 54 | # parser.add_argument('--tasks', type=str, default = ['EXPR','AU','VA'],nargs="+") 55 | # parser.add_argument('--save_dir', type=str, help='where to save the predictions') 56 | # parser.add_argument('--pretrained_dataset', type=str, default='ferplus', 57 | # choices = ['ferplus', 'sfew','imagenet'], 58 | # help="the pretrained_dataset of the face feature extractor, choices:['ferplus', 'sfew','imagenet']") 59 | # opt = parser.parse_args() -------------------------------------------------------------------------------- /Multitask-CNN-RNN/data/custom_dataset_data_loader.py: -------------------------------------------------------------------------------- 1 | import torch.utils.data 2 | from data.dataset import DatasetFactory 3 | from torchsampler.imbalanced_VA import ImbalancedDatasetSampler_VA 4 | from collections import OrderedDict 5 | 6 | class Multitask_Iterator_Wrapper: 7 | def __init__(self, multitask_dataloader): 8 | self.multitask_dataloader = multitask_dataloader 9 | self.dataloaders = OrderedDict([(k, iter(x)) for (k, x) in self.multitask_dataloader.items()]) 10 | self._index = 0 11 | self.max_n_iters = min([len(x) for (k,x) in self.dataloaders.items()]) 12 | def __iter__(self): 13 | return self 14 | def reset(self): 15 | self.dataloaders = OrderedDict([(k, iter(x)) for (k, x) in self.multitask_dataloader.items()]) 16 | self._index = 0 17 | self.max_n_iters = min([len(x) for (k,x) in self.dataloaders.items()]) 18 | def __next__(self): 19 | if self._index < self.max_n_iters: 20 | data_batch = dict() 21 | for (task, dataloader_iter) in self.dataloaders.items(): 22 | batch_per_task = next(dataloader_iter) 23 | data_batch[task] = batch_per_task 24 | self._index +=1 25 | return data_batch 26 | else: 27 | raise StopIteration 28 | def __len__(self): 29 | return self.max_n_iters 30 | 31 | class Multitask_DatasetDataLoader: 32 | def __init__(self, opt, train_mode, transform=None): 33 | self._opt = opt 34 | self.train_mode = train_mode 35 | self.transform = transform 36 | self._is_train = self.train_mode == 'Train' 37 | self._num_threds = opt.n_threads_train if self._is_train else opt.n_threads_test 38 | self._create_datasets() 39 | self._create_dataloaders() 40 | @staticmethod 41 | def cumsum(sequence): 42 | r, s = [], 0 43 | for e in sequence: 44 | l = len(e) 45 | r.append(l + s) 46 | s+=l 47 | return r 48 | def _create_datasets(self): 49 | self.datasets = OrderedDict() 50 | for i, dataset_name in enumerate(self._opt.dataset_names): 51 | task = self._opt.tasks[i] 52 | self.datasets[task] = DatasetFactory.get_by_name(dataset_name, self._opt, self.train_mode, self.transform) 53 | self.cumulative_sizes = self.cumsum([dataset for (k, dataset) in self.datasets.items()]) # number of instances, cumulative sizes 54 | 55 | def _create_dataloaders(self): 56 | self.dataloaders = OrderedDict() 57 | for i, dataset_name in enumerate(self._opt.dataset_names): 58 | task = self._opt.tasks[i] 59 | if (not self._is_train): 60 | dataloader = torch.utils.data.DataLoader( 61 | self.datasets[task], 62 | batch_size=self._opt.batch_size, 63 | shuffle= self._is_train, 64 | num_workers=int(self._num_threds), 65 | drop_last = self._is_train) 66 | else: 67 | # if self._opt.force_balance: 68 | # from torchsampler.imbalanced_sampler import SamplerFactory 69 | # sampler = SamplerFactory.get_by_name(dataset_name, self.datasets[task]) 70 | # dataloader = torch.utils.data.DataLoader( 71 | # self.datasets[task], 72 | # sampler = sampler, 73 | # shuffle= False, 74 | # batch_size = self._opt.batch_size, 75 | # num_workers=int(self._num_threds), 76 | # drop_last = self._is_train) 77 | # else: 78 | dataloader = torch.utils.data.DataLoader( 79 | self.datasets[task], 80 | batch_size=self._opt.batch_size, 81 | shuffle= self._is_train, 82 | num_workers=int(self._num_threds), 83 | drop_last = self._is_train) 84 | self.dataloaders[task] = dataloader 85 | 86 | def load_multitask_train_data(self): 87 | assert self._is_train 88 | return Multitask_Iterator_Wrapper(self.dataloaders) 89 | def load_multitask_val_test_data(self): 90 | return self.dataloaders 91 | def __len__(self): 92 | return min([len(x) for (k,x) in self.dataloaders.iteritems()]) 93 | 94 | 95 | -------------------------------------------------------------------------------- /Multitask-CNN/data/custom_dataset_data_loader.py: -------------------------------------------------------------------------------- 1 | import torch.utils.data 2 | from data.dataset import DatasetFactory 3 | from torchsampler.imbalanced_VA import ImbalancedDatasetSampler_VA 4 | from collections import OrderedDict 5 | 6 | class Multitask_Iterator_Wrapper: 7 | def __init__(self, multitask_dataloader): 8 | self.multitask_dataloader = multitask_dataloader 9 | self.dataloaders = OrderedDict([(k, iter(x)) for (k, x) in self.multitask_dataloader.items()]) 10 | self._index = 0 11 | self.max_n_iters = min([len(x) for (k,x) in self.dataloaders.items()]) 12 | def __iter__(self): 13 | return self 14 | def reset(self): 15 | self.dataloaders = OrderedDict([(k, iter(x)) for (k, x) in self.multitask_dataloader.items()]) 16 | self._index = 0 17 | self.max_n_iters = min([len(x) for (k,x) in self.dataloaders.items()]) 18 | def __next__(self): 19 | if self._index < self.max_n_iters: 20 | data_batch = dict() 21 | for (task, dataloader_iter) in self.dataloaders.items(): 22 | batch_per_task = next(dataloader_iter) 23 | data_batch[task] = batch_per_task 24 | self._index +=1 25 | return data_batch 26 | else: 27 | raise StopIteration 28 | def __len__(self): 29 | return self.max_n_iters 30 | 31 | class Multitask_DatasetDataLoader: 32 | def __init__(self, opt, train_mode, transform=None): 33 | self._opt = opt 34 | self.train_mode = train_mode 35 | self.transform = transform 36 | self._is_train = self.train_mode == 'Train' 37 | self._num_threds = opt.n_threads_train if self._is_train else opt.n_threads_test 38 | self._create_datasets() 39 | self._create_dataloaders() 40 | @staticmethod 41 | def cumsum(sequence): 42 | r, s = [], 0 43 | for e in sequence: 44 | l = len(e) 45 | r.append(l + s) 46 | s+=l 47 | return r 48 | def _create_datasets(self): 49 | self.datasets = OrderedDict() 50 | for i, dataset_name in enumerate(self._opt.dataset_names): 51 | task = self._opt.tasks[i] 52 | self.datasets[task] = DatasetFactory.get_by_name(dataset_name, self._opt, self.train_mode, self.transform) 53 | self.cumulative_sizes = self.cumsum([dataset for (k, dataset) in self.datasets.items()]) # number of instances, cumulative sizes 54 | 55 | def _create_dataloaders(self): 56 | self.dataloaders = OrderedDict() 57 | for i, dataset_name in enumerate(self._opt.dataset_names): 58 | task = self._opt.tasks[i] 59 | if (not self._is_train): 60 | dataloader = torch.utils.data.DataLoader( 61 | self.datasets[task], 62 | batch_size=self._opt.batch_size * len(self._opt.tasks), # the largest batch size for validation 63 | shuffle= self._is_train, 64 | num_workers=int(self._num_threds), 65 | drop_last = self._is_train) 66 | else: 67 | if self._opt.force_balance: 68 | from torchsampler.imbalanced_sampler import SamplerFactory 69 | sampler = SamplerFactory.get_by_name(dataset_name, self.datasets[task]) 70 | dataloader = torch.utils.data.DataLoader( 71 | self.datasets[task], 72 | sampler = sampler, 73 | shuffle= False, 74 | batch_size = self._opt.batch_size, 75 | num_workers=int(self._num_threds), 76 | drop_last = self._is_train) 77 | else: 78 | dataloader = torch.utils.data.DataLoader( 79 | self.datasets[task], 80 | batch_size=self._opt.batch_size, 81 | shuffle= self._is_train, 82 | num_workers=int(self._num_threds), 83 | drop_last = self._is_train) 84 | self.dataloaders[task] = dataloader 85 | 86 | def load_multitask_train_data(self): 87 | assert self._is_train 88 | return Multitask_Iterator_Wrapper(self.dataloaders) 89 | def load_multitask_val_test_data(self): 90 | return self.dataloaders 91 | def __len__(self): 92 | return min([len(x) for (k,x) in self.dataloaders.iteritems()]) 93 | 94 | 95 | -------------------------------------------------------------------------------- /api/video_processor.py: -------------------------------------------------------------------------------- 1 | import os 2 | class Video_Processor(object): 3 | def __init__(self, size=112, nomask=True, grey=False, quiet=True, 4 | tracked_vid=False, noface_save=False, 5 | OpenFace_exe = 'OpenFace/build/bin/FeatureExtraction'): 6 | ''' Video Processor using OpenFace to do face detection and face alignment 7 | Given an input video, this processor will create a directory where all cropped and aligned 8 | faces are saved. 9 | 10 | Parameters: 11 | size: int, default 112 12 | The output faces will be saved as images where the width and height are size pixels. 13 | nomask: bool, default True 14 | If True, the output face image will not be masked (mask the region except the face). 15 | Otherwise, the output face image will be masked, not containing background. 16 | grey: bool, default False 17 | If True, the output face image will be saved as greyscale images instead of RGB images. 18 | quiet: bool, default False 19 | If False, will print out the processing steps live. 20 | tracked_vid: bool, default False 21 | If True, will save the tracked video, which is an output video with detected landmarks. 22 | noface_save: bool, default False 23 | If True, those frames where face detection is failed will be saved (blank image); 24 | else those failed frames will not saved. 25 | OpenFace_exe: String, default is 'OpenFace/build/bin/FeatureExtraction' 26 | By default, the OpenFace library is installed in the same directory as Video_Processor. 27 | It can be changed to the current OpenFace executable file. 28 | ''' 29 | self.size = size 30 | self.nomask = nomask 31 | self.grey = grey 32 | self.quiet = quiet 33 | self.tracked_vid = tracked_vid 34 | self.noface_save = noface_save 35 | self.OpenFace_exe = OpenFace_exe 36 | if not isinstance(self.OpenFace_exe, str) or not os.path.exists(self.OpenFace_exe): 37 | raise ValueError("OpenFace_exe has to be string object and needs to exist.") 38 | self.OpenFace_exe = os.path.abspath(self.OpenFace_exe) 39 | def process(self, input_video, output_dir=None): 40 | ''' 41 | Arguments: 42 | input_video: String 43 | The input video path, or the input sequence directory, where each image representing a frame, e.g. 001.jpg, 002.jpg, 003.jpg ... 200.jpg 44 | output_dir: String, default None 45 | The output faces will be saved in output_dir. By default the output_dir will be in 46 | the same parent directory as the input video is. 47 | ''' 48 | 49 | if not isinstance(input_video, str) or not os.path.exists(input_video): 50 | raise ValueError("input video has to be string object and needs to exist.") 51 | if os.path.isdir(input_video): 52 | assert len(os.listdir(input_video))>0, "Input sequence directory {} cannot be empty".format(input_video) 53 | arg_input = '-fdir' 54 | else: 55 | arg_input = '-f' 56 | 57 | input_video = os.path.abspath(input_video) 58 | if output_dir is None: 59 | output_dir = os.path.join(os.path.dirname(input_video), 60 | os.path.basename(input_video).split('.')[0]) 61 | if isinstance(output_dir, str): 62 | if not os.path.exists(output_dir): 63 | os.makedirs(output_dir) 64 | else: 65 | print("output dir exists: {}. Video processing skipped.".format(output_dir)) 66 | return 67 | else: 68 | raise ValueError("output_dir should be string object.") 69 | opface_option = " {} ".format(arg_input)+input_video + " -out_dir "+ output_dir +" -simsize "+ str(self.size) 70 | opface_option += " -2Dfp -3Dfp -pdmparams -pose -aus -gaze -simalign " 71 | 72 | if not self.noface_save: 73 | opface_option +=" -nobadaligned " 74 | if self.tracked_vid: 75 | opface_option +=" -tracked " 76 | if self.nomask: 77 | opface_option+= " -nomask" 78 | if self.grey: 79 | opface_option += " -g" 80 | if self.quiet: 81 | opface_option += " -q" 82 | # execution 83 | call = self.OpenFace_exe + opface_option 84 | os.system(call) 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/data/test_video_dataset.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import torchvision.transforms as transforms 3 | from .dataset import DatasetBase 4 | from PIL import Image 5 | import random 6 | import numpy as np 7 | import pickle 8 | import pandas as pd 9 | from PATH import PATH 10 | import torch 11 | PRESET_VARS = PATH() 12 | 13 | class Test_dataset(object): 14 | def __init__(self, opt, video_data, train_mode='Train', transform = None): 15 | self._name = 'Test_dataset' 16 | self._opt = opt 17 | self._train_mode = train_mode 18 | if transform is not None: 19 | self._transform = transform 20 | else: 21 | self._transform = self._create_transform() 22 | # read dataset 23 | self._data = video_data 24 | self._read_dataset() 25 | 26 | def __getitem__(self, index): 27 | assert (index < self._dataset_size) 28 | # start_time = time.time() 29 | images = [] 30 | labels = [] 31 | img_paths = [] 32 | frames_ids = [] 33 | df = self.sample_seqs[index] 34 | for i, row in df.iterrows(): 35 | img_path = row['path'] 36 | image = Image.open(img_path).convert('RGB') 37 | image = self._transform(image) 38 | label = row[[str(i) for i in range(self._label_size)]].values.astype(np.float32) 39 | if self._label_size == 1: 40 | label = label.squeeze().astype(np.int64) #EXPR 41 | frame_id = row['frames_ids'] 42 | images.append(image) 43 | labels.append(label) 44 | img_paths.append(img_path) 45 | frames_ids.append(frame_id) 46 | # pack data 47 | sample = {'image': torch.stack(images,dim=0), 48 | 'label': np.array(labels), 49 | 'path': img_paths, 50 | 'index': index, 51 | 'frames_ids':frames_ids 52 | } 53 | # print (time.time() - start_time) 54 | return sample 55 | def _read_dataset(self): 56 | #sample them 57 | seq_len = self._opt.seq_len 58 | self.sample_seqs = [] 59 | N = seq_len 60 | data = {'path': self._data['path'], 'frames_ids':self._data['frames_ids']} # dataframe are easier for indexing 61 | self._label_size = self._data['label'].shape[1] if len(self._data['label'].shape)==2 else 1 62 | num_images = self._data['label'].shape[0] 63 | data.update(dict([(str(i), self._data['label'].reshape(num_images, -1)[:, i:(i+1)].squeeze()) \ 64 | for i in range(self._label_size)])) 65 | self._data = pd.DataFrame.from_dict(data) 66 | for i in range(len(self._data['path'])//N + 1): 67 | start, end = i*N, i*N + seq_len 68 | if end >= len(self._data): 69 | start, end = len(self._data) - seq_len, len(self._data) 70 | new_df = self._data.iloc[start:end] 71 | if not len(new_df) == seq_len: 72 | assert len(new_df) < seq_len 73 | count = seq_len - len(new_df) 74 | for _ in range(count): 75 | new_df = new_df.append(new_df.iloc[-1]) 76 | assert len(new_df) == seq_len 77 | self.sample_seqs.append(new_df) 78 | self._ids = np.arange(len(self.sample_seqs)) 79 | self._dataset_size = len(self._ids) 80 | def __len__(self): 81 | return self._dataset_size 82 | def _create_transform(self): 83 | if self._train_mode == 'Train': 84 | img_size = self._opt.image_size 85 | resize = int(img_size * 1.2) 86 | transform_list = [transforms.Resize(resize), 87 | transforms.RandomCrop(img_size), 88 | transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3), 89 | transforms.RandomHorizontalFlip(), 90 | transforms.ToTensor(), 91 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 92 | std=[0.229, 0.224, 0.225]), 93 | ] 94 | else: 95 | img_size = self._opt.image_size 96 | transform_list = [transforms.Resize(img_size), 97 | transforms.ToTensor(), 98 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 99 | std=[0.229, 0.224, 0.225]), 100 | ] 101 | self._transform = transforms.Compose(transform_list) 102 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/data/dataset_Mixed_EXPR.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import torchvision.transforms as transforms 3 | from .dataset import DatasetBase 4 | from PIL import Image 5 | import random 6 | import numpy as np 7 | import pickle 8 | import pandas as pd 9 | import torch 10 | from PATH import PATH 11 | PRESET_VARS = PATH() 12 | 13 | class dataset_Mixed_EXPR(DatasetBase): 14 | def __init__(self, opt, train_mode='Train', transform = None): 15 | super(dataset_Mixed_EXPR, self).__init__(opt, train_mode, transform) 16 | self._name = 'dataset_Mixed_EXPR' 17 | self._train_mode = train_mode 18 | if transform is not None: 19 | self._transform = transform 20 | # read dataset 21 | self._read_dataset_paths() 22 | def __getitem__(self, index): 23 | assert (index < self._dataset_size) 24 | # start_time = time.time() 25 | images = [] 26 | labels = [] 27 | img_paths = [] 28 | frames_ids = [] 29 | df = self.sample_seqs[index] 30 | for i, row in df.iterrows(): 31 | img_path = row['path'] 32 | image = Image.open(img_path).convert('RGB') 33 | image = self._transform(image) 34 | label = row['label'] 35 | frame_id = row['frames_ids'] 36 | images.append(image) 37 | labels.append(label) 38 | img_paths.append(img_path) 39 | frames_ids.append(frame_id) 40 | 41 | # pack data 42 | sample = {'image': torch.stack(images,dim=0), 43 | 'label': np.array(labels), 44 | 'path': img_paths, 45 | 'index': index, 46 | 'id':frames_ids 47 | } 48 | # print (time.time() - start_time) 49 | return sample 50 | def _read_dataset_paths(self): 51 | self._data = self._read_path_label(PRESET_VARS.Aff_wild2.data_file) 52 | #sample them 53 | seq_len = self._opt.seq_len 54 | self.sample_seqs = [] 55 | if self._train_mode == 'Train': 56 | N = seq_len//2 57 | else: 58 | N = seq_len 59 | for video in self._data.keys(): 60 | data = self._data[video] 61 | for i in range(len(data)//N): 62 | start, end = i*N, i*N + seq_len 63 | if end >= len(data): 64 | start, end = len(data) - seq_len, len(data) 65 | new_df = data.iloc[start:end] 66 | if not len(new_df) == seq_len: 67 | assert len(new_df) < seq_len 68 | count = seq_len - len(new_df) 69 | for _ in range(count): 70 | new_df = new_df.append(new_df.iloc[-1]) 71 | assert len(new_df) == seq_len 72 | self.sample_seqs.append(new_df) 73 | self._ids = np.arange(len(self.sample_seqs)) 74 | self._dataset_size = len(self._ids) 75 | def __len__(self): 76 | return self._dataset_size 77 | def _read_path_label(self, file_path): 78 | data = pickle.load(open(file_path, 'rb')) 79 | data = data['EXPR_Set'] 80 | # read frames ids 81 | if self._train_mode == 'Train': 82 | data = data['Training_Set'] 83 | elif self._train_mode == 'Validation': 84 | data = data['Validation_Set'] 85 | else: 86 | raise ValueError("train mode must be in : Train, Validation") 87 | 88 | return data 89 | 90 | def _create_transform(self): 91 | if self._train_mode == 'Train': 92 | img_size = self._opt.image_size 93 | resize = int(img_size * 1.2) 94 | transform_list = [transforms.Resize(resize), 95 | transforms.RandomCrop(img_size), 96 | transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3), 97 | transforms.RandomHorizontalFlip(), 98 | transforms.ToTensor(), 99 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 100 | std=[0.229, 0.224, 0.225]), 101 | ] 102 | else: 103 | img_size = self._opt.image_size 104 | transform_list = [transforms.Resize(img_size), 105 | transforms.ToTensor(), 106 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 107 | std=[0.229, 0.224, 0.225]), 108 | ] 109 | self._transform = transforms.Compose(transform_list) 110 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/data/dataset_Mixed_AU.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import torchvision.transforms as transforms 3 | from .dataset import DatasetBase 4 | from PIL import Image 5 | import random 6 | import numpy as np 7 | import pickle 8 | import pandas as pd 9 | from PATH import PATH 10 | import torch 11 | PRESET_VARS = PATH() 12 | 13 | class dataset_Mixed_AU(DatasetBase): 14 | def __init__(self, opt, train_mode='Train', transform = None): 15 | super(dataset_Mixed_AU, self).__init__(opt, train_mode, transform) 16 | self._name = 'dataset_Mixed_AU' 17 | self._train_mode = train_mode 18 | if transform is not None: 19 | self._transform = transform 20 | # read dataset 21 | self._read_dataset_paths() 22 | def _get_all_label(self): 23 | return self._data['label'] 24 | def __getitem__(self, index): 25 | assert (index < self._dataset_size) 26 | # start_time = time.time() 27 | images = [] 28 | labels = [] 29 | img_paths = [] 30 | frames_ids = [] 31 | df = self.sample_seqs[index] 32 | for i,row in df.iterrows(): 33 | img_path = row['path'] 34 | image = Image.open(img_path).convert('RGB') 35 | image = self._transform(image) 36 | label = row[PRESET_VARS.Aff_wild2.categories['AU']].values.astype(np.float32) 37 | frame_id = row['frames_ids'] 38 | images.append(image) 39 | labels.append(label) 40 | img_paths.append(img_path) 41 | frames_ids.append(frame_id) 42 | # pack data 43 | sample = {'image': torch.stack(images,dim=0), 44 | 'label': np.array(labels), 45 | 'path': img_paths, 46 | 'index': index, 47 | 'id':frames_ids 48 | } 49 | # print (time.time() - start_time) 50 | return sample 51 | def _read_dataset_paths(self): 52 | self._data = self._read_path_label(PRESET_VARS.Aff_wild2.data_file) 53 | #sample them 54 | seq_len = self._opt.seq_len 55 | self.sample_seqs = [] 56 | if self._train_mode == 'Train': 57 | N = seq_len//2 58 | else: 59 | N = seq_len 60 | for video in self._data.keys(): 61 | data = self._data[video] 62 | for i in range(len(data)//N): 63 | start, end = i*N, i*N + seq_len 64 | if end >= len(data): 65 | start, end = len(data) - seq_len, len(data) 66 | new_df = data.iloc[start:end] 67 | if not len(new_df) == seq_len: 68 | assert len(new_df) < seq_len 69 | count = seq_len - len(new_df) 70 | for _ in range(count): 71 | new_df = new_df.append(new_df.iloc[-1]) 72 | assert len(new_df) == seq_len 73 | self.sample_seqs.append(new_df) 74 | self._ids = np.arange(len(self.sample_seqs)) 75 | self._dataset_size = len(self._ids) 76 | 77 | def __len__(self): 78 | return self._dataset_size 79 | def _read_path_label(self, file_path): 80 | data = pickle.load(open(file_path, 'rb')) 81 | data = data['AU_Set'] 82 | # read frames ids 83 | if self._train_mode == 'Train': 84 | data = data['Training_Set'] 85 | elif self._train_mode == 'Validation': 86 | data = data['Validation_Set'] 87 | else: 88 | raise ValueError("train mode must be in : Train, Validation") 89 | return data 90 | 91 | def _create_transform(self): 92 | if self._train_mode == 'Train': 93 | img_size = self._opt.image_size 94 | resize = int(img_size * 1.2) 95 | transform_list = [transforms.Resize(resize), 96 | transforms.RandomCrop(img_size), 97 | transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3), 98 | transforms.RandomHorizontalFlip(), 99 | transforms.ToTensor(), 100 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 101 | std=[0.229, 0.224, 0.225]), 102 | ] 103 | else: 104 | img_size = self._opt.image_size 105 | transform_list = [transforms.Resize(img_size), 106 | transforms.ToTensor(), 107 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 108 | std=[0.229, 0.224, 0.225]), 109 | ] 110 | self._transform = transforms.Compose(transform_list) -------------------------------------------------------------------------------- /Multitask-CNN-RNN/data/dataset_Mixed_VA.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import torchvision.transforms as transforms 3 | from .dataset import DatasetBase 4 | from PIL import Image 5 | import random 6 | import numpy as np 7 | import pickle 8 | import pandas as pd 9 | from PATH import PATH 10 | import torch 11 | PRESET_VARS = PATH() 12 | 13 | class dataset_Mixed_VA(DatasetBase): 14 | def __init__(self, opt, train_mode='Train', transform = None): 15 | super(dataset_Mixed_VA, self).__init__(opt, train_mode, transform) 16 | self._name = 'dataset_Mixed_VA' 17 | self._train_mode = train_mode 18 | if transform is not None: 19 | self._transform = transform 20 | # read dataset 21 | self._read_dataset_paths() 22 | def _get_all_label(self): 23 | return self._data['label'] 24 | def __getitem__(self, index): 25 | assert (index < self._dataset_size) 26 | # start_time = time.time() 27 | images = [] 28 | labels = [] 29 | img_paths = [] 30 | frames_ids = [] 31 | df = self.sample_seqs[index] 32 | for i, row in df.iterrows(): 33 | img_path = row['path'] 34 | image = Image.open(img_path).convert('RGB') 35 | image = self._transform(image) 36 | label = row[PRESET_VARS.Aff_wild2.categories['VA']].values.astype(np.float32) 37 | frame_id = row['frames_ids'] 38 | images.append(image) 39 | labels.append(label) 40 | img_paths.append(img_path) 41 | frames_ids.append(frame_id) 42 | # pack data 43 | sample = {'image': torch.stack(images,dim=0), 44 | 'label': np.array(labels), 45 | 'path': img_paths, 46 | 'index': index, 47 | 'id':frames_ids 48 | } 49 | # print (time.time() - start_time) 50 | return sample 51 | def _read_dataset_paths(self): 52 | self._data = self._read_path_label(PRESET_VARS.Aff_wild2.data_file) 53 | #sample them 54 | seq_len = self._opt.seq_len 55 | self.sample_seqs = [] 56 | if self._train_mode == 'Train': 57 | N = seq_len//2 58 | else: 59 | N = seq_len 60 | for video in self._data.keys(): 61 | data = self._data[video] 62 | for i in range(len(data)//N): 63 | start, end = i*N, i*N + seq_len 64 | if end >= len(data): 65 | start, end = len(data) - seq_len, len(data) 66 | new_df = data.iloc[start:end] 67 | if not len(new_df) == seq_len: 68 | assert len(new_df) < seq_len 69 | count = seq_len - len(new_df) 70 | for _ in range(count): 71 | new_df = new_df.append(new_df.iloc[-1]) 72 | assert len(new_df) == seq_len 73 | self.sample_seqs.append(new_df) 74 | self._ids = np.arange(len(self.sample_seqs)) 75 | self._dataset_size = len(self._ids) 76 | def __len__(self): 77 | return self._dataset_size 78 | def _read_path_label(self, file_path): 79 | data = pickle.load(open(file_path, 'rb')) 80 | data = data['VA_Set'] 81 | # read frames ids 82 | if self._train_mode == 'Train': 83 | data = data['Training_Set'] 84 | elif self._train_mode == 'Validation': 85 | data = data['Validation_Set'] 86 | else: 87 | raise ValueError("train mode must be in : Train, Validation") 88 | return data 89 | 90 | def _create_transform(self): 91 | if self._train_mode == 'Train': 92 | img_size = self._opt.image_size 93 | resize = int(img_size * 1.2) 94 | transform_list = [transforms.Resize(resize), 95 | transforms.RandomCrop(img_size), 96 | transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3), 97 | transforms.RandomHorizontalFlip(), 98 | transforms.ToTensor(), 99 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 100 | std=[0.229, 0.224, 0.225]), 101 | ] 102 | else: 103 | img_size = self._opt.image_size 104 | transform_list = [transforms.Resize(img_size), 105 | transforms.ToTensor(), 106 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 107 | std=[0.229, 0.224, 0.225]), 108 | ] 109 | self._transform = transforms.Compose(transform_list) 110 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/models/models.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import torch.nn.functional as F 4 | from torch.optim import lr_scheduler 5 | 6 | class ModelsFactory: 7 | def __init__(self, opt): 8 | self._opt = opt 9 | @staticmethod 10 | def get_by_name(model_name, *args, **kwargs): 11 | model = None 12 | if model_name == 'resnet50': 13 | from models.resnet50 import ResNet50 14 | model = ResNet50(*args, **kwargs) 15 | else: 16 | raise ValueError("Model %s not recognized." % model_name) 17 | print("Model %s was created" % model.name) 18 | return model 19 | 20 | class BaseModel(object): 21 | 22 | def __init__(self, opt): 23 | self._name = 'BaseModel' 24 | 25 | self._opt = opt 26 | self._gpu_ids = opt.gpu_ids 27 | self._is_train = opt.is_train 28 | 29 | self._Tensor = torch.cuda.FloatTensor if self._gpu_ids else torch.Tensor 30 | self._save_dir = os.path.join(opt.checkpoints_dir, opt.name) 31 | @property 32 | def name(self): 33 | return self._name 34 | 35 | @property 36 | def is_train(self): 37 | return self._is_train 38 | 39 | def set_input(self, input): 40 | assert False, "set_input not implemented" 41 | 42 | def set_train(self): 43 | assert False, "set_train not implemented" 44 | 45 | def set_eval(self): 46 | assert False, "set_eval not implemented" 47 | 48 | def forward(self, keep_data_for_visuals=False): 49 | assert False, "forward not implemented" 50 | 51 | # used in test time, no backprop 52 | def test(self): 53 | assert False, "test not implemented" 54 | 55 | def get_image_paths(self): 56 | return {} 57 | 58 | def optimize_parameters(self): 59 | assert False, "optimize_parameters not implemented" 60 | 61 | def get_current_visuals(self): 62 | return {} 63 | 64 | def get_current_errors(self): 65 | return {} 66 | 67 | def get_current_scalars(self): 68 | return {} 69 | 70 | def save(self, label): 71 | assert False, "save not implemented" 72 | 73 | def load(self): 74 | assert False, "load not implemented" 75 | 76 | def _save_optimizer(self, optimizer, optimizer_label, epoch_label): 77 | save_filename = 'opt_epoch_%s_id_%s.pth' % (epoch_label, optimizer_label) 78 | save_path = os.path.join(self._save_dir, save_filename) 79 | torch.save(optimizer.state_dict(), save_path) 80 | 81 | def _load_optimizer(self, optimizer, optimizer_label, epoch_label): 82 | load_filename = 'opt_epoch_%s_id_%s.pth' % (epoch_label, optimizer_label) 83 | load_path = os.path.join(self._save_dir, load_filename) 84 | assert os.path.exists( 85 | load_path), 'Weights file not found. Have you trained a model!? We are not providing one' % load_path 86 | 87 | optimizer.load_state_dict(torch.load(load_path)) 88 | print ('loaded optimizer: %s' % load_path) 89 | 90 | def _save_network(self, network, network_label, epoch_label): 91 | save_filename = 'net_epoch_%s_id_%s.pth' % (epoch_label, network_label) 92 | save_path = os.path.join(self._save_dir, save_filename) 93 | torch.save(network.state_dict(), save_path) 94 | print ('saved net: %s' % save_path) 95 | 96 | def _load_network(self, network, network_label, epoch_label): 97 | load_filename = 'net_epoch_%s_id_%s.pth' % (epoch_label, network_label) 98 | load_path = os.path.join(self._save_dir, load_filename) 99 | assert os.path.exists( 100 | load_path), 'Weights file not found. Have you trained a model!? We are not providing one' % load_path 101 | 102 | network.load_state_dict(torch.load(load_path)) 103 | print ('loaded net: %s' % load_path) 104 | 105 | def update_learning_rate(self): 106 | pass 107 | 108 | def print_network(self, network): 109 | num_params = 0 110 | for param in network.parameters(): 111 | num_params += param.numel() 112 | print(network) 113 | print('Total number of parameters: %d' % num_params) 114 | 115 | def _get_scheduler(self, optimizer, opt): 116 | if opt.lr_policy == 'lambda': 117 | def lambda_rule(epoch): 118 | lr_l = 1.0 - max(0, epoch + 1 + opt.epoch_count - opt.niter) / float(opt.niter_decay + 1) 119 | return lr_l 120 | scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_rule) 121 | elif opt.lr_policy == 'step': 122 | scheduler = lr_scheduler.StepLR(optimizer, step_size=opt.lr_decay_epochs, gamma=0.1) 123 | elif opt.lr_policy == 'plateau': 124 | scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.1, threshold=0.01, patience=2) 125 | else: 126 | return NotImplementedError('learning rate policy [%s] is not implemented', opt.lr_policy) 127 | return scheduler -------------------------------------------------------------------------------- /Multitask-CNN/models/models.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import torch.nn.functional as F 4 | from torch.optim import lr_scheduler 5 | 6 | class ModelsFactory: 7 | def __init__(self, opt): 8 | self._opt = opt 9 | @staticmethod 10 | def get_by_name(model_name, *args, **kwargs): 11 | model = None 12 | if model_name == 'resnet50': 13 | from models.resnet50 import ResNet50 14 | model = ResNet50(*args, **kwargs) 15 | else: 16 | raise ValueError("Model %s not recognized." % model_name) 17 | print("Model %s was created" % model.name) 18 | return model 19 | 20 | class BaseModel(object): 21 | 22 | def __init__(self, opt): 23 | self._name = 'BaseModel' 24 | 25 | self._opt = opt 26 | self._gpu_ids = opt.gpu_ids 27 | self._is_train = opt.is_train 28 | 29 | self._Tensor = torch.cuda.FloatTensor if self._gpu_ids else torch.Tensor 30 | self._save_dir = os.path.join(opt.checkpoints_dir, opt.name) 31 | @property 32 | def name(self): 33 | return self._name 34 | 35 | @property 36 | def is_train(self): 37 | return self._is_train 38 | 39 | def set_input(self, input): 40 | assert False, "set_input not implemented" 41 | 42 | def set_train(self): 43 | assert False, "set_train not implemented" 44 | 45 | def set_eval(self): 46 | assert False, "set_eval not implemented" 47 | 48 | def forward(self, keep_data_for_visuals=False): 49 | assert False, "forward not implemented" 50 | 51 | # used in test time, no backprop 52 | def test(self): 53 | assert False, "test not implemented" 54 | 55 | def get_image_paths(self): 56 | return {} 57 | 58 | def optimize_parameters(self): 59 | assert False, "optimize_parameters not implemented" 60 | 61 | def get_current_visuals(self): 62 | return {} 63 | 64 | def get_current_errors(self): 65 | return {} 66 | 67 | def get_current_scalars(self): 68 | return {} 69 | 70 | def save(self, label): 71 | assert False, "save not implemented" 72 | 73 | def load(self): 74 | assert False, "load not implemented" 75 | 76 | def _save_optimizer(self, optimizer, optimizer_label, epoch_label): 77 | save_filename = 'opt_epoch_%s_id_%s.pth' % (epoch_label, optimizer_label) 78 | save_path = os.path.join(self._save_dir, save_filename) 79 | torch.save(optimizer.state_dict(), save_path) 80 | 81 | def _load_optimizer(self, optimizer, optimizer_label, epoch_label): 82 | load_filename = 'opt_epoch_%s_id_%s.pth' % (epoch_label, optimizer_label) 83 | load_path = os.path.join(self._save_dir, load_filename) 84 | assert os.path.exists( 85 | load_path), 'Weights file not found. Have you trained a model!? We are not providing one' % load_path 86 | 87 | optimizer.load_state_dict(torch.load(load_path)) 88 | print ('loaded optimizer: %s' % load_path) 89 | 90 | def _save_network(self, network, network_label, epoch_label): 91 | save_filename = 'net_epoch_%s_id_%s.pth' % (epoch_label, network_label) 92 | save_path = os.path.join(self._save_dir, save_filename) 93 | torch.save(network.state_dict(), save_path) 94 | print ('saved net: %s' % save_path) 95 | 96 | def _load_network(self, network, network_label, epoch_label): 97 | load_filename = 'net_epoch_%s_id_%s.pth' % (epoch_label, network_label) 98 | load_path = os.path.join(self._save_dir, load_filename) 99 | assert os.path.exists( 100 | load_path), 'Weights file not found. Have you trained a model!? We are not providing one' % load_path 101 | 102 | network.load_state_dict(torch.load(load_path)) 103 | print ('loaded net: %s' % load_path) 104 | 105 | def update_learning_rate(self): 106 | pass 107 | 108 | def print_network(self, network): 109 | num_params = 0 110 | for param in network.parameters(): 111 | num_params += param.numel() 112 | print(network) 113 | print('Total number of parameters: %d' % num_params) 114 | 115 | def _get_scheduler(self, optimizer, opt): 116 | if opt.lr_policy == 'lambda': 117 | def lambda_rule(epoch): 118 | lr_l = 1.0 - max(0, epoch + 1 + opt.epoch_count - opt.niter) / float(opt.niter_decay + 1) 119 | return lr_l 120 | scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_rule) 121 | elif opt.lr_policy == 'step': 122 | scheduler = lr_scheduler.StepLR(optimizer, step_size=opt.lr_decay_epochs, gamma=0.1) 123 | elif opt.lr_policy == 'plateau': 124 | scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.1, threshold=0.01, patience=2) 125 | else: 126 | return NotImplementedError('learning rate policy [%s] is not implemented', opt.lr_policy) 127 | return scheduler 128 | -------------------------------------------------------------------------------- /create_annotation_file/DISFA/create_annotation_Mixed_AU.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import os 3 | import numpy as np 4 | import argparse 5 | from matplotlib import pyplot as plt 6 | import matplotlib 7 | matplotlib.rcParams.update({'font.size': 22}) 8 | import glob 9 | import pandas as pd 10 | from PIL import Image 11 | from tqdm import tqdm 12 | import random 13 | parser = argparse.ArgumentParser(description='read two annotations files') 14 | parser.add_argument('--vis', action='store_true') 15 | parser.add_argument('--aff_wild2_pkl', type=str, default = '/media/Samsung/Aff-wild2-Challenge/annotations/annotations.pkl') 16 | parser.add_argument('--DISFA_pkl', type=str, default = '/media/Samsung/DISFA/annotations.pkl') 17 | parser.add_argument('--save_path', type=str, default='/media/Samsung/Aff-wild2-Challenge/exps/create_new_training_set_AU/create_annotation_file/mixed_AU_annotations.pkl') 18 | args = parser.parse_args() 19 | AU_list = ['AU1','AU2','AU4','AU6','AU12','AU15','AU20','AU25'] 20 | def read_data(data_dict): 21 | paths = [] 22 | labels = [] 23 | for video in data_dict.keys(): 24 | df = data_dict[video] 25 | labels.append(df[AU_list].values.astype(np.float32)) 26 | paths.append(df['path'].values) 27 | paths = np.concatenate(paths, axis=0) 28 | labels = np.concatenate(labels, axis=0) 29 | return {'label': labels, 'path': paths} 30 | def read_aff_wild2(): 31 | total_data = pickle.load(open(args.aff_wild2_pkl, 'rb')) 32 | # training set 33 | data = total_data['AU_Set']['Training_Set'] 34 | data = read_data(data) 35 | # validation set 36 | val_data = total_data['AU_Set']['Validation_Set'] 37 | val_data = read_data(val_data) 38 | return data, val_data 39 | def merge_two_datasets(): 40 | data_aff_wild2, data_aff_wild2_val = read_aff_wild2() 41 | data_DISFA = pickle.load(open(args.DISFA_pkl, 'rb')) 42 | data_DISFA = {**data_DISFA['Training_Set'], **data_DISFA['Validation_Set']} 43 | paths = [] 44 | labels = [] 45 | for video in data_DISFA.keys(): 46 | data = data_DISFA[video] 47 | labels.append(data['label']) 48 | paths.append(data['path']) 49 | paths = np.concatenate(paths, axis=0) 50 | labels = np.concatenate(labels, axis=0) 51 | data_DISFA = {'label': labels, 'path': paths} 52 | data_merged = {'label': np.concatenate((data_aff_wild2['label'], data_DISFA['label']), axis=0), 53 | 'path': list(data_aff_wild2['path']) + list(data_DISFA['path'])} 54 | if args.vis: 55 | plot_n_samples_each_cate(data_aff_wild2['label']) 56 | plot_n_samples_each_cate(data_DISFA['label']) 57 | plot_n_labels_each_instance(data_aff_wild2['label']) 58 | plot_n_labels_each_instance(data_DISFA['label']) 59 | return {'Training_Set': data_merged, 'Validation_Set': data_aff_wild2_val} 60 | def autolabel(rects, ax, bar_label): 61 | for idx,rect in enumerate(rects): 62 | height = rect.get_height() 63 | ax.text(rect.get_x() + rect.get_width()/2., 1.05*height, 64 | bar_label[idx], 65 | ha='center', va='bottom', rotation=0) 66 | def plot_n_samples_each_cate(labels): 67 | fig, ax = plt.subplots() 68 | pos_samples = np.sum(labels, axis=0) 69 | bar_plot = plt.bar(np.arange(len(AU_list)), pos_samples) 70 | autolabel(bar_plot, ax, [str(x) for x in pos_samples]) 71 | plt.xticks(np.arange(len(AU_list)), AU_list) 72 | plt.ylabel("Number of Samples") 73 | plt.show() 74 | def plot_n_labels_each_instance(labels): 75 | fig, ax = plt.subplots() 76 | pos_samples = np.sum(labels, axis=1) 77 | unique_nums = np.unique(pos_samples) 78 | count_nums = [sum(pos_samples==i)for i in unique_nums] 79 | bar_plot = plt.bar(np.arange(len(unique_nums)), count_nums) 80 | autolabel(bar_plot, ax, [str(x) for x in count_nums]) 81 | plt.xticks(np.arange(len(unique_nums)), unique_nums) 82 | plt.ylabel("Number of Samples") 83 | plt.xlabel("Number of Labels") 84 | plt.show() 85 | def IRLbl(labels): 86 | # imbalance ratio per label 87 | # Args: 88 | # labels is a 2d numpy array, each row is one instance, each column is one class; the array contains (0, 1) only 89 | N, C = labels.shape 90 | pos_nums_per_label = np.sum(labels, axis=0) 91 | max_pos_nums = np.max(pos_nums_per_label) 92 | return max_pos_nums/pos_nums_per_label 93 | 94 | def MeanIR(labels): 95 | IRLbl_VALUE = IRLbl(labels) 96 | return np.mean(IRLbl_VALUE) 97 | def ML_ROS(all_labels, Preset_MeanIR_value=None, sample_size =32): 98 | N, C = all_labels.shape 99 | MeanIR_value = MeanIR(all_labels) if Preset_MeanIR_value is None else Preset_MeanIR_value 100 | IRLbl_value = IRLbl(all_labels) 101 | indices_per_class = {} 102 | minority_classes = [] 103 | for i in range(C): 104 | ids = all_labels[:,i] == 1 105 | indices_per_class[i] = [ii for ii, x in enumerate(ids) if x ] 106 | if IRLbl_value[i] > MeanIR_value: 107 | minority_classes.append(i) 108 | new_all_labels = all_labels 109 | oversampled_ids = [] 110 | for i in minority_classes: 111 | while True: 112 | pick_id = list(np.random.choice(indices_per_class[i], sample_size)) 113 | indices_per_class[i].extend(pick_id) 114 | # recalculate the IRLbl_value 115 | new_all_labels = np.concatenate([new_all_labels, all_labels[pick_id]], axis=0) 116 | oversampled_ids.extend(pick_id) 117 | if IRLbl(new_all_labels)[i] <= MeanIR_value : 118 | break 119 | print("oversample length:{}".format(len(oversampled_ids)), end='\r') 120 | 121 | oversampled_ids = np.array(oversampled_ids) 122 | return new_all_labels 123 | 124 | if __name__== '__main__': 125 | #data_file = read_all_image() 126 | data_file = merge_two_datasets() 127 | if args.vis: 128 | plot_n_samples_each_cate(data_file['Training_Set']['label']) 129 | plot_n_labels_each_instance(data_file['Training_Set']['label']) 130 | # perform resample and visualize 131 | resampled_labels = ML_ROS(data_file['Training_Set']['label'], Preset_MeanIR_value=2.0) 132 | plot_n_samples_each_cate(resampled_labels) 133 | pickle.dump(data_file, open(args.save_path, 'wb')) 134 | 135 | -------------------------------------------------------------------------------- /eval_val_set.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | from sklearn.metrics import f1_score 4 | import numpy as np 5 | from tqdm import tqdm 6 | import glob 7 | # this script is used to evalute the pretrained models on the Aff-wild2 validation set 8 | AU_list = ['AU1','AU2','AU4','AU6','AU12','AU15','AU20','AU25'] 9 | all_crop_aligned = '/media/Samsung/Aff-wild2-Challenge/cropped_aligned' # containing subdirectories with video name. Each subdirectory contains a sequence of crop-aligned faces 10 | annotation_file = '/media/Samsung/Aff-wild2-Challenge/annotations/annotations.pkl' # annotation file created from create_annotation_file/Aff-wild2/create_train_val_annotation_file.py 11 | 12 | input_model_dir = 'Multitask-CNN' 13 | data = pickle.load(open(annotation_file, 'rb')) 14 | save_val_results = 'save_val' 15 | """ 16 | Evalution Metrics: F1 score, accuracy and CCC 17 | """ 18 | def averaged_f1_score(input, target): 19 | N, label_size = input.shape 20 | f1s = [] 21 | for i in range(label_size): 22 | f1 = f1_score(input[:, i], target[:, i]) 23 | f1s.append(f1) 24 | return np.mean(f1s), f1s 25 | def accuracy(input, target): 26 | assert len(input.shape) == 1 27 | return sum(input==target)/input.shape[0] 28 | def averaged_accuracy(x, y): 29 | assert len(x.shape) == 2 30 | N, C =x.shape 31 | accs = [] 32 | for i in range(C): 33 | acc = accuracy(x[:, i], y[:, i]) 34 | accs.append(acc) 35 | return np.mean(accs), accs 36 | def CCC_score(x, y): 37 | vx = x - np.mean(x) 38 | vy = y - np.mean(y) 39 | rho = np.sum(vx * vy) / (np.sqrt(np.sum(vx**2)) * np.sqrt(np.sum(vy**2))) 40 | x_m = np.mean(x) 41 | y_m = np.mean(y) 42 | x_s = np.std(x) 43 | y_s = np.std(y) 44 | ccc = 2*rho*x_s*y_s/(x_s**2 + y_s**2 + (x_m - y_m)**2) 45 | return ccc 46 | def VA_metric(x, y): 47 | items = [CCC_score(x[:,0], y[:,0]), CCC_score(x[:,1], y[:,1])] 48 | return items, sum(items) 49 | def EXPR_metric(x, y): 50 | if not len(x.shape) == 1: 51 | if x.shape[1] == 1: 52 | x = x.reshape(-1) 53 | else: 54 | x = np.argmax(x, axis=-1) 55 | if not len(y.shape) == 1: 56 | if y.shape[1] == 1: 57 | y = y.reshape(-1) 58 | else: 59 | y = np.argmax(y, axis=-1) 60 | f1 = f1_score(x, y, average= 'macro') 61 | acc = accuracy(x, y) 62 | return [f1, acc], 0.67*f1 + 0.33*acc 63 | def AU_metric(x, y): 64 | f1_av,_ = averaged_f1_score(x, y) 65 | x = x.reshape(-1) 66 | y = y.reshape(-1) 67 | acc_av = accuracy(x, y) 68 | return [f1_av, acc_av], 0.5*f1_av + 0.5*acc_av 69 | def read_AU(txt_file): 70 | with open(txt_file, 'r') as f: 71 | lines = f.readlines() 72 | lines = lines[1:] # skip first line 73 | lines = [x.strip() for x in lines] 74 | lines = [x.split(',') for x in lines] 75 | lines = [[float(y) for y in x ] for x in lines] 76 | return np.array(lines) 77 | def read_Expr(txt_file): 78 | with open(txt_file, 'r') as f: 79 | lines = f.readlines() 80 | lines = lines[1:] # skip first line 81 | lines = [x.strip() for x in lines] 82 | lines = [int(x) for x in lines] 83 | return np.array(lines) 84 | def read_VA(txt_file): 85 | with open(txt_file, 'r') as f: 86 | lines = f.readlines() 87 | lines = lines[1:] # skip first line 88 | lines = [x.strip() for x in lines] 89 | lines = [x.split(',') for x in lines] 90 | lines = [[float(y) for y in x ] for x in lines] 91 | return np.array(lines) 92 | def predict_on_val_set(data, original_task): 93 | assert original_task in ['AU', 'VA', 'EXPR'] 94 | data = data["{}_Set".format(original_task)]['Validation_Set'] 95 | for video in data.keys(): 96 | image_dir = os.path.join(all_crop_aligned, video) 97 | save_dir = os.path.join(save_val_results+"_{}_Set".format(original_task), video) 98 | if not os.path.exists(save_dir): 99 | os.makedirs(save_dir) 100 | order = 'python run_pretrained_model.py --image_dir {} --model_type CNN '.format(image_dir) +\ 101 | '--batch_size 12 --eval_with_teacher --eval_with_students --save_dir {} '.format(save_dir) + \ 102 | '--workers 8 --ensemble' 103 | os.system(order) 104 | def evaluate_on_val_set(data, task): 105 | assert task in ['AU', 'VA', 'EXPR'] 106 | data = data["{}_Set".format(task)]['Validation_Set'] 107 | labels = [] 108 | preds = {} 109 | read_functions = {'AU': read_AU, 'VA':read_VA, 'EXPR': read_Expr} 110 | eval_functions = {'AU': AU_metric, 'VA':VA_metric, 'EXPR': EXPR_metric} 111 | for video in tqdm(data.keys()): 112 | prediction_dir = os.path.join(save_val_results+"_{}_Set".format(task), video) 113 | model_list = ['teacher'] 114 | model_list += ['student_{}'.format(i) for i in range(5)] 115 | model_list += ['merged'] 116 | try: 117 | label = data[video]['label'] 118 | except: 119 | try: 120 | label = data[video][AU_list].values 121 | except: 122 | label = data[video][['valence', 'arousal']].values 123 | labels.append(label) 124 | for model_name in model_list: 125 | txt_file = os.path.join(prediction_dir, model_name, '{}.txt'.format(task)) 126 | assert os.path.exists(txt_file) 127 | pred = read_functions[task](txt_file) 128 | if pred.shape[0] != label.shape[0]: 129 | assert len(pred) > len(label) 130 | # this is because the 'run_pretrained_model.py' will predict every frame in the directory. 131 | # however, some frame has face, but does not have ground truth label (EXPR) 132 | label_frames_ids = data[video]['frames_ids'].values 133 | image_dir = os.path.join(all_crop_aligned, video) 134 | frames = sorted(glob.glob(os.path.join(image_dir, '*.jpg'))) # it depends on your extensions 135 | assert len(frames) == len(pred) 136 | frame_ids = [int(os.path.basename(x).split(".")[0]) for x in frames] 137 | mask = np.array([id-1 in label_frames_ids for id in frame_ids]) # the frame format is "00001.jpg" 138 | pred = pred[mask] 139 | if model_name not in preds.keys(): 140 | preds[model_name] = [] 141 | preds[model_name].append(pred) 142 | labels = np.concatenate(labels, axis=0) 143 | preds = dict([(key, np.concatenate(preds[key], axis=0)) for key in preds]) 144 | for model_name in model_list: 145 | res = eval_functions[task](preds[model_name], labels) 146 | print("Model {} performance on {}: {} ({}, {})".format(model_name, task, res[1], res[0][0], res[0][1])) 147 | 148 | if __name__=='__main__': 149 | print("For AU, the performance format is: Result (F1 score, Accuracy)") 150 | print("For EXPR, the performance format is: Result (F1 score, Accuracy)") 151 | print("For VA, the performance format is: Result (Valence CCC, Arousal CCC)") 152 | 153 | predict_on_val_set(data, 'AU') 154 | evaluate_on_val_set(data, 'AU') 155 | predict_on_val_set(data, 'VA') 156 | evaluate_on_val_set(data, 'VA') 157 | predict_on_val_set(data, 'EXPR') 158 | evaluate_on_val_set(data, 'EXPR') 159 | 160 | 161 | 162 | -------------------------------------------------------------------------------- /Multitask-CNN/options/base_options.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import torch 4 | 5 | class BaseOptions(): 6 | def __init__(self): 7 | self._parser = argparse.ArgumentParser() 8 | self._initialized = False 9 | 10 | def initialize(self): 11 | self._parser.add_argument('--load_epoch', type=int, default=-1, help='which epoch to load? set to -1 to use latest cached model') 12 | self._parser.add_argument('--temperature', type=float, default=1.5, help='temperature in distillation loss') 13 | self._parser.add_argument('--AU_label_size', type=int, default = 8, help='# of AUs') 14 | self._parser.add_argument('--EXPR_label_size', type=int, default = 7, help='# of EXpressions') 15 | self._parser.add_argument('--VA_label_size', type=int, default = 2, help='# of VA ') 16 | self._parser.add_argument('--digitize_num', type=int, default= 20, choices = [1, 20], help='1 means no digitization,\ 17 | 20 means to digitize continuous label to 20 one hot vector ') 18 | self._parser.add_argument('--AU_criterion', type=str, default = 'BCE', choices = ['FocalLoss', 'BCE']) 19 | self._parser.add_argument('--EXPR_criterion', type=str, default = 'CE', choices = ['FocalLoss', 'CE']) 20 | self._parser.add_argument('--VA_criterion', type=str, default = 'CCC_CE', choices = ['CCC', 'CCC_CE', 'CCC_FocalLoss']) 21 | self._parser.add_argument('--lambda_AU', type=float, default= 8., help='weight for AU.') 22 | self._parser.add_argument('--lambda_EXPR', type=float, default= 1., help='weight for EXPR.') 23 | self._parser.add_argument('--lambda_V', type=float, default= 1., help='weight for valence.') 24 | self._parser.add_argument('--lambda_A', type=float, default= 1., help='weight for arousal.') 25 | self._parser.add_argument('--lambda_ccc', type=float, default= 1., help='weight for ccc loss in (CE + lambda_ccc*ccc).') 26 | self._parser.add_argument('--lambda_teacher', type=float, default = 0.4, help='weight for distillation loss when the ground truth exists (between 0 to 1)') 27 | 28 | self._parser.add_argument('--force_balance', action='store_true', help='force data balanced for training set') 29 | self._parser.add_argument('--dataset_names', type=str, default = ['Mixed_EXPR','Mixed_AU','Mixed_VA'],nargs="+") 30 | self._parser.add_argument('--tasks', type=str, default = ['EXPR','AU','VA'],nargs="+") 31 | # 'dataset_names' need to be in the same order as the 'tasks' 32 | self._parser.add_argument('--hidden_size', type=int, default = 128, help='the embedding size of each output head' ) 33 | self._parser.add_argument('--batch_size', type=int, default= 20, help='input batch size per task') 34 | self._parser.add_argument('--image_size', type=int, default= 224, help='input image size') # reducing iamge size is acceptable 35 | self._parser.add_argument('--gpu_ids', type=str, default='0', help='gpu ids: e.g. 0 0,1,2, 0,2. use -1 for CPU') 36 | self._parser.add_argument('--name', type=str, default='experiment_1', help='name of the experiment. It decides where to store samples and models') 37 | self._parser.add_argument('--n_threads_train', default=8, type=int, help='# threads for loading data') 38 | self._parser.add_argument('--n_threads_test', default=8, type=int, help='# threads for loading data') 39 | self._parser.add_argument('--checkpoints_dir', type=str, default='./checkpoints', help='models are saved here') 40 | self._parser.add_argument('--loggings_dir', type=str, default='./loggings', help='loggings are saved here') 41 | self._parser.add_argument('--model_name', type=str, default='resnet50', help='the name of model') 42 | self._parser.add_argument('--pretrained_dataset', type=str, default='ferplus', 43 | choices = ['ferplus', 'sfew','imagenet'], 44 | help="the pretrained_dataset of the face feature extractor, choices:['ferplus', 'sfew','imagenet']") 45 | self._parser.add_argument('--pretrained_teacher_model', type=str, default='') 46 | 47 | self._initialized = True 48 | 49 | def parse(self): 50 | if not self._initialized: 51 | self.initialize() 52 | self._opt = self._parser.parse_args() 53 | 54 | # set is train or test 55 | self._opt.is_train = self.is_train 56 | 57 | # set and check load_epoch 58 | self._set_and_check_load_epoch() 59 | 60 | # get and set gpus 61 | self._get_set_gpus() 62 | 63 | args = vars(self._opt) 64 | 65 | # print in terminal args 66 | self._print(args) 67 | 68 | # save args to file 69 | self._save(args) 70 | 71 | return self._opt 72 | 73 | def _set_and_check_load_epoch(self): 74 | models_dir = os.path.join(self._opt.checkpoints_dir, self._opt.name) 75 | if os.path.exists(models_dir): 76 | if self._opt.load_epoch == -1: 77 | load_epoch = 0 78 | if self.is_train: 79 | for file in os.listdir(models_dir): 80 | if file.startswith("net_epoch_"): 81 | load_epoch = max(load_epoch, int(file.split('_')[2])) 82 | self._opt.load_epoch = load_epoch 83 | else: 84 | found = False 85 | for file in os.listdir(models_dir): 86 | if file.startswith("net_epoch_"): 87 | found = int(file.split('_')[2]) == self._opt.load_epoch 88 | if found: break 89 | assert found, 'Model for epoch %i not found' % self._opt.load_epoch 90 | else: 91 | assert self._opt.load_epoch < 1, 'Model for epoch %i not found' % self._opt.load_epoch 92 | self._opt.load_epoch = 0 93 | 94 | def _get_set_gpus(self): 95 | # get gpu ids 96 | str_ids = self._opt.gpu_ids.split(',') 97 | self._opt.gpu_ids = [] 98 | for str_id in str_ids: 99 | id = int(str_id) 100 | if id >= 0: 101 | self._opt.gpu_ids.append(id) 102 | 103 | # set gpu ids 104 | if len(self._opt.gpu_ids) > 0: 105 | torch.cuda.set_device(self._opt.gpu_ids[0]) 106 | 107 | def _print(self, args): 108 | print('------------ Options -------------') 109 | for k, v in sorted(args.items()): 110 | print('%s: %s' % (str(k), str(v))) 111 | print('-------------- End ----------------') 112 | 113 | def _save(self, args): 114 | expr_dir = os.path.join(self._opt.checkpoints_dir, self._opt.name) 115 | print(expr_dir) 116 | if self.is_train: 117 | os.makedirs(expr_dir) 118 | else: 119 | assert os.path.exists(expr_dir) 120 | file_name = os.path.join(expr_dir, 'opt_%s.txt' % ('train' if self.is_train else 'test')) 121 | with open(file_name, 'wt') as opt_file: 122 | opt_file.write('------------ Options -------------\n') 123 | for k, v in sorted(args.items()): 124 | opt_file.write('%s: %s\n' % (str(k), str(v))) 125 | opt_file.write('-------------- End ----------------\n') 126 | -------------------------------------------------------------------------------- /create_annotation_file/Aff-wild2/create_train_val_annotation_file.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import os 3 | import numpy as np 4 | import argparse 5 | from matplotlib import pyplot as plt 6 | import matplotlib 7 | import glob 8 | import pandas as pd 9 | from tqdm import tqdm 10 | parser = argparse.ArgumentParser(description='save annotations') 11 | parser.add_argument('--vis', action = 'store_true', 12 | help='whether to visualize the distribution') 13 | parser.add_argument('--annot_dir', type=str, default = '/media/Samsung/Aff-wild2-Challenge/annotations', 14 | help='annotation dir') 15 | parser.add_argument('--data_dir', type=str, default= '/media/Samsung/Aff-wild2-Challenge/cropped_aligned') 16 | 17 | args = parser.parse_args() 18 | def read_AU(txt_file): 19 | with open(txt_file, 'r') as f: 20 | lines = f.readlines() 21 | lines = lines[1:] # skip first line 22 | lines = [x.strip() for x in lines] 23 | lines = [x.split(',') for x in lines] 24 | lines = [[float(y) for y in x ] for x in lines] 25 | return np.array(lines) 26 | def read_Expr(txt_file): 27 | with open(txt_file, 'r') as f: 28 | lines = f.readlines() 29 | lines = lines[1:] # skip first line 30 | lines = [x.strip() for x in lines] 31 | lines = [int(x) for x in lines] 32 | return np.array(lines) 33 | def read_VA(txt_file): 34 | with open(txt_file, 'r') as f: 35 | lines = f.readlines() 36 | lines = lines[1:] # skip first line 37 | lines = [x.strip() for x in lines] 38 | lines = [x.split(',') for x in lines] 39 | lines = [[float(y) for y in x ] for x in lines] 40 | return np.array(lines) 41 | def plot_pie(AU_list, pos_freq, neg_freq): 42 | ploting_labels = [x+'+ {0:.2f}'.format(y) for x, y in zip(AU_list, pos_freq)] + [x+'- {0:.2f}'.format(y) for x, y in zip(AU_list, neg_freq)] 43 | cmap = matplotlib.cm.get_cmap('coolwarm') 44 | colors = [cmap(x) for x in pos_freq] + [cmap(x) for x in neg_freq] 45 | fracs = np.ones(len(AU_list)*2) 46 | plt.pie(fracs, labels=ploting_labels, autopct=None, shadow=False, colors=colors,startangle =78.75) 47 | plt.title("AUs distribution") 48 | plt.show() 49 | 50 | def frames_to_label(label_array, frames, discard_value): 51 | assert len(label_array) >= len(frames) # some labels need to be discarded 52 | frames_ids = [int(frame.split('/')[-1].split('.')[0]) - 1 for frame in frames] # frame_id start from 0 53 | N = label_array.shape[0] 54 | label_array = label_array.reshape((N, -1)) 55 | to_drop = (label_array == discard_value).sum(-1) 56 | drop_ids = [i for i in range(len(to_drop)) if to_drop[i]] 57 | frames_ids = [i for i in frames_ids if i not in drop_ids] 58 | indexes = [True if i in frames_ids else False for i in range(len(label_array)) ] 59 | label_array = label_array[indexes] 60 | assert len(label_array) == len(frames_ids) 61 | prefix = '/'.join(frames[0].split('/')[:-1]) 62 | return_frames = [prefix+'/{0:05d}.jpg'.format(id+1) for id in frames_ids] 63 | return label_array, return_frames, frames_ids 64 | def main(): 65 | annot_dir = args.annot_dir 66 | tasks = [x for x in os.listdir(annot_dir)] 67 | data_file = {} 68 | for task in tasks: 69 | if task == 'AU_Set': 70 | AU_list = ['AU1','AU2','AU4','AU6','AU12','AU15','AU20','AU25'] 71 | data_file[task] = {} 72 | for mode in ['Training_Set', 'Validation_Set']: 73 | txt_files = glob.glob(os.path.join(annot_dir, task, mode, '*.txt')) 74 | data_file[task][mode] = {} 75 | for txt_file in tqdm(txt_files): 76 | name = os.path.basename(txt_file).split('.')[0] 77 | au_array = read_AU(txt_file) 78 | frames_paths = sorted(glob.glob(os.path.join(args.data_dir, name, '*.jpg'))) 79 | au_array, frames_paths, frames_ids = frames_to_label(au_array, frames_paths, discard_value = -1) 80 | data_dict = dict([(AU_list[i], au_array[:, i])for i in range(len(AU_list))]) 81 | data_dict.update({'path': frames_paths, 'frames_ids':frames_ids}) 82 | data_file[task][mode][name] = pd.DataFrame.from_dict(data_dict) 83 | if args.vis: 84 | total_dict = {**data_file[task]['Training_Set'], **data_file[task]['Validation_Set']} 85 | all_samples = [] 86 | for name in total_dict.keys(): 87 | arr = [] 88 | for l in AU_list: 89 | arr.append(total_dict[name][l].values) 90 | arr = np.stack(arr, axis=1) 91 | all_samples.append(arr) 92 | all_samples = np.concatenate(all_samples, axis=0) 93 | pos_freq = np.sum(all_samples, axis=0)/all_samples.shape[0] 94 | neg_freq = -np.sum(all_samples-1, axis=0)/all_samples.shape[0] 95 | plot_pie(AU_list, pos_freq, neg_freq) 96 | if task == 'EXPR_Set': 97 | Expr_list = ['Neutral','Anger','Disgust','Fear','Happiness','Sadness','Surprise'] 98 | data_file[task] = {} 99 | for mode in ['Training_Set', 'Validation_Set']: 100 | txt_files = glob.glob(os.path.join(annot_dir, task, mode, '*.txt')) 101 | data_file[task][mode] = {} 102 | for txt_file in tqdm(txt_files): 103 | name = os.path.basename(txt_file).split('.')[0] 104 | expr_array = read_Expr(txt_file) 105 | frames_paths = sorted(glob.glob(os.path.join(args.data_dir, name, '*.jpg'))) 106 | expr_array, frames_paths, frames_ids = frames_to_label(expr_array, frames_paths, discard_value = -1) 107 | data_dict = {'label':expr_array.reshape(-1), 'path':frames_paths, 'frames_ids': frames_ids} 108 | data_file[task][mode][name] = pd.DataFrame.from_dict(data_dict) 109 | if args.vis: 110 | total_dict = {**data_file[task]['Training_Set'], **data_file[task]['Validation_Set']} 111 | all_samples = np.concatenate([total_dict[x]['label'].values for x in total_dict.keys()], axis=0) 112 | histogram = np.zeros(len(Expr_list)) 113 | for i in range(len(Expr_list)): 114 | find_true = sum(all_samples==i) 115 | histogram[i] =find_true/all_samples.shape[0] 116 | plt.bar(np.arange(len(Expr_list)), histogram) 117 | plt.xticks(np.arange(len(Expr_list)), Expr_list) 118 | plt.show() 119 | if task == 'VA_Set': 120 | VA_list = ['Valence', 'Arousal'] 121 | data_file[task] = {} 122 | for mode in ['Training_Set', 'Validation_Set']: 123 | txt_files = glob.glob(os.path.join(annot_dir, task, mode, '*.txt')) 124 | data_file[task][mode] = {} 125 | for txt_file in tqdm(txt_files): 126 | name = os.path.basename(txt_file).split('.')[0] 127 | va_array = read_VA(txt_file) 128 | frames_paths = sorted(glob.glob(os.path.join(args.data_dir, name, '*.jpg'))) 129 | va_array, frames_paths, frames_ids = frames_to_label(va_array, frames_paths, discard_value = -5.) 130 | data_dict = {'valence':va_array[:, 0],'arousal': va_array[:, 1], 'path':frames_paths, 'frames_ids': frames_ids} 131 | data_file[task][mode][name] = pd.DataFrame.from_dict(data_dict) 132 | if args.vis: 133 | total_dict = {**data_file[task]['Training_Set'], **data_file[task]['Validation_Set']} 134 | all_samples = [] 135 | for name in total_dict.keys(): 136 | arr = [] 137 | for l in ['valence', 'arousal']: 138 | arr.append(total_dict[name][l].values) 139 | arr = np.stack(arr, axis=1) 140 | all_samples.append(arr) 141 | all_samples = np.concatenate(all_samples, axis=0) 142 | pos_freq = np.sum(all_samples, axis=0)/all_samples.shape[0] 143 | plt.hist2d(all_samples[:,0], all_samples[:,1], bins=(20, 20), cmap=plt.cm.jet) 144 | plt.xlabel("Valence") 145 | plt.ylabel('Arousal') 146 | plt.colorbar() 147 | plt.show() 148 | save_path = os.path.join(annot_dir, 'annotations.pkl') 149 | pickle.dump(data_file, open(save_path, 'wb')) 150 | if __name__== '__main__': 151 | main() 152 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/options/base_options.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import torch 4 | 5 | class BaseOptions(): 6 | def __init__(self): 7 | self._parser = argparse.ArgumentParser() 8 | self._initialized = False 9 | 10 | def initialize(self): 11 | self._parser.add_argument('--load_epoch', type=int, default=-1, help='which epoch to load? set to -1 to use latest cached model') 12 | self._parser.add_argument('--temperature', type=float, default=1.5, help='temperature in distillation loss') 13 | self._parser.add_argument('--AU_label_size', type=int, default = 8, help='# of AUs') 14 | self._parser.add_argument('--EXPR_label_size', type=int, default = 7, help='# of EXpressions') 15 | self._parser.add_argument('--VA_label_size', type=int, default = 2, help='# of VA ') 16 | self._parser.add_argument('--digitize_num', type=int, default= 20, choices = [1, 20], help='1 means no digitization,\ 17 | 20 means to digitize continuous label to 20 one hot vector ') 18 | self._parser.add_argument('--AU_criterion', type=str, default = 'BCE', choices = ['FocalLoss', 'BCE']) 19 | self._parser.add_argument('--EXPR_criterion', type=str, default = 'CE', choices = ['FocalLoss', 'CE']) 20 | self._parser.add_argument('--VA_criterion', type=str, default = 'CCC_CE', choices = ['CCC', 'CCC_CE', 'CCC_FocalLoss']) 21 | self._parser.add_argument('--lambda_teacher', type=float, default = 0.4, help='weight for distillation loss when the ground truth exists (between 0 to 1)') 22 | self._parser.add_argument('--lambda_AU', type=float, default= 8., help='weight for AU.') 23 | self._parser.add_argument('--lambda_EXPR', type=float, default= 1., help='weight for EXPR.') 24 | self._parser.add_argument('--lambda_V', type=float, default= 1., help='weight for valence.') 25 | self._parser.add_argument('--lambda_A', type=float, default= 1., help='weight for arousal.') 26 | self._parser.add_argument('--lambda_ccc', type=float, default= 1., help='weight for ccc loss in (CE + lambda_ccc*ccc).') 27 | #self._parser.add_argument('--force_balance', action='store_true', help='force data balanced for training set') 28 | self._parser.add_argument('--dataset_names', type=str, default = ['Mixed_EXPR','Mixed_AU','Mixed_VA'],nargs="+") 29 | self._parser.add_argument('--tasks', type=str, default = ['EXPR','AU','VA'],nargs="+") 30 | # 'dataset_names' need to be in the same order as the 'tasks' 31 | self._parser.add_argument('--seq_len', type=int, default=64, help='length of input seq ') 32 | self._parser.add_argument('--frozen', action='store_true') 33 | self._parser.add_argument('--hidden_size', type=int, default = 128, help='the embedding size of each output head' ) 34 | self._parser.add_argument('--batch_size', type=int, default= 20, help='input batch size per task') 35 | self._parser.add_argument('--image_size', type=int, default= 224, help='input image size') # reducing iamge size is acceptable 36 | self._parser.add_argument('--gpu_ids', type=str, default='0', help='gpu ids: e.g. 0 0,1,2, 0,2. use -1 for CPU') 37 | self._parser.add_argument('--name', type=str, default='experiment_1', help='name of the experiment. It decides where to store samples and models') 38 | self._parser.add_argument('--n_threads_train', default=8, type=int, help='# threads for loading data') 39 | self._parser.add_argument('--n_threads_test', default=8, type=int, help='# threads for loading data') 40 | self._parser.add_argument('--checkpoints_dir', type=str, default='./checkpoints', help='models are saved here') 41 | self._parser.add_argument('--loggings_dir', type=str, default='./loggings', help='loggings are saved here') 42 | self._parser.add_argument('--model_name', type=str, default='resnet50', help='the name of model') 43 | self._parser.add_argument('--pretrained_dataset', type=str, default='ferplus', 44 | choices = ['ferplus', 'sfew','imagenet'], 45 | help="the pretrained_dataset of the face feature extractor, choices:['ferplus', 'sfew','imagenet']") 46 | 47 | self._parser.add_argument('--pretrained_resnet50_model', type=str, default = '', help='pretrained model') 48 | self._parser.add_argument('--pretrained_teacher_model', type=str, default='') 49 | self._initialized = True 50 | 51 | def parse(self): 52 | if not self._initialized: 53 | self.initialize() 54 | self._opt = self._parser.parse_args() 55 | 56 | # set is train or test 57 | self._opt.is_train = self.is_train 58 | 59 | # set and check load_epoch 60 | self._set_and_check_load_epoch() 61 | 62 | # get and set gpus 63 | self._get_set_gpus() 64 | 65 | args = vars(self._opt) 66 | 67 | # print in terminal args 68 | self._print(args) 69 | 70 | # save args to file 71 | self._save(args) 72 | 73 | return self._opt 74 | 75 | def _set_and_check_load_epoch(self): 76 | models_dir = os.path.join(self._opt.checkpoints_dir, self._opt.name) 77 | if os.path.exists(models_dir): 78 | if self._opt.load_epoch == -1: 79 | load_epoch = 0 80 | if self.is_train: 81 | for file in os.listdir(models_dir): 82 | if file.startswith("net_epoch_"): 83 | load_epoch = max(load_epoch, int(file.split('_')[2])) 84 | self._opt.load_epoch = load_epoch 85 | else: 86 | found = False 87 | for file in os.listdir(models_dir): 88 | if file.startswith("net_epoch_"): 89 | found = int(file.split('_')[2]) == self._opt.load_epoch 90 | if found: break 91 | assert found, 'Model for epoch %i not found' % self._opt.load_epoch 92 | else: 93 | assert self._opt.load_epoch < 1, 'Model for epoch %i not found' % self._opt.load_epoch 94 | self._opt.load_epoch = 0 95 | 96 | def _get_set_gpus(self): 97 | # get gpu ids 98 | str_ids = self._opt.gpu_ids.split(',') 99 | self._opt.gpu_ids = [] 100 | for str_id in str_ids: 101 | id = int(str_id) 102 | if id >= 0: 103 | self._opt.gpu_ids.append(id) 104 | 105 | # set gpu ids 106 | if len(self._opt.gpu_ids) > 0: 107 | torch.cuda.set_device(self._opt.gpu_ids[0]) 108 | 109 | def _print(self, args): 110 | print('------------ Options -------------') 111 | for k, v in sorted(args.items()): 112 | print('%s: %s' % (str(k), str(v))) 113 | print('-------------- End ----------------') 114 | 115 | def _save(self, args): 116 | expr_dir = os.path.join(self._opt.checkpoints_dir, self._opt.name) 117 | print(expr_dir) 118 | if self.is_train: 119 | os.makedirs(expr_dir) 120 | else: 121 | assert os.path.exists(expr_dir) 122 | file_name = os.path.join(expr_dir, 'opt_%s.txt' % ('train' if self.is_train else 'test')) 123 | with open(file_name, 'wt') as opt_file: 124 | opt_file.write('------------ Options -------------\n') 125 | for k, v in sorted(args.items()): 126 | opt_file.write('%s: %s\n' % (str(k), str(v))) 127 | opt_file.write('-------------- End ----------------\n') 128 | -------------------------------------------------------------------------------- /Multitask-CNN/test.py: -------------------------------------------------------------------------------- 1 | import time 2 | from options.test_options import TestOptions 3 | from data.test_video_dataset import Test_dataset 4 | from models.models import ModelsFactory 5 | from collections import OrderedDict 6 | import os 7 | import numpy as np 8 | import torch 9 | from sklearn.metrics import f1_score 10 | from PATH import PATH 11 | import pandas as pd 12 | from tqdm import tqdm 13 | from copy import deepcopy 14 | from scipy.stats import mode 15 | from scipy.special import softmax 16 | import pickle 17 | def sigmoid(x): 18 | return 1/(1+np.exp(-x)) 19 | PRESET_VARS = PATH() 20 | #################RuntimeError: received 0 items of ancdata ########################### 21 | torch.multiprocessing.set_sharing_strategy("file_system") 22 | ######################################################################### 23 | class Tester: 24 | def __init__(self): 25 | self._opt = TestOptions().parse() 26 | self._model = ModelsFactory.get_by_name(self._opt.model_name, self._opt) 27 | test_data_file = PRESET_VARS.Aff_wild2.test_data_file 28 | self.test_data_file = pickle.load(open(test_data_file, 'rb')) 29 | self.save_dir = self._opt.save_dir 30 | if not os.path.exists(self.save_dir): 31 | os.makedirs(self.save_dir) 32 | if self._opt.mode == 'Test': 33 | self._test() 34 | else: 35 | raise ValueError("do not call test.py with validation mode.") 36 | def _test(self): 37 | self._model.set_eval() 38 | val_transforms = self._model.resnet50.backbone.compose_transforms 39 | if self._opt.eval_with_teacher: 40 | model_paths = [self._opt.teacher_model_path] 41 | else: 42 | model_paths = [] 43 | if self._opt.ensemble: 44 | for i in range(self._opt.n_students): 45 | path = os.path.join(self._opt.checkpoints_dir, self._opt.name, 'net_epoch_student_{}_id_resnet50.pth'.format(i)) 46 | assert os.path.exists(path) 47 | model_paths.append(path) 48 | outputs_record = {} 49 | estimates_record = {} 50 | frames_ids_record = {} 51 | for i, path in enumerate(model_paths): 52 | self._model.resnet50.load_state_dict(torch.load(path)) 53 | outputs_record[i] = {} 54 | estimates_record[i] = {} 55 | frames_ids_record[i] = {} 56 | for task in self._opt.tasks: 57 | task = task+"_Set" 58 | task_data_file = self.test_data_file[task]['Test_Set'] 59 | outputs_record[i][task] = {} 60 | estimates_record[i][task] = {} 61 | frames_ids_record[i][task] = {} 62 | for i_video, video in enumerate(task_data_file.keys()): 63 | video_data = task_data_file[video] 64 | test_dataset = Test_dataset(self._opt, video_data, transform=val_transforms) 65 | test_dataloader = torch.utils.data.DataLoader( 66 | test_dataset, 67 | batch_size=self._opt.batch_size, 68 | shuffle= False, 69 | num_workers=int(self._opt.n_threads_test), 70 | drop_last=False) 71 | track = self.test_one_video(test_dataloader, task = task[:-4]) 72 | torch.cuda.empty_cache() 73 | outputs_record[i][task][video] = track['outputs'] 74 | estimates_record[i][task][video] = track['estimates'] 75 | frames_ids_record[i][task][video] = track['frames_ids'] 76 | print("Model ID {} Task {} Current {}/{}".format(i, task[:-4], i_video, len(task_data_file.keys()))) 77 | save_path = '{}/{}/{}.txt'.format(i, task, video) 78 | self.save_to_file(track['frames_ids'], track['estimates'], save_path, task=task[:-4]) 79 | # if i_video>=1: 80 | # break 81 | #merge the raw outputs 82 | for task in self._opt.tasks: 83 | task = task+"_Set" 84 | for video in outputs_record[0][task].keys(): 85 | preds = [] 86 | for i in range(len(outputs_record.keys())): 87 | preds.append(outputs_record[i][task][video]) 88 | preds = np.array(preds) 89 | #assert frames_ids_record[0][task][video] == frames_ids_record[1][task][video] 90 | video_frames_ids = frames_ids_record[0][task][video] 91 | if task == 'AU_Set': 92 | merged_preds = sigmoid(preds) 93 | best_thresholds_over_models = [] # fill in using the results obtained from val.py 94 | print("The best AU thresholds over models: {}".format(best_thresholds_over_models)) 95 | merged_preds = np.mean(merged_preds, axis=0) 96 | merged_preds = merged_preds > (np.ones_like(merged_preds)*best_thresholds_over_models) 97 | merged_preds = merged_preds.astype(np.int64) 98 | save_path = '{}/{}/{}.txt'.format('merged', task, video) 99 | self.save_to_file(video_frames_ids, merged_preds, save_path, task='AU') 100 | elif task == 'EXPR_Set': 101 | merged_preds = softmax(preds, axis=-1).mean(0).argmax(-1).astype(np.int).squeeze() 102 | save_path = '{}/{}/{}.txt'.format('merged',task, video) 103 | self.save_to_file(video_frames_ids, merged_preds, save_path, task='EXPR') 104 | else: 105 | N = self._opt.digitize_num 106 | v = softmax(preds[:, :, :N], axis=-1) 107 | a = softmax(preds[:, :, N:], axis=-1) 108 | bins = np.linspace(-1, 1, num=self._opt.digitize_num) 109 | v = (bins * v).sum(-1) 110 | a = (bins * a).sum(-1) 111 | merged_preds = np.stack([v.mean(0), a.mean(0)], axis = 1).squeeze() 112 | save_path = '{}/{}/{}.txt'.format( 'merged',task, video) 113 | self.save_to_file(video_frames_ids, merged_preds, save_path, task='VA') 114 | 115 | def save_to_file(self, frames_ids, predictions, save_path, task= 'AU'): 116 | save_path =os.path.join(self.save_dir, save_path) 117 | save_dir = os.path.dirname(os.path.abspath(save_path)) 118 | if not os.path.exists(save_dir): 119 | os.makedirs(save_dir) 120 | categories = PRESET_VARS.Aff_wild2.categories[task] 121 | assert len(frames_ids) == len(predictions) 122 | assert frames_ids[-1] == len(frames_ids) - 1 123 | with open(save_path, 'w') as f: 124 | f.write(",".join(categories)+"\n") 125 | for i, line in enumerate(predictions): 126 | if isinstance(line, np.ndarray): 127 | digits = [] 128 | for x in line: 129 | if isinstance(x, float): 130 | digits.append("{:.4f}".format(x)) 131 | elif isinstance(x, np.int64): 132 | digits.append(str(x)) 133 | line = ','.join(digits)+'\n' 134 | elif isinstance(line, np.int64): 135 | line = str(line)+'\n' 136 | if i == len(predictions)-1: 137 | line = line[:-1] 138 | f.write(line) 139 | 140 | def test_one_video(self, data_loader, task = 'AU'): 141 | track_val = {'outputs':[], 'estimates':[], 'frames_ids':[]} 142 | for i_val_batch, val_batch in tqdm(enumerate(data_loader), total = len(data_loader)): 143 | # evaluate model 144 | wrapped_v_batch = {task: val_batch} 145 | self._model.set_input(wrapped_v_batch, input_tasks = [task]) 146 | outputs, _ = self._model.forward(return_estimates=False, input_tasks = [task]) 147 | estimates, _ = self._model.forward(return_estimates=True, input_tasks = [task]) 148 | #store the predictions and labels 149 | track_val['outputs'].append(outputs[task][task]) 150 | track_val['frames_ids'].append(np.array(val_batch['frames_ids'])) 151 | track_val['estimates'].append(estimates[task][task]) 152 | 153 | for key in track_val.keys(): 154 | track_val[key] = np.concatenate(track_val[key], axis=0) 155 | assert len(track_val['frames_ids']) -1 == track_val['frames_ids'][-1] 156 | return track_val 157 | if __name__ == "__main__": 158 | Tester() 159 | 160 | 161 | -------------------------------------------------------------------------------- /create_annotation_file/AFEW-VA/read_annotation_and_align_faces.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import math 4 | import pandas 5 | from PIL import Image 6 | import numpy as np 7 | import glob 8 | import pickle 9 | from tqdm import tqdm 10 | from matplotlib import pyplot as plt 11 | np.random.seed(0) 12 | 13 | def crop_face(image, keypoints, rotate = True, quiet_mode=True): 14 | lex, ley = (keypoints[36] + keypoints[39])/2 15 | rex, rey = (keypoints[42] + keypoints[45])/2 16 | rmx, rmy = keypoints[54] 17 | lmx, lmy = keypoints[48] 18 | nex, ney = keypoints[33] 19 | # roation using PIL image 20 | 21 | if rotate: 22 | angle = calculate_angle(lex, ley, rex, rey) 23 | image, lex, ley, rex, rey, lmx, lmy, rmx, rmy \ 24 | = image_rote(image, angle, lex, ley, rex, rey, lmx, lmy, rmx, rmy) 25 | eye_width = rex - lex # distance between two eyes 26 | ecx, ecy = (lex + rex) / 2.0, (ley + rey) / 2.0 # the center between two eyes 27 | mouth_width = rmx - lmx 28 | mcx, mcy = (lmx + rmx) / 2.0, (lmy + rmy) / 2.0 #mouth center coordinate 29 | em_height = mcy - ecy # height between mouth center to eyes center 30 | fcx, fcy = (ecx + mcx) / 2.0, (ecy + mcy) / 2.0 # face center 31 | # face 32 | if eye_width > em_height: 33 | alpha = eye_width 34 | else: 35 | alpha = em_height 36 | g_beta = 2.0 37 | g_left = fcx - alpha / 2.0 * g_beta 38 | g_upper = fcy - alpha / 2.0 * g_beta 39 | g_right = fcx + alpha / 2.0 * g_beta 40 | g_lower = fcy + alpha / 2.0 * g_beta 41 | g_face = image.crop((g_left, g_upper, g_right, g_lower)) 42 | 43 | return g_face 44 | 45 | def image_rote(img, angle, elx, ely, erx, ery, mlx, mly, mrx, mry, expand=1): 46 | w,h= img.size 47 | img = img.rotate(angle, expand=expand) #whether to expand after rotation 48 | if expand == 0: 49 | elx, ely = pos_transform_samesize(angle, elx, ely, w, h) 50 | erx, ery = pos_transform_samesize(angle, erx, ery, w, h) 51 | mlx, mly = pos_transform_samesize(angle, mlx, mly, w, h) 52 | mrx, mry = pos_transform_samesize(angle, mrx, mry, w, h) 53 | if expand == 1: 54 | elx, ely = pos_transform_resize(angle, elx, ely, w, h) 55 | erx, ery = pos_transform_resize(angle, erx, ery, w, h) 56 | mlx, mly = pos_transform_resize(angle, mlx, mly, w, h) 57 | mrx, mry = pos_transform_resize(angle, mrx, mry, w, h) 58 | return img, elx, ely, erx, ery, mlx, mly, mrx, mry 59 | 60 | def calculate_angle(elx, ely, erx, ery): 61 | """ 62 | calculate image rotate angle 63 | :param elx: lefy eye x 64 | :param ely: left eye y 65 | :param erx: right eye x 66 | :param ery: right eye y 67 | :return: rotate angle 68 | """ 69 | dx = erx - elx 70 | dy = ery - ely 71 | angle = math.atan(dy / dx) * 180 / math.pi 72 | return angle 73 | 74 | def pos_transform_resize(angle, x, y, w, h): 75 | """ 76 | after rotation, new coordinate with expansion 77 | :param angle: 78 | :param x: 79 | :param y: 80 | :param w: 81 | :param h: 82 | :return: 83 | """ 84 | angle = angle * math.pi / 180 85 | matrix = [ math.cos(angle), math.sin(angle), 0.0, -math.sin(angle), math.cos(angle), 0.0 ] 86 | def transform(x, y, matrix=matrix): 87 | (a, b, c, d, e, f) = matrix 88 | return a * x + b * y + c, d * x + e * y + f # calculate output size 89 | xx = [] 90 | yy = [] 91 | for x_, y_ in ((0, 0), (w, 0), (w, h), (0, h)): 92 | x_, y_ = transform(x_, y_) 93 | xx.append(x_) 94 | yy.append(y_) 95 | ww = int(math.ceil(max(xx)) - math.floor(min(xx))) 96 | hh = int(math.ceil(max(yy)) - math.floor(min(yy))) 97 | # adjust center 98 | cx, cy = transform(w / 2.0, h / 2.0) 99 | matrix[2] = ww / 2.0 - cx 100 | matrix[5] = hh / 2.0 - cy 101 | tx, ty = transform(x, y) 102 | return tx, ty 103 | 104 | def pos_transform_samesize(angle, x, y, w, h): 105 | """ 106 | after rotation, new coordinate without expansion 107 | :param angle: 108 | :param x: 109 | :param y: 110 | :param w: 111 | :param h: 112 | :return: 113 | """ 114 | angle = angle * math.pi / 180 115 | matrix = [ math.cos(angle), math.sin(angle), 0.0, -math.sin(angle), math.cos(angle), 0.0 ] 116 | def transform(x, y, matrix=matrix): 117 | (a, b, c, d, e, f) = matrix 118 | return a * x + b * y + c, d * x + e * y + f 119 | cx, cy = transform(w / 2.0, h / 2.0) 120 | matrix[2] = w / 2.0 - cx 121 | matrix[5] = h / 2.0 - cy 122 | x, y = transform(x, y) 123 | return x, y 124 | def PIL_image_convert(cv2_im): 125 | cv2_im = cv2.cvtColor(cv2_im,cv2.COLOR_BGR2RGB) 126 | pil_im = Image.fromarray(cv2_im) 127 | return pil_im 128 | def create_annotation_file(): 129 | root_dir= '.' 130 | output_dir = 'cropped_aligned_faces' 131 | data_file_path = 'annotations.pkl' 132 | if not os.path.exists(output_dir): 133 | os.makedirs(output_dir) 134 | subfolders = ['{0:02d}'.format(i) for i in range(1, 13)] 135 | subfolders = [x for x in subfolders if os.path.isdir(os.path.join(root_dir, x))] 136 | all_videos_dirs = [] 137 | for subfolder in subfolders: 138 | subsubfolders = os.listdir(os.path.join(root_dir, subfolder)) 139 | all_videos_dirs.extend([os.path.join(root_dir, subfolder, x) for x in subsubfolders if os.path.isdir(os.path.join(root_dir, subfolder, x))]) 140 | all_videos_names = ['_'.join(x.split('/')[-2:]) for x in all_videos_dirs] 141 | data_file = {} 142 | ids = np.random.permutation(len(all_videos_dirs)) 143 | all_videos_names = [all_videos_names[i] for i in ids] 144 | all_videos_dirs = [all_videos_dirs[i] for i in ids] 145 | N = int(len(all_videos_dirs)*0.8) 146 | data_file['Training_Set'] = {} 147 | data_file['Validation_Set'] = {} 148 | for ii in range(2): 149 | if ii==0: 150 | video_dirs = all_videos_dirs[:N] 151 | video_names = all_videos_names[:N] 152 | mode = 'Training_Set' 153 | else: 154 | video_dirs = all_videos_dirs[N:] 155 | video_names = all_videos_names[N:] 156 | mode = 'Validation_Set' 157 | print("Extract :{}".format(mode)) 158 | for video_dir, video_name in tqdm(zip(video_dirs, video_names), total = len(video_dirs)): 159 | assert video_name == '_'.join(video_dir.split('/')[-2:]) 160 | data_file[mode][video_name] = {} 161 | frames = sorted(glob.glob(os.path.join(video_dir, '*.png'))) 162 | json_file = glob.glob(os.path.join(video_dir, '*.json'))[0] 163 | with open(json_file, 'r') as f: 164 | json_dict = json.load(f) 165 | frames_dict = json_dict['frames'] 166 | len_frames = len(list(frames_dict.keys())) 167 | assert len_frames == len(frames), "Detected frames length is different from the labeled frames length" 168 | valences, arousals = [], [] 169 | paths = [] 170 | for id, frame_path in zip(sorted(frames_dict.keys()), frames): 171 | assert id==os.path.basename(frame_path).split('.')[0] 172 | img = Image.open(frame_path).convert("RGB") 173 | ldm = np.array(frames_dict[id]['landmarks']) 174 | arousal = frames_dict[id]['arousal'] 175 | valence = frames_dict[id]['valence'] 176 | arousals.append(arousal) 177 | valences.append(valence) 178 | save_path = os.path.join(output_dir, video_name, os.path.basename(frame_path)) 179 | paths.append(os.path.abspath(save_path)) 180 | if not os.path.exists(save_path): 181 | crop_aligned_face = crop_face(img, ldm) 182 | if not os.path.exists(os.path.join(output_dir, video_name)): 183 | os.makedirs(os.path.join(output_dir, video_name)) 184 | crop_aligned_face.save(save_path) 185 | valences, arousals = np.array(valences), np.array(arousals) 186 | data_file[mode][video_name]['valence'] = valences/10. 187 | data_file[mode][video_name]['arousal'] = arousals/10. # rescale to -1.0, 1.0 188 | data_file[mode][video_name]['path'] = paths 189 | pickle.dump(data_file, open(data_file_path, 'wb')) 190 | if __name__=='__main__': 191 | data_file_path = 'annotations.pkl' 192 | if not os.path.exists(data_file_path): 193 | create_annotation_file() 194 | data_file = pickle.load(open(data_file_path, 'rb')) 195 | all_samples_arousal = np.concatenate([data_file['Training_Set'][x]['arousal'] for x in data_file['Training_Set'].keys()] + 196 | [data_file['Validation_Set'][x]['arousal'] for x in data_file['Validation_Set'].keys()], axis=0) 197 | 198 | all_samples_valence = np.concatenate([data_file['Training_Set'][x]['valence'] for x in data_file['Training_Set'].keys()] + 199 | [data_file['Validation_Set'][x]['valence'] for x in data_file['Validation_Set'].keys()], axis=0) 200 | plt.hist2d(all_samples_valence , all_samples_arousal, bins=(20, 20), cmap=plt.cm.jet) 201 | plt.xlabel("Valence") 202 | plt.ylabel('Arousal') 203 | plt.colorbar() 204 | plt.show() 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/test.py: -------------------------------------------------------------------------------- 1 | import time 2 | from options.test_options import TestOptions 3 | from data.test_video_dataset import Test_dataset 4 | from models.models import ModelsFactory 5 | from collections import OrderedDict 6 | import os 7 | import numpy as np 8 | from sklearn.metrics import f1_score 9 | from PATH import PATH 10 | import pandas as pd 11 | from tqdm import tqdm 12 | from copy import deepcopy 13 | from scipy.stats import mode 14 | from scipy.special import softmax 15 | import pickle 16 | from sklearn.metrics import precision_recall_curve 17 | def sigmoid(x): 18 | return 1/(1+np.exp(-x)) 19 | PRESET_VARS = PATH() 20 | #################RuntimeError: received 0 items of ancdata ########################### 21 | import torch 22 | torch.multiprocessing.set_sharing_strategy("file_system") 23 | ######################################################################### 24 | class Tester: 25 | def __init__(self): 26 | self._opt = TestOptions().parse() 27 | self._model = ModelsFactory.get_by_name(self._opt.model_name, self._opt) 28 | test_data_file = PRESET_VARS.Aff_wild2.test_data_file 29 | self.test_data_file = pickle.load(open(test_data_file, 'rb')) 30 | self.save_dir = self._opt.save_dir 31 | if not os.path.exists(self.save_dir): 32 | os.makedirs(self.save_dir) 33 | if self._opt.mode == 'Test': 34 | self._test() 35 | else: 36 | raise ValueError("do not call test.py with validation mode.") 37 | def _test(self): 38 | self._model.set_eval() 39 | val_transforms = self._model.resnet50_GRU.backbone.backbone.compose_transforms 40 | model_paths = [self._opt.teacher_model_path] 41 | if self._opt.ensemble: 42 | for i in range(self._opt.n_students): 43 | path = os.path.join(self._opt.checkpoints_dir, self._opt.name, 'net_epoch_student_{}_id_resnet50_GRU.pth'.format(i)) 44 | assert os.path.exists(path) 45 | model_paths.append(path) 46 | outputs_record = {} 47 | estimates_record = {} 48 | frames_ids_record = {} 49 | for i, path in enumerate(model_paths): 50 | self._model.resnet50_GRU.load_state_dict(torch.load(path)) 51 | outputs_record[i] = {} 52 | estimates_record[i] = {} 53 | frames_ids_record[i] = {} 54 | for task in self._opt.tasks: 55 | task = task+"_Set" 56 | task_data_file = self.test_data_file[task]['Test_Set'] 57 | outputs_record[i][task] = {} 58 | estimates_record[i][task] = {} 59 | frames_ids_record[i][task] = {} 60 | for i_video, video in enumerate(task_data_file.keys()): 61 | video_data = task_data_file[video] 62 | test_dataset = Test_dataset(self._opt, video_data, transform=val_transforms) 63 | test_dataloader = torch.utils.data.DataLoader( 64 | test_dataset, 65 | batch_size=self._opt.batch_size, 66 | shuffle= False, 67 | num_workers=int(self._opt.n_threads_test), 68 | drop_last=False) 69 | track = self.test_one_video(test_dataloader, task = task[:-4]) 70 | torch.cuda.empty_cache() 71 | outputs_record[i][task][video] = track['outputs'] 72 | estimates_record[i][task][video] = track['estimates'] 73 | frames_ids_record[i][task][video] = track['frames_ids'] 74 | print("Model ID {} Task {} Current {}/{}".format(i, task[:-4], i_video, len(task_data_file.keys()))) 75 | save_path = '{}/{}/{}.txt'.format(i, task, video) 76 | self.save_to_file(track['frames_ids'], track['estimates'], save_path, task=task[:-4]) 77 | # if i_video>=1: 78 | # break 79 | #merge the raw outputs 80 | for task in self._opt.tasks: 81 | task = task+"_Set" 82 | for video in outputs_record[0][task].keys(): 83 | preds = [] 84 | for i in range(len(outputs_record.keys())): 85 | preds.append(outputs_record[i][task][video]) 86 | preds = np.array(preds) 87 | #assert frames_ids_record[0][task][video] == frames_ids_record[1][task][video] 88 | video_frames_ids = frames_ids_record[0][task][video] 89 | if task == 'AU_Set': 90 | merged_preds = sigmoid(preds) 91 | best_thresholds_over_models = [] # fill in using the results obtained by val.py 92 | print("The best AU thresholds over models: {}".format(best_thresholds_over_models)) 93 | merged_preds = np.mean(merged_preds, axis=0) 94 | merged_preds = merged_preds > (np.ones_like(merged_preds)*best_thresholds_over_models) 95 | merged_preds = merged_preds.astype(np.int64) 96 | save_path = '{}/{}/{}.txt'.format('merged', task, video) 97 | self.save_to_file(video_frames_ids, merged_preds, save_path, task='AU') 98 | elif task == 'EXPR_Set': 99 | merged_preds = softmax(preds, axis=-1).mean(0).argmax(-1).astype(np.int).squeeze() 100 | save_path = '{}/{}/{}.txt'.format('merged',task, video) 101 | self.save_to_file(video_frames_ids, merged_preds, save_path, task='EXPR') 102 | else: 103 | N = self._opt.digitize_num 104 | v = softmax(preds[:, :, :N], axis=-1) 105 | a = softmax(preds[:, :, N:], axis=-1) 106 | bins = np.linspace(-1, 1, num=self._opt.digitize_num) 107 | v = (bins * v).sum(-1) 108 | a = (bins * a).sum(-1) 109 | merged_preds = np.stack([v.mean(0), a.mean(0)], axis = 1).squeeze() 110 | save_path = '{}/{}/{}.txt'.format( 'merged',task, video) 111 | self.save_to_file(video_frames_ids, merged_preds, save_path, task='VA') 112 | 113 | def save_to_file(self, frames_ids, predictions, save_path, task= 'AU'): 114 | save_path =os.path.join(self.save_dir, save_path) 115 | save_dir = os.path.dirname(os.path.abspath(save_path)) 116 | if not os.path.exists(save_dir): 117 | os.makedirs(save_dir) 118 | categories = PRESET_VARS.Aff_wild2.categories[task] 119 | #filtered out repeated frames 120 | mask = np.zeros_like(frames_ids, dtype=bool) 121 | mask[np.unique(frames_ids, return_index=True)[1]] = True 122 | frames_ids = frames_ids[mask] 123 | predictions = predictions[mask] 124 | assert len(frames_ids) == len(predictions) 125 | with open(save_path, 'w') as f: 126 | f.write(",".join(categories)+"\n") 127 | for i, line in enumerate(predictions): 128 | if isinstance(line, np.ndarray): 129 | digits = [] 130 | for x in line: 131 | if isinstance(x, float): 132 | digits.append("{:.4f}".format(x)) 133 | elif isinstance(x, np.int64): 134 | digits.append(str(x)) 135 | line = ','.join(digits)+'\n' 136 | elif isinstance(line, np.int64): 137 | line = str(line)+'\n' 138 | if i == len(predictions)-1: 139 | line = line[:-1] 140 | f.write(line) 141 | 142 | def test_one_video(self, data_loader, task = 'AU'): 143 | track_val = {'outputs':[], 'estimates':[], 'frames_ids':[]} 144 | for i_val_batch, val_batch in tqdm(enumerate(data_loader), total = len(data_loader)): 145 | # evaluate model 146 | wrapped_v_batch = {task: val_batch} 147 | self._model.set_input(wrapped_v_batch, input_tasks = [task]) 148 | outputs, _ = self._model.forward(return_estimates=False, input_tasks = [task]) 149 | estimates, _ = self._model.forward(return_estimates=True, input_tasks = [task]) 150 | #store the predictions and labels 151 | B, N, C = outputs[task][task].shape 152 | track_val['outputs'].append(outputs[task][task].reshape(B*N, C)) 153 | track_val['frames_ids'].append(np.array([np.array(x) for x in val_batch['frames_ids']]).reshape(B*N, -1).squeeze()) 154 | track_val['estimates'].append(estimates[task][task].reshape(B*N, -1).squeeze()) 155 | 156 | for key in track_val.keys(): 157 | track_val[key] = np.concatenate(track_val[key], axis=0) 158 | #assert len(track_val['frames_ids']) -1 == track_val['frames_ids'][-1] 159 | return track_val 160 | if __name__ == "__main__": 161 | Tester() 162 | 163 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Multitask-Emotion-Recognition-with-Incomplete-Labels 2 | This is the repository containing the solution for FG-2020 ABAW Competition 3 | 4 | Pretrained models can be downloaded through this [link](https://hkustconnect-my.sharepoint.com/:f:/g/personal/ddeng_connect_ust_hk/EnX91m9VSHlFobaIag82W_8B3YRkir97H1QmiUlkZu1zAw?e=LGgDNE) under the `Multitask-CNN` and `Multitask-CNN-RNN` folders. 5 | 6 | [Paper](https://www.computer.org/csdl/pds/api/csdl/proceedings/download-article/1kecJ0EYZxK/pdf), [Presentation](https://hkustconnect-my.sharepoint.com/:v:/g/personal/ddeng_connect_ust_hk/EZc6HS2fLe5MjzJM4WpzJQsBITfaK2jtGYvYSsSxJsCIbQ?e=UyXiaK) 7 | 8 | We aim for a unifed model to solve three tasks: Facial Action Units (FAU) prediction, Facial Expression (7 basic emotions) prediction, Valence and Arousal prediction. For abbreviation, we refer to them as FAU, EXPR and VA. 9 | 10 | *UPDATES*: The challenge [leaderboard](https://ibug.doc.ic.ac.uk/resources/fg-2020-competition-affective-behavior-analysis/) has been released. Our solution won two challenege tracks (FAU and VA) among six teams! 11 | 12 | --- 13 | ## DEMO: We made our latest demo available! 14 | [![Watch the video](https://github.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels/blob/master/imgs/thumnail.jpg)](https://youtu.be/0-dnW0Rb5_U) 15 | 16 | To run such a demo, make sure you have installed the requirements listed in [Requirements](#requirements) in addition to [MTCNN](https://github.com/ipazc/mtcnn) and downloaded all pretrained weights. Afterwards, you can modify the `video_file` in `emotion_demo.py` to the video you want to process, and then run `python emotion_demo.py`. The output video will be saved under the `save_dir`. 17 | 18 | --- 19 | ## Data Balancing 20 | 21 | Before training, we change the data distribution of experiment datasets by (1) importing external datasets, such as the [DISFA](http://mohammadmahoor.com/disfa/) dataset for FAU, the [ExpW](http://mmlab.ie.cuhk.edu.hk/projects/socialrelation/index.html) dataset for EXPR, and the [AFEW-VA](https://ibug.doc.ic.ac.uk/resources/afew-va-database/) dataset for VA; (2) resampling the minority class and the majority class. Our purpose is to create a more balanced data distribution for each individual class. 22 | 23 | This the data disribution of the Aff-wild2 dataset, the DISFA dataset and the merged dataset. We resampled the merged dataset using ML-ROS, which is short for [Multilabel Randomly Oversampling](https://www.sciencedirect.com/science/article/pii/S0925231215004269) 24 | 25 | 26 | 27 | This the data distribution of the Aff-wild2 dataset, the ExpW dataset and the merged dataset. We resample the merged dataset to ensure the instances of each class have the same probability of appearing in one epoch. 28 | 29 | 30 | 31 | This the data distribution of the Aff-wild2 dataset, the AFEW-VA dataset and the merged dataset. We discretize the continuous valence/arousal scores in [-1, 1] into 20 bins of the same width. We treat each bin as a category, and apply the oversampling/undersampling strategy. 32 | 33 | 34 | 35 | ## Learning With Partial Labels 36 | 37 | For the current datasets, each dataset only contain one type of labels (FAU, EXPR or VA). Therefore we propose an algorithm for a deep neural network to learn multitask from partial labels. 38 | The algorithm has two steps: firstly, we train a teacher model to perform all three tasks, where each instance is trained by the ground truth label of its corresponding task. Secondly, we refer to the outputs of the teacher model as the soft labels. We use the soft labels and the ground truths to train the student model. 39 | 40 | This is the diagram for our proposed algorithm. Given the input images of three tasks and the ground truths of three tasks , we first train the teacher model using the teacher loss between the teacher outputs and the ground truth . Secondly, we train the student model using the student loss which consists of two parts: one is calcaluted from the teacher outputs and the student outputs , another is calculated from the ground truth and the student outputs . 41 | 42 | 43 | 44 | --- 45 | ## Requirements 46 | * Pytorch 1.3.1 or higher version 47 | * Numpy 48 | * [pytorch benchmark](https://github.com/albanie/pytorch-benchmarks) 49 | * pandas, pickle, matplotlib 50 | 51 | ## Pretrained Weights 52 | 53 | Before training our models on the Aff-wild2 dataset, we loaded the pretrained weights on other emotion datasets, such as FER2013 dataset. This is part of [pytorch benchmark](https://github.com/albanie/pytorch-benchmarks) repository but you need to download them manually. To download these pretrained weights, we provide this [link](https://hkustconnect-my.sharepoint.com/:f:/g/personal/ddeng_connect_ust_hk/ElE4ifQSxLBCgtfQLvgE5Z8B8wGo-SpoUJc-So_3FruMcg?e=LL3u6V) where you can download the three folders, `fer+`, `fer`, and `sfew`. Please save the three folders under `pytorch_benchmarks/models`. 54 | 55 | You can download weights of the multitask CNN and multitask CNN-RNN pretrained on the Aff-wild2 by running: 56 | 57 | ``` 58 | bash download_pretrained_weights_aff_wild2.sh 59 | ``` 60 | 61 | 62 | ## How to replicate our results 63 | 1. Download all required datasets, crop and align face images; 64 | 65 | 2. Create the annotation files for each dataset, using the script in `create_annotation_file` directory; 66 | 67 | 3. Change the annotation file paths in the `Multitask-CNN(Multitask-CNN-RNN)/PATH/__init__.py`; 68 | 69 | 4. Training: For Multitask-CNN, run `python train.py --force_balance --name image_size_112_n_students_5 --image_size 112 --pretrained_teacher_model path-to-teacher-model-if-exists`, the argument `name` is experiment name (save path), the `--force_balance` will make the sampled dataset more balanced. 70 | For Multitask-CNN-RNN, run `python train.py --name image_size_112_n_students_5_seq_len=32 --image_size 112 --seq_len 32 --frozen --pretrained_resnet50_model path-to-the-pretrained-Multitask-CNN-model --pretrained_teacher_model path-to-teacher-model-if-exists ` 71 | 72 | 5. Validation: Run the `python val.py --name image_size_112_n_students_5 --image_size 112 --teacher_model_path path-to-teacher-model --mode Validation --ensemble ` for Multitask-CNN, and run `python val.py --name image_size_112_n_students_5_seq_len=32 --image_size 112 --teacher_model_path path-to-teacher-model --pretrained_resnet50_model path-to-the-pretrained-Multitask-CNN-model --mode Validation --ensemble --seq_len 32` for Multitask-CNN-RNN. 73 | 74 | 6. From the results on the validation set, we obtain the best AU thresholds on the validation set. 75 | Modify this line `best_thresholds_over_models = [] ` in the `test.py` to the best thresholds on the validation set. 76 | 77 | 7. Testing: run `python test.py --name image_size_112_n_students_5 --image_size 112 --teacher_model_path path-to-teacher-model --mode Test --save_dir Predictions --ensemble` for Multitask-CNN, and run `python test.py --name image_size_112_n_students_5_seq_len=32 --image_size 112 --teacher_model_path path-to-teacher-model --pretrained_resnet50_model path-to-the-pretrained-Multitask-CNN-model --mode Test --ensemble --seq_len 32` for Multitask-CNN-RNN. 78 | 79 | --- 80 | ## How to implement our model on your data 81 | 1. Download the pretrained CNNs and unzip them. 82 | 83 | 2. Crop and align face images, save them to a directory. 84 | 85 | 3. For CNN model: `python run_pretrained_model.py --image_dir directory-containing-sequence-of-face-images --model_type CNN --batch_size 12 --eval_with_teacher --eval_with_students --save_dir save-directory --workers 8 --ensemble`. For CNN-RNN model: `python run_pretrained_model.py --image_dir directory-containing-sequence-of-face-images --model_type CNN-RNN --seq_len 32 --batch_size 6 --eval_with_teacher --eval_with_students --save_dir save-directory --workers 8 --ensemble` 86 | 87 | --- 88 | ## If you are interested in our work, please cite 89 | ``` 90 | @inproceedings{deng2020multitask, 91 | title={Multitask Emotion Recognition with Incomplete Labels}, 92 | author={Deng, Didan and Chen, Zhaokang and Shi, Bertram E}, 93 | booktitle={2020 15th IEEE International Conference on Automatic Face and Gesture Recognition (FG 2020)(FG)}, 94 | pages={828--835}, 95 | organization={IEEE Computer Society} 96 | } 97 | ``` 98 | -------------------------------------------------------------------------------- /Multitask-CNN-RNN/val.py: -------------------------------------------------------------------------------- 1 | import time 2 | from options.test_options import TestOptions 3 | from data.custom_dataset_data_loader import Multitask_DatasetDataLoader 4 | from models.models import ModelsFactory 5 | from collections import OrderedDict 6 | import os 7 | import numpy as np 8 | from sklearn.metrics import f1_score 9 | from PATH import PATH 10 | import pandas as pd 11 | from tqdm import tqdm 12 | from copy import deepcopy 13 | from scipy.stats import mode 14 | from scipy.special import softmax 15 | import pickle 16 | from sklearn.metrics import precision_recall_curve 17 | def sigmoid(x): 18 | return 1/(1+np.exp(-x)) 19 | #################RuntimeError: received 0 items of ancdata ########################### 20 | import torch 21 | torch.multiprocessing.set_sharing_strategy("file_system") 22 | ######################################################################### 23 | class Tester: 24 | def __init__(self): 25 | self._opt = TestOptions().parse() 26 | PRESET_VARS = PATH() 27 | self._model = ModelsFactory.get_by_name(self._opt.model_name, self._opt) 28 | val_transforms = self._model.resnet50_GRU.backbone.backbone.compose_transforms 29 | self.validation_dataloaders = Multitask_DatasetDataLoader(self._opt, train_mode = self._opt.mode, transform = val_transforms) 30 | self.validation_dataloaders = self.validation_dataloaders.load_multitask_val_test_data() 31 | print("{} sets".format(self._opt.mode)) 32 | for task in self._opt.tasks: 33 | data_loader = self.validation_dataloaders[task] 34 | print("{}: {} images".format(task, len(data_loader)*self._opt.batch_size * len(self._opt.tasks) * self._opt.seq_len)) 35 | if self._opt.mode == 'Validation': 36 | self._validate() 37 | else: 38 | raise ValueError("do not call val.py with test mode.") 39 | 40 | def _validate(self): 41 | # set model to eval 42 | self._model.set_eval() 43 | if self._opt.eval_with_teacher: 44 | model_paths = [self._opt.teacher_model_path] 45 | else: 46 | model_paths = [] 47 | if self._opt.ensemble: 48 | for i in range(self._opt.n_students): 49 | path = os.path.join(self._opt.checkpoints_dir, self._opt.name, 'net_epoch_student_{}_id_resnet50_GRU.pth'.format(i)) 50 | assert os.path.exists(path) 51 | model_paths.append(path) 52 | print("Evaluation: {} models".format(len(model_paths))) 53 | outputs_record = {} 54 | estimates_record = {} 55 | metrics_record = {} 56 | labels_record = {} 57 | for i, path in enumerate(model_paths): 58 | self._model.resnet50_GRU.load_state_dict(torch.load(path)) 59 | outputs_record[i] = {} 60 | estimates_record[i] = {} 61 | metrics_record[i] = {} 62 | labels_record[i] = {} 63 | for task in self._opt.tasks: 64 | track_val = {'outputs':[],'labels':[], 'estimates':[]} 65 | data_loader = self.validation_dataloaders[task] 66 | for i_val_batch, val_batch in tqdm(enumerate(data_loader), total = len(data_loader)): 67 | # evaluate model 68 | wrapped_v_batch = {task: val_batch} 69 | self._model.set_input(wrapped_v_batch, input_tasks = [task]) 70 | torch.cuda.empty_cache() 71 | outputs, _ = self._model.forward(return_estimates=False, input_tasks = [task]) 72 | estimates, _ = self._model.forward(return_estimates=True, input_tasks = [task]) 73 | 74 | #store the predictions and labels 75 | B, N, C = outputs[task][task].shape 76 | track_val['outputs'].append(outputs[task][task].reshape(B*N, C)) 77 | track_val['labels'].append(wrapped_v_batch[task]['label'].reshape(B*N, -1).squeeze()) 78 | track_val['estimates'].append(estimates[task][task].reshape(B*N, -1).squeeze()) 79 | # if i_val_batch> 3: 80 | # break 81 | # calculate metric 82 | for key in track_val.keys(): 83 | track_val[key] = np.concatenate(track_val[key], axis=0) 84 | preds = track_val['estimates'] 85 | labels = track_val['labels'] 86 | metric_func = self._model.get_metrics_per_task()[task] 87 | eval_items, eval_res = metric_func(preds, labels) 88 | now_time = time.strftime("%H:%M", time.localtime(time.time())) 89 | output = "Model id {} {} Validation {}: Eval_0 {:.4f} Eval_1 {:.4f} eval_res {:.4f}".format(i, task, 90 | now_time, eval_items[0], eval_items[1], eval_res) 91 | print(output) 92 | outputs_record[i][task] = track_val['outputs'] 93 | estimates_record[i][task] = track_val['estimates'] 94 | labels_record[i][task] = track_val['labels'] 95 | metrics_record[i][task] = [eval_items, eval_res] 96 | # one choice, merge the estimates 97 | for task in self._opt.tasks: 98 | preds = [] 99 | labels = [] 100 | for i in range(len(estimates_record.keys())): 101 | preds.append(estimates_record[i][task]) 102 | labels.append(labels_record[i][task]) 103 | preds = np.array(preds) 104 | labels = np.array(labels) 105 | #assert labels[0] == labels[1] 106 | if task == 'AU' or task == 'EXPR': 107 | merged_preds = mode(preds, axis=0)[0] 108 | elif task == 'VA': 109 | merged_preds = np.mean(preds, axis=0) 110 | labels = np.mean(labels,axis=0) 111 | #assert labels.shape[0] == merged_preds.shape[0] 112 | metric_func = self._model.get_metrics_per_task()[task] 113 | eval_items, eval_res = metric_func(merged_preds.squeeze(), labels.squeeze()) 114 | now_time = time.strftime("%H:%M", time.localtime(time.time())) 115 | output = "Merged First method {} Validation {}: Eval_0 {:.4f} Eval_1 {:.4f} eval_res {:.4f}".format( task, 116 | now_time, eval_items[0], eval_items[1], eval_res) 117 | print(output) 118 | # one choice, average the raw outputs 119 | for task in self._opt.tasks: 120 | preds = [] 121 | labels = [] 122 | for i in range(len(estimates_record.keys())): 123 | preds.append(outputs_record[i][task]) 124 | labels.append(labels_record[i][task]) 125 | preds = np.array(preds) 126 | labels = np.array(labels) 127 | #assert labels[0] == labels[1] 128 | if task == 'AU': 129 | merged_preds = sigmoid(preds) 130 | best_thresholds_over_models = [] 131 | for i in range(len(merged_preds)): 132 | f1_optimal_thresholds = [] 133 | merged_preds_per_model = merged_preds[i] 134 | for j in range(merged_preds_per_model.shape[1]): 135 | precision, recall, thresholds = precision_recall_curve(labels[i][:, j].astype(np.int),merged_preds[i][:, j]) 136 | f1_optimal_thresholds.append(thresholds[np.abs(precision-recall).argmin(0)]) 137 | f1_optimal_thresholds = np.array(f1_optimal_thresholds) 138 | best_thresholds_over_models.append(f1_optimal_thresholds) 139 | best_thresholds_over_models = np.array(best_thresholds_over_models).mean(0) 140 | merged_preds = np.mean(merged_preds, axis=0) 141 | merged_preds = merged_preds > (np.ones_like(merged_preds)*best_thresholds_over_models) 142 | merged_preds = merged_preds.astype(np.int64) 143 | print("The best AU thresholds over models: {}".format(best_thresholds_over_models)) 144 | elif task=='EXPR': 145 | merged_preds = softmax(preds, axis=-1).mean(0).argmax(-1).astype(np.int) 146 | else: 147 | N = self._opt.digitize_num 148 | v = softmax(preds[:, :, :N], axis=-1).argmax(-1) 149 | a = softmax(preds[:, :, N:], axis=-1).argmax(-1) 150 | v = mode(v, axis=0)[0] 151 | a = mode(a, axis=0)[0] 152 | v = np.eye(N)[v] 153 | a = np.eye(N)[a] 154 | bins = np.linspace(-1, 1, num=self._opt.digitize_num) 155 | v = (bins * v).sum(-1) 156 | a = (bins * a).sum(-1) 157 | merged_preds = np.stack([v.squeeze(), a.squeeze()], axis = 1) 158 | labels = np.mean(labels, axis=0) 159 | metric_func = self._model.get_metrics_per_task()[task] 160 | eval_items, eval_res = metric_func(merged_preds.squeeze(), labels.squeeze()) 161 | now_time = time.strftime("%H:%M", time.localtime(time.time())) 162 | output = "Merged Second method {} Validation {}: Eval_0 {:.4f} Eval_1 {:.4f} eval_res {:.4f}".format(task, 163 | now_time, eval_items[0], eval_items[1], eval_res) 164 | print(output) 165 | save_path = 'evaluate_val_set.pkl' 166 | data = {'outputs':outputs_record, 'estimates':estimates_record, 'labels':labels_record, 'metrics':metrics_record} 167 | pickle.dump(data, open(save_path, 'wb')) 168 | 169 | if __name__ == "__main__": 170 | Tester() 171 | --------------------------------------------------------------------------------