├── src ├── .DS_Store ├── read_mat.py ├── process_Ekman_theta.py ├── shuffle_data.py ├── tf_utils.py ├── transform.py ├── train_Emotion6.py └── random_emotion6.m ├── evaluation ├── .DS_Store ├── evaluate_Emotion6.py └── evaluate_Ekman6.py └── README.md /src/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kittenish/Frame-Transformer-Network/HEAD/src/.DS_Store -------------------------------------------------------------------------------- /evaluation/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kittenish/Frame-Transformer-Network/HEAD/evaluation/.DS_Store -------------------------------------------------------------------------------- /src/read_mat.py: -------------------------------------------------------------------------------- 1 | import scipy.io as sio 2 | import h5py 3 | 4 | def read_mat(url): 5 | data = sio.loadmat(url) 6 | return data 7 | 8 | def read_mat_v(url,name): 9 | with h5py.File(url, 'r') as f: 10 | data = f[name][()] 11 | return data 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Frame-Transformer-Network 2 | This repository is the released code and data for our ICMR 2017 [paper][1] "Frame-Transformer Emotion Classification Network". 3 | 4 | Contact: jrgao14@fudan.edu.cn 5 | 6 | ## Requirements 7 | - Tensorflow 8 | - Python Numpy 9 | 10 | ## Dataset 11 | Our Emotion6 video dataset and attribution labels for Ekman6 dataset are released [here][2]. 12 | 13 | ## Citation 14 | 15 | Please cite our paper in your publication if it helps your research: 16 | 17 | @inproceedings{gao2017frame, 18 | title={Frame-Transformer Emotion Classification Network}, 19 | author={Gao, Jiarui and Fu, Yanwei and Jiang, Yu-Gang and Xue, Xiangyang}, 20 | booktitle={Proceedings of the 2017 ACM on International Conference on Multimedia Retrieval}, 21 | pages={78--83}, 22 | year={2017}, 23 | organization={ACM} 24 | } 25 | 26 | [1]: http://www.yugangjiang.info/publication/icmr17-emotion.pdf 27 | [2]: https://drive.google.com/open?id=0B-iork9xj4brQmlYYjlsUUtVVGM -------------------------------------------------------------------------------- /src/process_Ekman_theta.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | #f = open('/Users/mac/Desktop/train_theta_result_541.txt','r') 5 | def cal_position(filepath,newpath): 6 | f = open(filepath,'r') 7 | theta_1 = 0.0 8 | theta_2 = 0.0 9 | position = np.zeros([461, 2]) 10 | #position = np.zeros([100, 2]) 11 | i = 0 12 | 13 | for i in range(461): 14 | #for i in range(100): 15 | line = f.readline() 16 | 17 | if line: 18 | p=line.rfind(' ') 19 | theta_0 = float(line[0:p]) 20 | theta_1 = float(line[p:]) 21 | # position[i][1] = 15 * (theta_0 + theta_1 + 1) 22 | # position[i][0] = 15 * (theta_1 + 1 - theta_0) 23 | position[i][1] = 100 * (theta_0 + theta_1 + 1) 24 | position[i][0] = 100 * (theta_1 + 1 - theta_0) 25 | temp = 0 26 | if position[i][0] > position[i][1]: 27 | temp = position[i][1] 28 | position[i][1] = position[i][0] 29 | position[i][0] = temp 30 | if position[i][0] < 0: 31 | position[i][0] = 0 32 | if position[i][1] > 200: 33 | position[i][1] = 200 34 | i = i + 1 35 | 36 | np.savetxt(newpath,position,fmt="%d") 37 | f.close() 38 | 39 | def dirlist(path): 40 | 41 | filelist = os.listdir(path) 42 | #gt = '/Users/mac/Desktop/Ekman/5*1_initial=0.2/train/train_position.txt' 43 | 44 | for filename in filelist: 45 | filepath = os.path.join(path, filename) 46 | if filename == 'train': 47 | pass 48 | elif os.path.isdir(filepath): 49 | dirlist(filepath) 50 | else: 51 | #allfile.append(filepath) 52 | if filename[-4:len(filename)] == '.txt' and filename != 'accuracy.txt' : 53 | 54 | newpath = '/'.join(filepath.split('/')[0:6]) + '/position/' + filename 55 | cal_position(filepath,newpath) 56 | 57 | print newpath 58 | 59 | dirlist('/Users/mac/Desktop/Ekman/') 60 | 61 | #cal_position('/Users/mac/Desktop/video_spacial/syhthetic/color/result/4096*3_initial=0.5/train/train_theta.txt','/Users/mac/Desktop/video_spacial/syhthetic/color/result/4096*3_initial=0.5/train/train_position.txt') 62 | -------------------------------------------------------------------------------- /src/shuffle_data.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import random 3 | import json 4 | from read_mat import read_mat 5 | 6 | def shuffle(x_train, y_train, num): 7 | 8 | x = np.linspace(0, num - 1, num) 9 | random.shuffle(x) 10 | 11 | X_train = np.zeros([num, 60, 4096]) 12 | Y_train = np.zeros((num,), dtype=np.int) 13 | 14 | for i in range(num): 15 | X_train[i,:,:] = x_train[int(x[i]),:,:] 16 | Y_train[i] = y_train[int(x[i])] 17 | 18 | print 'finish train data shuffle' 19 | return X_train, Y_train 20 | 21 | def shuffle_100(x_train, y_train, num): 22 | 23 | x = np.linspace(0, num - 1, num) 24 | random.shuffle(x) 25 | 26 | X_train = np.zeros([num, 100, 4096]) 27 | Y_train = np.zeros((num,), dtype=np.int) 28 | 29 | for i in range(num): 30 | X_train[i,:,:] = x_train[int(x[i]),:,:] 31 | Y_train[i] = y_train[int(x[i])] 32 | 33 | print 'finish train data shuffle' 34 | return X_train, Y_train 35 | 36 | 37 | def shuffle_emotion6(x_train, y_train, position, num): 38 | 39 | print x_train.shape 40 | print y_train.shape 41 | print position.shape 42 | x = np.linspace(0, num - 1, num) 43 | random.shuffle(x) 44 | 45 | X_train = np.zeros([num, 30, 4096]) 46 | Y_train = np.zeros((num,), dtype=np.int) 47 | Position = np.zeros([num, 2], dtype=np.int) 48 | 49 | for i in range(num): 50 | X_train[i,:,:] = x_train[int(x[i]),:,:] 51 | Y_train[i] = y_train[0,int(x[i])] 52 | Position[i,:] = position[int(x[i]),:] 53 | 54 | theta = np.zeros([num, 2]) 55 | for i in range(num): 56 | theta[i,0] = (Position[i,1] - Position[i,0]) / 30.0 57 | theta[i,1] = ((Position[i,1] + Position[i,0]) / 15.0 - 2.0) / 2.0 58 | 59 | np.savetxt("/home/g_jiarui/video_spacial/result/1/train/train_position.txt",Position,fmt="%d") 60 | np.savetxt("/home/g_jiarui/video_spacial/result/1/train/train_theta.txt",theta,fmt="%f") 61 | 62 | print 'finish train data shuffle' 63 | return X_train, Y_train, Position 64 | 65 | def shuffle_conv5(x_train, y_train, num): 66 | 67 | x = np.linspace(0, num - 1, num) 68 | random.shuffle(x) 69 | 70 | X_train = x_train.copy() 71 | Y_train = np.zeros((num,), dtype=np.int) 72 | 73 | for i in range(num): 74 | X_train[i,:,:,:,:] = x_train[int(x[i]),:,:,:,:] 75 | Y_train[i] = y_train[int(x[i])] 76 | 77 | print 'finish train data shuffle' 78 | return X_train, Y_train 79 | 80 | -------------------------------------------------------------------------------- /evaluation/evaluate_Emotion6.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import os 4 | import json 5 | 6 | def interpolated_prec_rec(prec, rec): 7 | """Interpolated AP - VOCdevkit from VOC 2011. 8 | """ 9 | mprec = np.hstack([[0], prec, [0]]) 10 | mrec = np.hstack([[0], rec, [1]]) 11 | for i in range(len(mprec) - 1)[::-1]: 12 | mprec[i] = max(mprec[i], mprec[i + 1]) 13 | idx = np.where(mrec[1::] != mrec[0:-1])[0] + 1 14 | ap = np.sum((mrec[idx] - mrec[idx - 1]) * mprec[idx]) 15 | return ap 16 | 17 | def cal_mAP(url1, url2, alpha): 18 | f = open(url1, 'r') 19 | data = f.read() 20 | 21 | g = open(url2, 'r') 22 | gt = g.read() 23 | 24 | data = data.split('\n') 25 | data = data[0:len(data) - 1] 26 | pre = np.zeros((len(data), 2)) 27 | 28 | num = 0 29 | for i in data: 30 | temp = i.split(' ') 31 | pre[num,0] = int(temp[0]) 32 | pre[num,1] = int(temp[1]) 33 | num = num + 1 34 | 35 | gt = gt.split('\n') 36 | gt = gt[0:len(gt) - 1] 37 | 38 | ground = np.zeros((len(gt), 2)) 39 | 40 | num = 0 41 | for i in gt: 42 | temp = i.split(' ') 43 | ground[num,0] = int(temp[0]) 44 | ground[num,1] = int(temp[1]) 45 | num = num + 1 46 | 47 | tiou = np.zeros(len(data)) 48 | tp = np.zeros(len(data)) 49 | fp = np.zeros(len(data)) 50 | 51 | b_i = np.maximum(pre[:,0], ground[:,0]) 52 | e_i = np.minimum(pre[:,1], ground[:,1]) 53 | 54 | b_u = np.minimum(pre[:,0], ground[:,0]) 55 | e_u = np.maximum(pre[:,1], ground[:,1]) 56 | 57 | tiou = (e_i - b_i) / (e_u - b_u) 58 | 59 | for i in range(len(tiou)): 60 | if tiou[i] < alpha: 61 | fp[i] = 1 62 | else: 63 | tp[i] = 1 64 | tp = np.cumsum(tp).astype(np.float) 65 | fp = np.cumsum(fp).astype(np.float) 66 | rec = tp / len(data) 67 | prec = tp / (tp + fp) 68 | return interpolated_prec_rec(prec, rec) 69 | 70 | def cal_filter_mAP(url, alpha): 71 | ground_url = url + '/train/train_position.txt' 72 | filelist = os.listdir(url + '/position/') 73 | filelist = sorted(filelist) 74 | mAP = np.zeros(100) 75 | num = 0 76 | for filename in filelist : 77 | if filename[-4:len(filename)] == '.txt' and filename[0] == 't': 78 | print url + '/position/' + filename 79 | mAP[num] = cal_mAP(url + '/position/' + filename, ground_url, alpha) 80 | num = num + 1 81 | return mAP 82 | 83 | def rename(): 84 | path = '/Users/mac/Desktop/ff/' 85 | filelist = os.listdir(path) 86 | for i in filelist: 87 | if i != '.DS_Store': 88 | url = path + i + '/position/' 89 | filelist_n = os.listdir(url) 90 | for j in filelist_n: 91 | if j != '.DS_Store': 92 | name = j.split('_') 93 | if len(name[3]) != '7': 94 | if len(name[3]) == 6: 95 | name[3] = '0' + name[3] 96 | name = '_'.join(name) 97 | os.rename(url + j, url + name) 98 | if len(name[3]) == 5: 99 | name[3] = '00' + name[3] 100 | name = '_'.join(name) 101 | os.rename(url + j, url + name) 102 | 103 | 104 | def cal_all_mAP(alpha): 105 | path = '/Users/mac/Desktop/ff/' 106 | filelist = os.listdir(path) 107 | all_mAP = {} 108 | for i in filelist: 109 | if i != '.DS_Store' and i[0] != 'z': 110 | url = path + i + '/position/' 111 | mAP = cal_filter_mAP(path + i, alpha) 112 | all_mAP[i] = list(mAP) 113 | 114 | ff = open('mAP.txt', 'w') 115 | ff.write(json.dumps(all_mAP)) 116 | ff.close() 117 | 118 | def draw_mAP(): 119 | ff = open('/Users/mac/Desktop/mAP.txt', 'r') 120 | all_mAP = ff.read() 121 | all_mAP = json.loads(all_mAP) 122 | num = 0 123 | color = ['#00BFFF','#6495ED','darkorange','r'] 124 | plt.figure(figsize=(10,5)) 125 | keys = all_mAP.keys() 126 | keys = sorted(keys) 127 | for i in keys: 128 | 129 | plt.subplot(1,2,num+1) 130 | plt.title(str(i) + 'test' ) 131 | x = np.arange(0,500,5) 132 | y = all_mAP[i] 133 | plt.xlabel('Epoch') 134 | plt.ylabel('mAP') 135 | plt.plot(x,y) 136 | # mAP = all_mAP[i] 137 | # x = np.arange(1,1001,40) 138 | # y = np.zeros(25) 139 | # #print len(y) 140 | # for j in range(25): 141 | # #print i*5 142 | # y[j] = max(mAP[j*8:j*8+8]) 143 | # #y = mAP 144 | # if i == '5*1': 145 | # plt.plot(x,y, label = i)#, color=color[num], linewidth=2.5) 146 | # else: 147 | # plt.plot(x,y,label=i)#, color=color[num]) 148 | num = num + 1 149 | plt.legend(loc='lower right') 150 | plt.show() 151 | 152 | alpha = 0.7 153 | 154 | if __name__=='__main__': 155 | #rename() 156 | #cal_all_mAP(alpha) 157 | draw_mAP() 158 | -------------------------------------------------------------------------------- /evaluation/evaluate_Ekman6.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import os 4 | import json 5 | 6 | def interpolated_prec_rec(prec, rec): 7 | """Interpolated AP - VOCdevkit from VOC 2011. 8 | """ 9 | mprec = np.hstack([[0], prec, [0]]) 10 | mrec = np.hstack([[0], rec, [1]]) 11 | for i in range(len(mprec) - 1)[::-1]: 12 | mprec[i] = max(mprec[i], mprec[i + 1]) 13 | idx = np.where(mrec[1::] != mrec[0:-1])[0] + 1 14 | ap = np.sum((mrec[idx] - mrec[idx - 1]) * mprec[idx]) 15 | return ap 16 | 17 | def cal_mAP(url1, url2, alpha): 18 | f = open(url1, 'r') 19 | data = f.read() 20 | 21 | g = open(url2, 'r') 22 | gt = g.read() 23 | 24 | data = data.split('\n') 25 | data = data[0:len(data) - 1] 26 | pre = np.zeros((len(data), 2)) 27 | 28 | num = 0 29 | for i in data: 30 | temp = i.split(' ') 31 | pre[num,0] = eval(temp[0]) 32 | pre[num,1] = eval(temp[1]) 33 | num = num + 1 34 | 35 | gt = gt.split('\n') 36 | gt = gt[0:len(gt) - 1] 37 | 38 | ground = np.zeros((len(gt)-2, 7)) 39 | 40 | num = 0 41 | for i in gt: 42 | temp = i.split(' ') 43 | ground[num,0] = eval(temp[0]) 44 | ground[num,1] = eval(temp[1]) 45 | ground[num,2] = eval(temp[2]) 46 | ground[num,3] = eval(temp[3]) 47 | ground[num,4] = eval(temp[4]) 48 | ground[num,5] = eval(temp[5]) 49 | ground[num,6] = eval(temp[6]) 50 | if ground[num,6] <= 10: 51 | pre[num,0] = pre[num,0] / 10 52 | pre[num,1] = pre[num,1] / 10 53 | if pre[num,1] > ground[num,6]: 54 | pre[num,1] = ground[num,6] 55 | else: 56 | pre[num,0] = pre[num,0] * ground[num,6] / 100 57 | pre[num,1] = pre[num,1] * ground[num,6] / 100 58 | num = num + 1 59 | 60 | tiou = np.zeros(len(data)) 61 | tp = np.zeros(len(data)) 62 | fp = np.zeros(len(data)) 63 | 64 | b_i_1 = np.maximum(pre[:,0], ground[:,0]) 65 | e_i_1 = np.minimum(pre[:,1], ground[:,1]) 66 | 67 | b_u_1 = np.minimum(pre[:,0], ground[:,0]) 68 | e_u_1 = np.maximum(pre[:,1], ground[:,1]) 69 | 70 | b_i_2 = np.maximum(pre[:,0], ground[:,2]) 71 | e_i_2 = np.minimum(pre[:,1], ground[:,3]) 72 | 73 | b_u_2 = np.minimum(pre[:,0], ground[:,2]) 74 | e_u_2 = np.maximum(pre[:,1], ground[:,3]) 75 | 76 | b_i_3 = np.maximum(pre[:,0], ground[:,4]) 77 | e_i_3 = np.minimum(pre[:,1], ground[:,5]) 78 | 79 | b_u_3 = np.minimum(pre[:,0], ground[:,4]) 80 | e_u_3 = np.maximum(pre[:,1], ground[:,5]) 81 | 82 | tiou_1 = (e_i_1 - b_i_1) / (e_u_1 - b_u_1) 83 | tiou_2 = (e_i_2 - b_i_2) / (e_u_2 - b_u_2) 84 | tiou_3 = (e_i_3 - b_i_3) / (e_u_3 - b_u_3) 85 | print 86 | count = 0 87 | for i in range(len(tiou)): 88 | 89 | if (tiou_1[i] < alpha and (tiou_2[i] < alpha or np.isnan(tiou_2[i])) and (tiou_3[i] < alpha or np.isnan(tiou_3[i]))) : 90 | fp[i] = 1 91 | 92 | else: 93 | tp[i] = 1 94 | 95 | tp = np.cumsum(tp).astype(np.float) 96 | fp = np.cumsum(fp).astype(np.float) 97 | rec = tp / len(data) 98 | prec = tp / (tp + fp) 99 | return interpolated_prec_rec(prec, rec) 100 | 101 | def cal_filter_mAP(url, alpha): 102 | ground_url = url + '/train/position.txt' 103 | filelist = os.listdir(url + '/position/') 104 | filelist = sorted(filelist) 105 | mAP = np.zeros(200) 106 | num = 0 107 | for filename in filelist : 108 | if filename[-4:len(filename)] == '.txt' : 109 | print ground_url 110 | print url + '/position/' + filename 111 | mAP[num] = cal_mAP(url + '/position/' + filename, ground_url, alpha) 112 | num = num + 1 113 | return mAP 114 | 115 | 116 | def rename(): 117 | path = '/Users/mac/Desktop/Ekman_200/' 118 | filelist = os.listdir(path) 119 | for i in filelist: 120 | if i != '.DS_Store': 121 | url = path + i + '/position/' 122 | filelist_n = os.listdir(url) 123 | for j in filelist_n: 124 | if j != '.DS_Store': 125 | name = j.split('_') 126 | if len(name[2]) != '7': 127 | if len(name[2]) == 6: 128 | name[2] = '0' + name[2] 129 | name = '_'.join(name) 130 | os.rename(url + j, url + name) 131 | if len(name[2]) == 5: 132 | name[2] = '00' + name[2] 133 | name = '_'.join(name) 134 | os.rename(url + j, url + name) 135 | 136 | 137 | def cal_all_mAP(): 138 | path = '/Users/mac/Desktop/video_spacial/Ekman_100/' 139 | filelist = os.listdir(path) 140 | all_mAP = {} 141 | for i in filelist: 142 | if i != '.DS_Store' and i[0] != 'z' : 143 | 144 | url = path + i + '/position/' 145 | mAP = cal_filter_mAP(path + i) 146 | all_mAP[i] = list(mAP) 147 | 148 | ff = open('mAP_Ekman6.txt', 'w') 149 | ff.write(json.dumps(all_mAP)) 150 | ff.close() 151 | 152 | def draw_mAP(): 153 | ff = open('mAP_Ekman6.txt', 'r') 154 | all_mAP = ff.read() 155 | all_mAP = json.loads(all_mAP) 156 | num = 1 157 | v = 0 158 | plt.figure(figsize=(20,10)) 159 | for i in all_mAP.keys(): 160 | plt.subplot(2,3,num) 161 | plt.title(i) 162 | plt.xlabel('Epoch') 163 | plt.ylabel('mAP') 164 | mAP = all_mAP[i] 165 | x = np.arange(0,200,1) 166 | y = mAP 167 | 168 | if int(i) != 3 and int(i) != 6: 169 | #print max(y) 170 | v = v + max(y) 171 | 172 | plt.plot(x,y) 173 | num = num + 1 174 | plt.show() 175 | print v / 4.0 176 | 177 | 178 | if __name__=='__main__': 179 | rename() 180 | cal_all_mAP(alpha) 181 | draw_mAP() 182 | 183 | -------------------------------------------------------------------------------- /src/tf_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | # %% Borrowed utils from here: https://github.com/pkmital/tensorflow_tutorials/ 17 | import tensorflow as tf 18 | import numpy as np 19 | 20 | def conv2d(x, n_filters, 21 | k_h=5, k_w=5, 22 | stride_h=2, stride_w=2, 23 | stddev=0.02, 24 | activation=lambda x: x, 25 | bias=True, 26 | padding='SAME', 27 | name="Conv2D"): 28 | """2D Convolution with options for kernel size, stride, and init deviation. 29 | Parameters 30 | ---------- 31 | x : Tensor 32 | Input tensor to convolve. 33 | n_filters : int 34 | Number of filters to apply. 35 | k_h : int, optional 36 | Kernel height. 37 | k_w : int, optional 38 | Kernel width. 39 | stride_h : int, optional 40 | Stride in rows. 41 | stride_w : int, optional 42 | Stride in cols. 43 | stddev : float, optional 44 | Initialization's standard deviation. 45 | activation : arguments, optional 46 | Function which applies a nonlinearity 47 | padding : str, optional 48 | 'SAME' or 'VALID' 49 | name : str, optional 50 | Variable scope to use. 51 | Returns 52 | ------- 53 | x : Tensor 54 | Convolved input. 55 | """ 56 | with tf.variable_scope(name): 57 | w = tf.get_variable( 58 | 'w', [k_h, k_w, x.get_shape()[-1], n_filters], 59 | initializer=tf.truncated_normal_initializer(stddev=stddev)) 60 | conv = tf.nn.conv2d( 61 | x, w, strides=[1, stride_h, stride_w, 1], padding=padding) 62 | if bias: 63 | b = tf.get_variable( 64 | 'b', [n_filters], 65 | initializer=tf.truncated_normal_initializer(stddev=stddev)) 66 | conv = conv + b 67 | return conv 68 | 69 | def conv2d_VALID(x, n_filters, 70 | k_h=5, k_w=5, 71 | stride_h=2, stride_w=2, 72 | stddev=0.02, 73 | activation=lambda x: x, 74 | bias=True, 75 | padding='VALID', 76 | name="Conv2D"): 77 | """2D Convolution with options for kernel size, stride, and init deviation. 78 | Parameters 79 | ---------- 80 | x : Tensor 81 | Input tensor to convolve. 82 | n_filters : int 83 | Number of filters to apply. 84 | k_h : int, optional 85 | Kernel height. 86 | k_w : int, optional 87 | Kernel width. 88 | stride_h : int, optional 89 | Stride in rows. 90 | stride_w : int, optional 91 | Stride in cols. 92 | stddev : float, optional 93 | Initialization's standard deviation. 94 | activation : arguments, optional 95 | Function which applies a nonlinearity 96 | padding : str, optional 97 | 'SAME' or 'VALID' 98 | name : str, optional 99 | Variable scope to use. 100 | Returns 101 | ------- 102 | x : Tensor 103 | Convolved input. 104 | """ 105 | with tf.variable_scope(name): 106 | w = tf.get_variable( 107 | 'w', [k_h, k_w, x.get_shape()[-1], n_filters], 108 | initializer=tf.truncated_normal_initializer(stddev=stddev)) 109 | conv = tf.nn.conv2d( 110 | x, w, strides=[1, stride_h, stride_w, 1], padding=padding) 111 | if bias: 112 | b = tf.get_variable( 113 | 'b', [n_filters], 114 | initializer=tf.truncated_normal_initializer(stddev=stddev)) 115 | conv = conv + b 116 | return conv 117 | 118 | def linear(x, n_units, scope=None, stddev=0.02, 119 | activation=lambda x: x): 120 | """Fully-connected network. 121 | Parameters 122 | ---------- 123 | x : Tensor 124 | Input tensor to the network. 125 | n_units : int 126 | Number of units to connect to. 127 | scope : str, optional 128 | Variable scope to use. 129 | stddev : float, optional 130 | Initialization's standard deviation. 131 | activation : arguments, optional 132 | Function which applies a nonlinearity 133 | Returns 134 | ------- 135 | x : Tensor 136 | Fully-connected output. 137 | """ 138 | shape = x.get_shape().as_list() 139 | 140 | with tf.variable_scope(scope or "Linear"): 141 | matrix = tf.get_variable("Matrix", [shape[1], n_units], tf.float32, 142 | tf.random_normal_initializer(stddev=stddev)) 143 | return activation(tf.matmul(x, matrix)) 144 | 145 | # %% 146 | def weight_variable(shape): 147 | '''Helper function to create a weight variable initialized with 148 | a normal distribution 149 | Parameters 150 | ---------- 151 | shape : list 152 | Size of weight variable 153 | ''' 154 | #initial = tf.random_normal(shape, mean=0.0, stddev=0.01) 155 | initial = tf.zeros(shape) 156 | #initial = tf.truncated_normal(shape, stddev=0.1) 157 | #initial = tf.zeros(shape) 158 | return tf.Variable(initial) 159 | 160 | def weight_variable_cnn(shape): 161 | '''Helper function to create a weight variable initialized with 162 | a normal distribution 163 | Parameters 164 | ---------- 165 | shape : list 166 | Size of weight variable 167 | ''' 168 | #initial = tf.random_normal(shape, mean=0.0, stddev=0.01) 169 | #initial = tf.zeros(shape) 170 | initial = tf.truncated_normal(shape, stddev=0.1) 171 | #initial = tf.zeros(shape) 172 | return tf.Variable(initial) 173 | 174 | # %% 175 | def bias_variable(shape): 176 | '''Helper function to create a bias variable initialized with 177 | a constant value. 178 | Parameters 179 | ---------- 180 | shape : list 181 | Size of weight variable 182 | ''' 183 | initial = tf.random_normal(shape, mean=0.0, stddev=0.01) 184 | return tf.Variable(initial) 185 | 186 | # %% 187 | def dense_to_one_hot(labels, n_classes=2): 188 | """Convert class labels from scalars to one-hot vectors.""" 189 | labels = np.array(labels) 190 | n_labels = labels.shape[0] 191 | index_offset = np.arange(n_labels) * n_classes 192 | labels_one_hot = np.zeros((n_labels, n_classes), dtype=np.float32) 193 | labels_one_hot.flat[index_offset + labels.ravel()] = 1 194 | return labels_one_hot 195 | -------------------------------------------------------------------------------- /src/transform.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | # Modified for frame-transform. 17 | 18 | import tensorflow as tf 19 | 20 | 21 | def transformer(U, theta, out_size, name='SpatialTransformer', **kwargs): 22 | """Spatial Transformer Layer 23 | 24 | Implements a spatial transformer layer as described in [1]_. 25 | Based on [2]_ and edited by David Dao for Tensorflow. 26 | 27 | Parameters 28 | ---------- 29 | U : float 30 | The output of a convolutional net should have the 31 | shape [num_batch, height, width, num_channels]. 32 | theta: float 33 | The output of the 34 | localisation network should be [num_batch, 6]. 35 | out_size: tuple of two ints 36 | The size of the output of the network (height, width) 37 | 38 | References 39 | ---------- 40 | .. [1] Spatial Transformer Networks 41 | Max Jaderberg, Karen Simonyan, Andrew Zisserman, Koray Kavukcuoglu 42 | Submitted on 5 Jun 2015 43 | .. [2] https://github.com/skaae/transformer_network/blob/master/transformerlayer.py 44 | 45 | Notes 46 | ----- 47 | To initialize the network to the identity transform init 48 | ``theta`` to : 49 | identity = np.array([[1., 0., 0.], 50 | [0., 1., 0.]]) 51 | identity = identity.flatten() 52 | theta = tf.Variable(initial_value=identity) 53 | 54 | """ 55 | 56 | def _repeat(x, n_repeats): 57 | with tf.variable_scope('_repeat'): 58 | rep = tf.transpose( 59 | tf.expand_dims(tf.ones(shape=tf.pack([n_repeats, ])), 1), [1, 0]) 60 | rep = tf.cast(rep, 'int32') 61 | x = tf.matmul(tf.reshape(x, (-1, 1)), rep) 62 | return tf.reshape(x, [-1]) 63 | 64 | def _interpolate(im, x, y, out_size): 65 | with tf.variable_scope('_interpolate'): 66 | # constants 67 | num_batch = tf.shape(im)[0] 68 | height = tf.shape(im)[1] 69 | width = tf.shape(im)[2] 70 | channels = tf.shape(im)[3] 71 | 72 | x = tf.cast(x, 'float32') 73 | y = tf.cast(y, 'float32') 74 | height_f = tf.cast(height, 'float32') 75 | width_f = tf.cast(width, 'float32') 76 | out_height = out_size[0] 77 | out_width = out_size[1] 78 | zero = tf.zeros([], dtype='int32') 79 | max_y = tf.cast(tf.shape(im)[1] - 1, 'int32') 80 | max_x = tf.cast(tf.shape(im)[2] - 1, 'int32') 81 | 82 | # scale indices from [-1, 1] to [0, width/height] 83 | x = (x + 1.0)*(width_f) / 2.0 84 | y = (y + 1.0)*(height_f) / 2.0 85 | 86 | # do sampling 87 | x0 = tf.cast(tf.floor(x), 'int32') 88 | x1 = x0 + 1 89 | y0 = tf.cast(tf.floor(y), 'int32') 90 | y1 = y0 + 1 91 | 92 | x0 = tf.clip_by_value(x0, zero, max_x) 93 | x1 = tf.clip_by_value(x1, zero, max_x) 94 | y0 = tf.clip_by_value(y0, zero, max_y) 95 | y1 = tf.clip_by_value(y1, zero, max_y) 96 | dim2 = width 97 | dim1 = width*height 98 | base = _repeat(tf.range(0,num_batch)*dim1, out_height*out_width) 99 | base_y0 = base + y0*dim2 100 | base_y1 = base + y1*dim2 101 | idx_a = base_y0 + x0 102 | idx_b = base_y1 + x0 103 | idx_c = base_y0 + x1 104 | idx_d = base_y1 + x1 105 | 106 | # use indices to lookup pixels in the flat image and restore 107 | # channels dim 108 | im_flat = tf.reshape(im, tf.pack([-1, channels])) 109 | im_flat = tf.cast(im_flat, 'float32') 110 | Ia = tf.gather(im_flat, idx_a) 111 | Ib = tf.gather(im_flat, idx_b) 112 | Ic = tf.gather(im_flat, idx_c) 113 | Id = tf.gather(im_flat, idx_d) 114 | 115 | # and finally calculate interpolated values 116 | x0_f = tf.cast(x0, 'float32') 117 | x1_f = tf.cast(x1, 'float32') 118 | y0_f = tf.cast(y0, 'float32') 119 | y1_f = tf.cast(y1, 'float32') 120 | #wa = tf.expand_dims(((x1_f-x) * (y1_f-y)), 1) 121 | #wb = tf.expand_dims(((x1_f-x) * (y-y0_f)), 1) 122 | wa = tf.expand_dims((y1_f-y), 1) 123 | wb = tf.expand_dims((y-y0_f), 1) 124 | #wc = tf.expand_dims(((x-x0_f) * (y1_f-y)), 1) 125 | #wd = tf.expand_dims(((x-x0_f) * (y-y0_f)), 1) 126 | #output = tf.add_n([wa*Ia, wb*Ib, wc*Ic, wd*Id]) 127 | #output = tf.add_n([wa*Ia*2]) 128 | output = tf.add_n([wa*Ia, wb*Ib]) 129 | return output 130 | 131 | def _meshgrid(height, width): 132 | with tf.variable_scope('_meshgrid'): 133 | # This should be equivalent to: 134 | # x_t, y_t = np.meshgrid(np.linspace(-1, 1, width), 135 | # np.linspace(-1, 1, height)) 136 | # ones = np.ones(np.prod(x_t.shape)) 137 | # grid = np.vstack([x_t.flatten(), y_t.flatten(), ones]) 138 | x_t = tf.matmul(tf.ones(shape=tf.pack([height, 1])), 139 | tf.transpose(tf.expand_dims(tf.linspace(-1.0, 1.0, width), 1), [1, 0])) 140 | y_t = tf.matmul(tf.expand_dims(tf.linspace(-1.0, 1.0, height), 1), 141 | tf.ones(shape=tf.pack([1, width]))) 142 | 143 | x_t_flat = tf.reshape(x_t, (1, -1)) 144 | y_t_flat = tf.reshape(y_t, (1, -1)) 145 | 146 | ones = tf.ones_like(x_t_flat) 147 | grid = tf.concat(0, [x_t_flat, y_t_flat, ones]) 148 | return grid 149 | 150 | def _transform(theta, input_dim, out_size): 151 | with tf.variable_scope('_transform'): 152 | num_batch = tf.shape(input_dim)[0] 153 | height = tf.shape(input_dim)[1] 154 | width = tf.shape(input_dim)[2] 155 | num_channels = tf.shape(input_dim)[3] 156 | theta = tf.reshape(theta, (-1, 1, 2)) 157 | theta = tf.cast(theta, 'float32') 158 | 159 | # grid of (x_t, y_t, 1), eq (1) in ref [1] 160 | height_f = tf.cast(height, 'float32') 161 | width_f = tf.cast(width, 'float32') 162 | out_height = out_size[0] 163 | out_width = out_size[1] 164 | grid = _meshgrid(out_height, out_width) 165 | grid = tf.expand_dims(grid, 0) 166 | grid = tf.reshape(grid, [-1]) 167 | grid = tf.tile(grid, tf.pack([num_batch])) 168 | grid = tf.reshape(grid, tf.pack([num_batch, 3, -1])) 169 | 170 | # Transform A x (x_t, y_t, 1)^T -> (x_s, y_s) 171 | # change here 172 | x = tf.slice(grid, [0, 0, 0], [-1, 1, -1]) 173 | y = tf.slice(grid, [0, 1, 0], [-1, 2, -1]) 174 | trans_y = tf.batch_matmul(theta, y) 175 | T_g = tf.concat(1, [x, trans_y]) 176 | ## 177 | #T_g = tf.batch_matmul(theta, grid) 178 | x_s = tf.slice(T_g, [0, 0, 0], [-1, 1, -1]) 179 | y_s = tf.slice(T_g, [0, 1, 0], [-1, 1, -1]) 180 | x_s_flat = tf.reshape(x_s, [-1]) 181 | y_s_flat = tf.reshape(y_s, [-1]) 182 | 183 | input_transformed = _interpolate( 184 | input_dim, x_s_flat, y_s_flat, 185 | out_size) 186 | 187 | output = tf.reshape( 188 | input_transformed, tf.pack([num_batch, out_height, out_width, num_channels])) 189 | return output 190 | 191 | with tf.variable_scope(name): 192 | 193 | output = _transform(theta, U, out_size) 194 | return output 195 | 196 | 197 | def batch_transformer(U, thetas, out_size, name='BatchSpatialTransformer'): 198 | """Batch Spatial Transformer Layer 199 | 200 | Parameters 201 | ---------- 202 | 203 | U : float 204 | tensor of inputs [num_batch,height,width,num_channels] 205 | thetas : float 206 | a set of transformations for each input [num_batch,num_transforms,6] 207 | out_size : int 208 | the size of the output [out_height,out_width] 209 | 210 | Returns: float 211 | Tensor of size [num_batch*num_transforms,out_height,out_width,num_channels] 212 | """ 213 | with tf.variable_scope(name): 214 | num_batch, num_transforms = map(int, thetas.get_shape().as_list()[:2]) 215 | indices = [[i]*num_transforms for i in xrange(num_batch)] 216 | input_repeated = tf.gather(U, tf.reshape(indices, [-1])) 217 | return transformer(input_repeated, thetas, out_size) 218 | -------------------------------------------------------------------------------- /src/train_Emotion6.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import read_mat as read_mat 3 | import numpy as np 4 | from tf_utils import weight_variable, bias_variable, dense_to_one_hot, weight_variable_cnn 5 | from shuffle_data import shuffle, shuffle_emotion6 6 | from transform import transformer 7 | 8 | x_train = read_mat.read_mat_v('/home/g_jiarui/video_spacial/data/multi_train_data.mat', 'train_data')/200 9 | y_train = read_mat.read_mat('/home/g_jiarui/video_spacial/data/multi_train_label.mat')['train_label'] 10 | 11 | X_valid = read_mat.read_mat('/home/g_jiarui/video_spacial/data/multi_validation_data.mat')['validation_data']/200 12 | y_valid = read_mat.read_mat('/home/g_jiarui/video_spacial/data/multi_validation_label.mat')['validation_label'][0] 13 | 14 | X_test = read_mat.read_mat('/home/g_jiarui/video_spacial/data/multi_test_data.mat')['test_data']/200 15 | y_test = read_mat.read_mat('/home/g_jiarui/video_spacial/data/multi_test_label.mat')['test_label'][0] 16 | 17 | position_train = read_mat.read_mat('/home/g_jiarui/video_spacial/data/multi_train_position.mat')['train_position'] 18 | position_val = read_mat.read_mat('/home/g_jiarui/video_spacial/data/multi_validation_position.mat')['validation_position'] 19 | np.savetxt("/home/g_jiarui/video_spacial/result/1/train/val_position.txt",position_val,fmt="%d") 20 | 21 | x_train_trans = np.transpose(x_train) 22 | [X_train,y_train,position] = shuffle_emotion6(x_train_trans, y_train, position_train, 2400) 23 | Y_train = dense_to_one_hot(y_train, n_classes=6) 24 | Y_valid = dense_to_one_hot(y_valid, n_classes=6) 25 | Y_test = dense_to_one_hot(y_test, n_classes=6) 26 | 27 | X_train = np.resize(X_train, (2400, 30, 4096, 1)) 28 | X_valid = np.resize(X_valid, (600, 30, 4096, 1)) 29 | X_test = np.resize(X_test, (600, 30, 4096, 1)) 30 | 31 | x = tf.placeholder(tf.float32, [None, 30, 4096, 1]) 32 | y = tf.placeholder(tf.float32,[None, 6]) 33 | keep_prob = tf.placeholder(tf.float32) 34 | x_flat = tf.reshape(x, [-1, 30 * 4096]) 35 | x_trans = tf.reshape(x, [-1, 30, 4096, 1]) 36 | 37 | W_fc_loc1 = weight_variable([30 * 4096, 20]) 38 | b_fc_loc1 = bias_variable([20]) 39 | 40 | initial = np.array([0.5, 0]) 41 | initial = initial.astype('float32') 42 | initial = initial.flatten() 43 | 44 | W_fc_loc2 = weight_variable([20, 2]) 45 | b_fc_loc2 = bias_variable([2]) 46 | b_fc_loc2 = tf.Variable(initial_value=initial, name='b_fc_loc2') 47 | 48 | h_fc_loc1 = tf.nn.tanh(tf.matmul(x_flat, W_fc_loc1) + b_fc_loc1) 49 | h_fc_loc1_drop = tf.nn.dropout(h_fc_loc1, keep_prob) 50 | h_fc_loc2 = tf.nn.tanh(tf.matmul(h_fc_loc1_drop, W_fc_loc2) + b_fc_loc2) 51 | 52 | out_size = (10, 4096) 53 | h_trans = transformer(x_trans, h_fc_loc2, out_size) 54 | h_flat = tf.reshape(h_trans, [-1, 10, 4096, 1]) 55 | 56 | # start cnn 57 | 58 | #filter_size = 3 59 | filter_size = 5 60 | n_filters_1 = 8 61 | W_conv1 = weight_variable_cnn([filter_size, 1, 1, n_filters_1]) 62 | b_conv1 = bias_variable([n_filters_1]) 63 | h_conv1 = tf.nn.relu( 64 | tf.nn.conv2d(input=h_trans, 65 | filter=W_conv1, 66 | strides=[1, 1, 1, 1], 67 | padding='SAME') + 68 | b_conv1) 69 | 70 | h_conv1_flat = tf.reshape(h_conv1, [-1, 10*4096*n_filters_1]) 71 | 72 | n_fc = 32 73 | W_fc1 = weight_variable_cnn([10 * 4096 * n_filters_1, n_fc]) 74 | b_fc1 = bias_variable([n_fc]) 75 | h_fc1 = tf.nn.tanh(tf.matmul(h_conv1_flat/200, W_fc1) + b_fc1) 76 | h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) 77 | W_fc2 = weight_variable_cnn([n_fc, 6]) 78 | b_fc2 = bias_variable([6]) 79 | y_logits = tf.matmul(h_fc1_drop, W_fc2) + b_fc2 80 | 81 | 82 | ''' 83 | #1*3 84 | #filter_size = 3 85 | filter_size = 3 86 | n_filters_1 = 8 87 | W_conv1 = weight_variable_cnn([1, filter_size, 1, n_filters_1]) 88 | b_conv1 = bias_variable([n_filters_1]) 89 | h_conv1 = tf.nn.relu( 90 | tf.nn.conv2d(input=h_trans, 91 | filter=W_conv1, 92 | strides=[1, 1, 1, 1], 93 | padding='SAME') + 94 | b_conv1) 95 | 96 | h_conv1_flat = tf.reshape(h_conv1, [-1, 10*4096*n_filters_1]) 97 | 98 | n_fc = 32 99 | W_fc1 = weight_variable_cnn([10 * 4096 * n_filters_1, n_fc]) 100 | b_fc1 = bias_variable([n_fc]) 101 | h_fc1 = tf.nn.tanh(tf.matmul(h_conv1_flat, W_fc1) + b_fc1) 102 | h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) 103 | W_fc2 = weight_variable_cnn([n_fc, 6]) 104 | b_fc2 = bias_variable([6]) 105 | y_logits = tf.matmul(h_fc1_drop, W_fc2) + b_fc2 106 | ''' 107 | 108 | ''' 109 | #3*3 110 | filter_size = 3 111 | n_filters_1 = 16 112 | W_conv1 = weight_variable_cnn([filter_size, filter_size, 1, n_filters_1]) 113 | b_conv1 = bias_variable([n_filters_1]) 114 | 115 | h_conv1 = tf.nn.relu( 116 | tf.nn.conv2d(input=h_trans, 117 | filter=W_conv1, 118 | strides=[1, 2, 2, 1], 119 | padding='SAME') + 120 | b_conv1) 121 | 122 | h_conv1_flat = tf.reshape(h_conv1, [-1, 3 * 2048 * n_filters_1]) 123 | n_fc = 32 124 | W_fc1 = weight_variable_cnn([3 * 2048 * n_filters_1, n_fc]) 125 | b_fc1 = bias_variable([n_fc]) 126 | h_fc1 = tf.nn.tanh(tf.matmul(h_conv1_flat, W_fc1) + b_fc1) 127 | h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) 128 | W_fc2 = weight_variable_cnn([n_fc, 2]) 129 | b_fc2 = bias_variable([2]) 130 | y_logits = tf.matmul(h_fc1_drop, W_fc2) + b_fc2 131 | ''' 132 | 133 | 134 | ''' 135 | #3*4096 136 | filter_size = 3 137 | n_filters_1 = 256 138 | W_conv1 = weight_variable_cnn([filter_size, 4096, 1, n_filters_1]) 139 | b_conv1 = bias_variable([n_filters_1]) 140 | 141 | h_conv1 = tf.nn.relu( 142 | tf.nn.conv2d(input=h_trans, 143 | filter=W_conv1, 144 | strides=[1, 1, 1, 1], 145 | padding='VALID') + 146 | b_conv1) 147 | 148 | h_conv1_flat = tf.reshape(h_conv1, [-1, 4 * 1 * n_filters_1]) 149 | 150 | n_fc = 64 151 | W_fc1 = weight_variable_cnn([4 * 1 * n_filters_1, n_fc]) 152 | b_fc1 = bias_variable([n_fc]) 153 | h_fc1 = tf.nn.tanh(tf.matmul(h_conv1_flat, W_fc1) + b_fc1) 154 | h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) 155 | 156 | W_fc2 = weight_variable_cnn([n_fc, 2]) 157 | b_fc2 = bias_variable([2]) 158 | y_logits = tf.matmul(h_fc1_drop, W_fc2) + b_fc2 159 | ''' 160 | 161 | ''' 162 | #fc_2 163 | n_fc = 64 164 | #W_fc1 = weight_variable_cnn([4 * 1 * n_filters_1, n_fc]) 165 | W_fc1 = weight_variable_cnn([30*4096, n_fc]) 166 | b_fc1 = bias_variable([n_fc]) 167 | h_fc1 = tf.nn.tanh(tf.matmul(x_flat, W_fc1) + b_fc1) 168 | #h_fc1 = tf.nn.tanh(tf.matmul(h_flat, W_fc1) + b_fc1) 169 | 170 | h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) 171 | 172 | W_fc2 = weight_variable_cnn([n_fc, 6]) 173 | b_fc2 = bias_variable([6]) 174 | y_logits = tf.matmul(h_fc1_drop, W_fc2) + b_fc2 175 | ''' 176 | 177 | cross_entropy = tf.reduce_mean( 178 | tf.nn.softmax_cross_entropy_with_logits(y_logits, y)) 179 | opt = tf.train.AdamOptimizer() 180 | optimizer = opt.minimize(cross_entropy) 181 | 182 | correct_prediction = tf.equal(tf.argmax(y_logits, 1), tf.argmax(y, 1)) 183 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float')) 184 | grads = opt.compute_gradients(cross_entropy, [b_fc_loc2]) 185 | saver = tf.train.Saver() 186 | 187 | sess = tf.Session() 188 | sess.run(tf.initialize_all_variables()) 189 | 190 | iter_per_epoch = 60 191 | n_epochs = 1000 192 | train_size = 2400 193 | 194 | indices = np.linspace(0, train_size - 1, iter_per_epoch) 195 | indices = indices.astype('int') 196 | max_acc = 0; 197 | out_theta = [] 198 | out_position = [] 199 | ff = open('/home/g_jiarui/video_spacial/result/1/train/accuracy.txt','w+') 200 | 201 | for epoch_i in range(n_epochs): 202 | for iter_i in range(iter_per_epoch - 1): 203 | batch_xs = X_train[indices[iter_i]:indices[iter_i+1]] 204 | batch_ys = Y_train[indices[iter_i]:indices[iter_i+1]] 205 | 206 | if iter_i % 10 == 0: 207 | loss = sess.run(cross_entropy, 208 | feed_dict={ 209 | x: batch_xs, 210 | y: batch_ys, 211 | keep_prob: 1.0 212 | }) 213 | print('Iteration: ' + str(iter_i) + ' Loss: ' + str(loss)) 214 | 215 | sess.run(optimizer, feed_dict={ 216 | x: batch_xs, y: batch_ys, keep_prob: 0.75}) 217 | 218 | acc = str(sess.run(accuracy,feed_dict={ 219 | x: X_valid, 220 | y: Y_valid, 221 | keep_prob: 1.0 222 | })) 223 | print('Accuracy (%d): ' %epoch_i + acc) 224 | print('test (%d): ' % epoch_i + str(sess.run(accuracy, 225 | feed_dict={ 226 | x: X_test, 227 | y: Y_test, 228 | keep_prob: 1.0 229 | }))) 230 | 231 | #theta = sess.run(h_fc_loc2, feed_dict={ 232 | # x: batch_xs, keep_prob: 1.0}) 233 | #print theta 234 | 235 | #grad_vals = sess.run([g for (g,v) in grads], feed_dict={ 236 | # x: batch_xs, y: batch_ys, keep_prob: 1.0}) 237 | #print 'grad_vals: ', grad_vals 238 | 239 | if epoch_i % 5 == 1: 240 | for iter_i in range(iter_per_epoch - 1): 241 | batch_xs = X_train[indices[iter_i]:indices[iter_i+1]] 242 | batch_ys = Y_train[indices[iter_i]:indices[iter_i+1]] 243 | 244 | theta = sess.run(h_fc_loc2, feed_dict={ 245 | x: batch_xs, keep_prob: 1.0}) 246 | 247 | if iter_i == 0: 248 | out_theta = theta 249 | else: 250 | out_theta = np.concatenate((out_theta,theta),axis=0) 251 | 252 | np.savetxt("/home/g_jiarui/video_spacial/result/1/train_theta_result_"+str(epoch_i)+".txt",out_theta,fmt="%f") 253 | 254 | val_theta = sess.run(h_fc_loc2, feed_dict={ 255 | x: X_valid, y: Y_valid, keep_prob: 1.0}) 256 | 257 | np.savetxt("/home/g_jiarui/video_spacial/result/1/val_theta_result_"+str(epoch_i)+".txt",val_theta,fmt="%f") 258 | 259 | #saver.save(sess, '/home/hs/video_spacial/syhthetic/color/resul_10t/3*3_initial=0.5/synthetic_data.tfmodel'); 260 | 261 | ff.write('Accuracy_' + str(epoch_i) + '_' + str(acc) + '\n') 262 | 263 | ff.close() 264 | 265 | -------------------------------------------------------------------------------- /src/random_emotion6.m: -------------------------------------------------------------------------------- 1 | %% random emotion6 2 | %avg : joy : 3230 / 4096 = 0 3 | %most: 0-10 4 | % function random_emotion6(opt,url) 5 | % feats = load(url); 6 | % feats = feats.feats; 7 | % label = regexp(url,'/','split'); 8 | % label = label{7}; 9 | % data = zeros(500, opt, 4096); 10 | % position = zeros(500, 2); 11 | % 12 | % for i = 1:500 13 | % i 14 | % [data(i,:,:), position(i,:)] = random_data(opt,feats); 15 | % end 16 | % 17 | % save('joy_data_500_fc7.mat', 'data'); 18 | % save('joy_position_500_fc7.mat', 'position'); 19 | % end 20 | % 21 | % function [data,position] = random_data(opt,feats) 22 | % data = zeros(opt, 4096); 23 | % senti_len = randi(5, 1) + 5; 24 | % 25 | % position = zeros(1, 2); 26 | % begin = randi(opt - senti_len - 1, 1); 27 | % stop = begin + senti_len - 1; 28 | % position(1,1) = begin; 29 | % position(1,2) = stop; 30 | % 31 | % for i = 1:opt 32 | % if i >= begin && i <= stop 33 | % j = randi(330, 1); 34 | % data(i,:) = feats(j,:); 35 | % else 36 | % for j = 1:4096 37 | % m = randi(4, 1); 38 | % if m / 4 == 1 39 | % data(i,j) = rand(1) * 10; 40 | % end 41 | % end 42 | % % random_num = rand(1,1000) * 15; 43 | % % data(i,:) = random_num; 44 | % end 45 | % end 46 | % end 47 | 48 | %% random emotion6 distort 49 | %joy data: joy:12-20; anger: 3-5 50 | %anger data: anger:12-20; joy: 3-5 51 | function [train_data, train_position, test_data, test_position, validation_data, validation_position] = random_emotion6(opt,url,anger_feats, joy_feats, disgust_feats, sadness_feats, surprise_feats, fear_feats) 52 | feats = load(url); 53 | feats = feats.feats; 54 | l = {'anger', 'fear', 'joy', 'disgust', 'surprise', 'sadness'}; 55 | black_feats = load('/Volumes/Transcend/mat/black_white/black_fc7_100.mat'); 56 | black = black_feats.feats; 57 | white_feats = load('/Volumes/Transcend/mat/black_white/white_fc7_100.mat'); 58 | white = white_feats.feats; 59 | train_data = zeros(400, opt, 4096); 60 | train_position = zeros(400, 2); 61 | test_data = zeros(100, opt, 4096); 62 | test_position = zeros(100, 2); 63 | validation_data = zeros(100, opt, 4096); 64 | validation_position = zeros(100, 2); 65 | 66 | for i = 1:400 67 | i 68 | [train_data(i,:,:), train_position(i,:)] = random_data_train(l,opt,feats,black,white,anger_feats, joy_feats, disgust_feats, sadness_feats, surprise_feats, fear_feats); 69 | end 70 | 71 | for i = 1:100 72 | i 73 | [test_data(i,:,:), test_position(i,:)] = random_data_test(l,opt,feats,black,white,anger_feats, joy_feats, disgust_feats, sadness_feats, surprise_feats, fear_feats); 74 | [validation_data(i,:,:), validation_position(i,:)] = random_data_validation(l,opt,feats,black,white,anger_feats, joy_feats, disgust_feats, sadness_feats, surprise_feats, fear_feats); 75 | end 76 | 77 | end 78 | 79 | function [data,position,position_dis] = random_data_train(l,opt,feats,black,white,anger_feats, joy_feats, disgust_feats, sadness_feats, surprise_feats, fear_feats) 80 | data = zeros(opt, 4096); 81 | senti_len = randi(8, 1) + 12; 82 | dis_len = randi(3, 1) + 2; 83 | tmp = randi(2, 1); 84 | 85 | if tmp == 1 86 | position = zeros(1, 2); 87 | begin = randi(opt - senti_len - dis_len - 1, 1); 88 | stop = begin + senti_len - 1; 89 | position(1,1) = begin; 90 | position(1,2) = stop; 91 | 92 | position_dis = zeros(1, 2); 93 | begin_dis = randi(opt - stop - 1 - dis_len, 1) + stop; 94 | stop_dis = begin_dis + dis_len - 1; 95 | position_dis(1,1) = begin_dis; 96 | position_dis(1,2) = stop_dis; 97 | else 98 | position_dis = zeros(1, 2); 99 | begin_dis = randi(opt - senti_len - dis_len - 1, 1); 100 | stop_dis = begin_dis + dis_len - 1; 101 | position_dis(1,1) = begin_dis; 102 | position_dis(1,2) = stop_dis; 103 | 104 | position = zeros(1, 2); 105 | begin = randi(opt - stop_dis - 1 - senti_len, 1) + stop_dis; 106 | stop = begin + senti_len - 1; 107 | position(1,1) = begin; 108 | position(1,2) = stop; 109 | end 110 | 111 | for i = 1:opt 112 | if i >= begin && i <= stop 113 | j = randi(210, 1); 114 | data(i,:) = feats(j,:); 115 | elseif i >= begin_dis && i <= stop_dis 116 | color = randi(2,1); 117 | if color == 1 118 | j = randi(60, 1); 119 | data(i,:) = black(j,:); 120 | else 121 | j = randi(60, 1); 122 | data(i,:) = white(j,:); 123 | end 124 | else 125 | j = randi(210, 1); 126 | temp = randi(6, 1); 127 | c = char(l{temp}); 128 | c = eval([c,'_feats']); 129 | data(i,:) = c(j,:); 130 | end 131 | end 132 | end 133 | 134 | function [data,position,position_dis] = random_data_test(l,opt,feats,black,white,anger_feats, joy_feats, disgust_feats, sadness_feats, surprise_feats, fear_feats) 135 | data = zeros(opt, 4096); 136 | senti_len = randi(8, 1) + 12; 137 | dis_len = randi(3, 1) + 2; 138 | tmp = randi(2, 1); 139 | 140 | if tmp == 1 141 | position = zeros(1, 2); 142 | begin = randi(opt - senti_len - dis_len - 1, 1); 143 | stop = begin + senti_len - 1; 144 | position(1,1) = begin; 145 | position(1,2) = stop; 146 | 147 | position_dis = zeros(1, 2); 148 | begin_dis = randi(opt - stop - 1 - dis_len, 1) + stop; 149 | stop_dis = begin_dis + dis_len - 1; 150 | position_dis(1,1) = begin_dis; 151 | position_dis(1,2) = stop_dis; 152 | else 153 | position_dis = zeros(1, 2); 154 | begin_dis = randi(opt - senti_len - dis_len - 1, 1); 155 | stop_dis = begin_dis + dis_len - 1; 156 | position_dis(1,1) = begin_dis; 157 | position_dis(1,2) = stop_dis; 158 | 159 | position = zeros(1, 2); 160 | begin = randi(opt - stop_dis - 1 - senti_len, 1) + stop_dis; 161 | stop = begin + senti_len - 1; 162 | position(1,1) = begin; 163 | position(1,2) = stop; 164 | end 165 | 166 | for i = 1:opt 167 | if i >= begin && i <= stop 168 | j = randi(60, 1) + 210; 169 | data(i,:) = feats(j,:); 170 | elseif i >= begin_dis && i <= stop_dis 171 | color = randi(2,1); 172 | if color == 1 173 | j = randi(20, 1) + 60; 174 | data(i,:) = black(j,:); 175 | else 176 | j = randi(20, 1) + 60; 177 | data(i,:) = white(j,:); 178 | end 179 | else 180 | j = randi(60, 1) + 210; 181 | temp = randi(6, 1); 182 | c = char(l{temp}); 183 | c = eval([c,'_feats']); 184 | data(i,:) = c(j,:); 185 | end 186 | end 187 | end 188 | 189 | function [data,position,position_dis] = random_data_validation(l,opt,feats,black,white,anger_feats, joy_feats, disgust_feats, sadness_feats, surprise_feats, fear_feats) 190 | data = zeros(opt, 4096); 191 | senti_len = randi(8, 1) + 12; 192 | dis_len = randi(3, 1) + 2; 193 | tmp = randi(2, 1); 194 | 195 | if tmp == 1 196 | position = zeros(1, 2); 197 | begin = randi(opt - senti_len - dis_len - 1, 1); 198 | stop = begin + senti_len - 1; 199 | position(1,1) = begin; 200 | position(1,2) = stop; 201 | 202 | position_dis = zeros(1, 2); 203 | begin_dis = randi(opt - stop - 1 - dis_len, 1) + stop; 204 | stop_dis = begin_dis + dis_len - 1; 205 | position_dis(1,1) = begin_dis; 206 | position_dis(1,2) = stop_dis; 207 | else 208 | position_dis = zeros(1, 2); 209 | begin_dis = randi(opt - senti_len - dis_len - 1, 1); 210 | stop_dis = begin_dis + dis_len - 1; 211 | position_dis(1,1) = begin_dis; 212 | position_dis(1,2) = stop_dis; 213 | 214 | position = zeros(1, 2); 215 | begin = randi(opt - stop_dis - 1 - senti_len, 1) + stop_dis; 216 | stop = begin + senti_len - 1; 217 | position(1,1) = begin; 218 | position(1,2) = stop; 219 | end 220 | 221 | for i = 1:opt 222 | if i >= begin && i <= stop 223 | j = randi(60, 1) + 270; 224 | data(i,:) = feats(j,:); 225 | elseif i >= begin_dis && i <= stop_dis 226 | color = randi(2,1); 227 | if color == 1 228 | j = randi(20, 1) + 80; 229 | data(i,:) = black(j,:); 230 | else 231 | j = randi(20, 1) + 80; 232 | data(i,:) = white(j,:); 233 | end 234 | else 235 | j = randi(60, 1) + 270; 236 | temp = randi(6, 1); 237 | c = char(l{temp}); 238 | c = eval([c,'_feats']); 239 | data(i,:) = c(j,:); 240 | end 241 | end 242 | end 243 | 244 | 245 | %% random emotion6 fc7 distort all by color 246 | % %joy data: joy:10-20; other: black / white 247 | % %anger data: anger:10-20; other: black / white 248 | % function random_emotion6(opt,url) 249 | % feats = load(url); 250 | % feats = feats.feats; 251 | % black_feats = load('/Users/mac/Desktop/video_spacial/mat/black_white/black.mat'); 252 | % black = black_feats.feats; 253 | % white_feats = load('/Users/mac/Desktop/video_spacial/mat/black_white/white.mat'); 254 | % white = white_feats.feats; 255 | % label = regexp(url,'/','split'); 256 | % label = label{7}; 257 | % data = zeros(500, opt, 4096); 258 | % position = zeros(500, 2); 259 | % 260 | % for i = 1:500 261 | % i 262 | % [data(i,:,:), position(i,:)] = random_data(opt,feats,black,white); 263 | % end 264 | % 265 | % save('./mat/emotion6/all_color_joy_data_500_fc7.mat', 'data'); 266 | % save('./mat/emotion6/all_color_joy_position_500_fc7.mat', 'position'); 267 | % end 268 | % 269 | % function [data,position] = random_data(opt,feats,black,white) 270 | % data = zeros(opt, 4096); 271 | % senti_len = randi(10, 1) + 10; 272 | % tmp_begin = randi(2, 1); 273 | % tmp_end = randi(2, 1); 274 | % 275 | % position = zeros(1, 2); 276 | % begin = randi(opt - senti_len - 1, 1); 277 | % stop = begin + senti_len - 1; 278 | % position(1,1) = begin; 279 | % position(1,2) = stop; 280 | % 281 | % for i = 1:opt 282 | % if i >= begin && i <= stop 283 | % j = randi(330, 1); 284 | % data(i,:) = feats(j,:); 285 | % elseif i >= stop 286 | % if tmp_end == 1 287 | % j = randi(36, 1); 288 | % data(i,:) = black(j,:); 289 | % else 290 | % j = randi(34, 1); 291 | % data(i,:) = white(j,:); 292 | % end 293 | % else 294 | % if tmp_begin == 1 295 | % j = randi(36, 1); 296 | % data(i,:) = black(j,:); 297 | % else 298 | % j = randi(34, 1); 299 | % data(i,:) = white(j,:); 300 | % end 301 | % end 302 | % end 303 | % end 304 | 305 | %% random emotion6 conv5 distort all by color 306 | % %joy data: joy:10-20; other: black / white 307 | % %anger data: anger:10-20; other: black / white 308 | % function random_emotion6(opt,url) 309 | % feats = load(url); 310 | % feats = feats.feats; 311 | % black_feats = load('/Volumes/Transcend/mat/black_white/black_conv5.mat'); 312 | % black = black_feats.feats; 313 | % white_feats = load('/Volumes/Transcend/mat/black_white/white_conv5.mat'); 314 | % white = white_feats.feats; 315 | % label = regexp(url,'/','split'); 316 | % label = label{7}; 317 | % data = zeros(500, opt, 43264); 318 | % position = zeros(500, 2); 319 | % 320 | % for i = 1:500 321 | % i 322 | % [data(i,:,:), position(i,:)] = random_data(opt,feats,black,white); 323 | % end 324 | % 325 | % save('/Volumes/Transcend/mat/emotion6/all_color_joy_data_500_conv5.mat', 'data'); 326 | % save('/Volumes/Transcend/mat/emotion6/all_color_joy_position_500_conv5.mat', 'position'); 327 | % end 328 | % 329 | % function [data,position] = random_data(opt,feats,black,white) 330 | % data = zeros(opt, 43264); 331 | % senti_len = randi(10, 1) + 10; 332 | % tmp_begin = randi(2, 1); 333 | % tmp_end = randi(2, 1); 334 | % 335 | % position = zeros(1, 2); 336 | % begin = randi(opt - senti_len - 1, 1); 337 | % stop = begin + senti_len - 1; 338 | % position(1,1) = begin; 339 | % position(1,2) = stop; 340 | % 341 | % for i = 1:opt 342 | % if i >= begin && i <= stop 343 | % j = randi(330, 1); 344 | % data(i,:) = feats(j,:); 345 | % elseif i >= stop 346 | % if tmp_end == 1 347 | % j = randi(36, 1); 348 | % data(i,:) = black(j,:); 349 | % else 350 | % j = randi(34, 1); 351 | % data(i,:) = white(j,:); 352 | % end 353 | % else 354 | % if tmp_begin == 1 355 | % j = randi(36, 1); 356 | % data(i,:) = black(j,:); 357 | % else 358 | % j = randi(34, 1); 359 | % data(i,:) = white(j,:); 360 | % end 361 | % end 362 | % end 363 | % end 364 | 365 | %% random all classes in emotion6 fc7 distort by color 366 | %joy data: joy:10-20; other: black / white 367 | %anger data: anger:10-20; other: black / white 368 | % function [train_data, train_position, test_data, test_position, validation_data, validation_position] = random_emotion6(opt,url) 369 | % feats = load(url); 370 | % feats = feats.feats; 371 | % black_feats = load('/Volumes/Transcend/mat/black_white/black_fc7_100.mat'); 372 | % black = black_feats.feats; 373 | % white_feats = load('/Volumes/Transcend/mat/black_white/white_fc7_100.mat'); 374 | % white = white_feats.feats; 375 | % label = regexp(url,'/','split'); 376 | % label = label{7}; 377 | % train_data = zeros(400, opt, 4096); 378 | % train_position = zeros(400, 2); 379 | % test_data = zeros(100, opt, 4096); 380 | % test_position = zeros(100, 2); 381 | % validation_data = zeros(100, opt, 4096); 382 | % validation_position = zeros(100, 2); 383 | % 384 | % for i = 1:400 385 | % i 386 | % [train_data(i,:,:), train_position(i,:)] = random_data_train(opt,feats,black,white); 387 | % end 388 | % 389 | % for i = 1:100 390 | % i 391 | % [test_data(i,:,:), test_position(i,:)] = random_data_test(opt,feats,black,white); 392 | % [validation_data(i,:,:), validation_position(i,:)] = random_data_validation(opt,feats,black,white); 393 | % end 394 | % 395 | % %save('./mat/emotion6/all_color_joy_data_500_fc7.mat', 'data'); 396 | % %save('./mat/emotion6/all_color_joy_position_500_fc7.mat', 'position'); 397 | % end 398 | % 399 | % function [data,position] = random_data_train(opt,feats,black,white) 400 | % data = zeros(opt, 4096); 401 | % senti_len = randi(10, 1) + 10; 402 | % tmp_begin = randi(2, 1); 403 | % tmp_end = randi(2, 1); 404 | % 405 | % position = zeros(1, 2); 406 | % begin = randi(opt - senti_len - 1, 1); 407 | % stop = begin + senti_len - 1; 408 | % position(1,1) = begin; 409 | % position(1,2) = stop; 410 | % 411 | % for i = 1:opt 412 | % if i >= begin && i <= stop 413 | % j = randi(210, 1); 414 | % data(i,:) = feats(j,:); 415 | % elseif i >= stop 416 | % if tmp_end == 1 417 | % j = randi(60, 1); 418 | % data(i,:) = black(j,:); 419 | % else 420 | % j = randi(60, 1); 421 | % data(i,:) = white(j,:); 422 | % end 423 | % else 424 | % if tmp_begin == 1 425 | % j = randi(60, 1); 426 | % data(i,:) = black(j,:); 427 | % else 428 | % j = randi(60, 1); 429 | % data(i,:) = white(j,:); 430 | % end 431 | % end 432 | % end 433 | % end 434 | % 435 | % function [data,position] = random_data_test(opt,feats,black,white) 436 | % data = zeros(opt, 4096); 437 | % senti_len = randi(10, 1) + 10; 438 | % tmp_begin = randi(2, 1); 439 | % tmp_end = randi(2, 1); 440 | % 441 | % position = zeros(1, 2); 442 | % begin = randi(opt - senti_len - 1, 1); 443 | % stop = begin + senti_len - 1; 444 | % position(1,1) = begin; 445 | % position(1,2) = stop; 446 | % 447 | % for i = 1:opt 448 | % if i >= begin && i <= stop 449 | % j = randi(60, 1) + 210; 450 | % data(i,:) = feats(j,:); 451 | % elseif i >= stop 452 | % if tmp_end == 1 453 | % j = randi(20, 1) + 60; 454 | % data(i,:) = black(j,:); 455 | % else 456 | % j = randi(20, 1) + 60; 457 | % data(i,:) = white(j,:); 458 | % end 459 | % else 460 | % if tmp_begin == 1 461 | % j = randi(20, 1) + 60; 462 | % data(i,:) = black(j,:); 463 | % else 464 | % j = randi(20, 1) + 60; 465 | % data(i,:) = white(j,:); 466 | % end 467 | % end 468 | % end 469 | % end 470 | % 471 | % function [data,position] = random_data_validation(opt,feats,black,white) 472 | % data = zeros(opt, 4096); 473 | % senti_len = randi(10, 1) + 10; 474 | % tmp_begin = randi(2, 1); 475 | % tmp_end = randi(2, 1); 476 | % 477 | % position = zeros(1, 2); 478 | % begin = randi(opt - senti_len - 1, 1); 479 | % stop = begin + senti_len - 1; 480 | % position(1,1) = begin; 481 | % position(1,2) = stop; 482 | % 483 | % for i = 1:opt 484 | % if i >= begin && i <= stop 485 | % j = randi(60, 1) + 270; 486 | % data(i,:) = feats(j,:); 487 | % elseif i >= stop 488 | % if tmp_end == 1 489 | % j = randi(20, 1) + 80; 490 | % data(i,:) = black(j,:); 491 | % else 492 | % j = randi(20, 1) + 80; 493 | % data(i,:) = white(j,:); 494 | % end 495 | % else 496 | % if tmp_begin == 1 497 | % j = randi(20, 1) + 80; 498 | % data(i,:) = black(j,:); 499 | % else 500 | % j = randi(20, 1) + 80; 501 | % data(i,:) = white(j,:); 502 | % end 503 | % end 504 | % end 505 | % end 506 | --------------------------------------------------------------------------------