├── README.md ├── function ├── data_generator.py ├── data_generator_v2.py └── log_file.py ├── layers ├── network.png └── transformer.py └── networks ├── basic_4_fc.py ├── model_add.py ├── model_average.py └── model_multiply.py /README.md: -------------------------------------------------------------------------------- 1 | # Two-Stream-CNN implement in Keras 2 | Two Stream CNN is proposed in [__SKELETON-BASED ACTION RECOGNITION WITH CONVOLUTIONAL NEURAL NETWORKS__](https://arxiv.org/abs/1704.07595), which is used for skeleton-based action recognition. It maps a skeleton sequence to an image( coordinates x,y,z to image R,G,B ). And they specially designed skeleton transformer module to rearrange and select important skeleton joints automatically. 3 | ## Requirments 4 | * Python3 5 | * Keras 6 | * h5py 7 | * matplotlib 8 | * numpy 9 | ## Network Architecture 10 | The network mainly consists of four modules which are `Skeleton Transformer`, `ConvNet`, `Feature Fusion` and `Classification`. The inputs of two stream are raw data(x, y, z) and frame difference respectively. As show below : 11 | ![Two Stream CNN](/layers/network.png) 12 | ## Usage 13 | 1. __function/data_generator.py__ : generate the inputs numpy array of two stream 14 | 15 | 2. __layers/transformer__ : the layer of Skeleton Transformer implement in Keras 16 | 17 | 3. __network/__ : the fold has four flies with different feature fusion way 18 | ## Result 19 | | model | accuracy(cs) | 20 | | :---------: | :---------: | 21 | | base line | 83.2% | 22 | | my model | 80.7% | 23 | 24 | Introduce `attention mechanism` to Skeleton Transformer module. Then, the accurancy can reach at 82.1%. 25 | ## Contact 26 | If you have any questions, please feel free to contact me. 27 | Duohan Liang (duohanl@outlook.com) 28 | -------------------------------------------------------------------------------- /function/data_generator.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import h5py 3 | import random 4 | 5 | 6 | class DataGenerator(object): 7 | 8 | def __init__(self, h5file_path, batch_size=64, frames=30): 9 | self._h5file_path = h5file_path 10 | self._batch_size = batch_size 11 | self._frames = frames 12 | 13 | def get_data_name(self): 14 | # 获取hdf5文件中的data set名字，返回一个list 15 | with h5py.File(self._h5file_path, 'r') as h5file: 16 | return [data_name for data_name in h5file] 17 | 18 | def get_cursors(self, data_name): 19 | with h5py.File(self._h5file_path, 'r') as h5file: 20 | data = h5file[data_name] 21 | data_frames = data.shape[1] 22 | # 生成30帧随机帧 23 | output_cursors = [] 24 | add_num = 1 25 | remainder = data_frames % self._frames 26 | integer = data_frames // self._frames 27 | # 这里的+1是为了取末尾的index，-1是因为list索引从0开始 28 | max_num = integer * (self._frames + 1) - 1 29 | min_num = integer * 1 - 1 30 | # data_cursors每一段数据的结束位置 31 | data_cursors = np.arange(min_num, max_num, integer) 32 | 33 | if remainder != 0: 34 | # random.sample()生成一个在一定范围内不重复的list 35 | # extra_cursors增加的数据的索引 36 | extra_cursors = sorted(random.sample(range(self._frames), remainder)) 37 | 38 | # 改变一个位置的index其以后的所有都要改变 39 | for add_index in extra_cursors: 40 | # 这里由于data_cursors是一个numpy array所以利用broadcast性质 41 | data_cursors[add_index:] = data_cursors[add_index:] + add_num 42 | assert (data_cursors[-1] == data_frames - 1) 43 | 44 | output_cursors.append(random.randint(0, data_cursors[0])) 45 | for cursor_index in range(1, self._frames): 46 | output_cursors.append(random.randint( 47 | data_cursors[cursor_index - 1] + 1, data_cursors[cursor_index])) 48 | # print(output_cursors) 49 | # 返回帧数的选择list 50 | return output_cursors 51 | 52 | def get_single_data(self, data_name, output_cursors, body_id): 53 | # body_id是取primary和secondary的时候用的，当4维数据取出期中一维的某个则变成了三维数据 54 | zero_metric = np.zeros([1, 25, 3], dtype=float) 55 | with h5py.File(self._h5file_path, 'r') as h5file: 56 | data = h5file[data_name] 57 | output_data = data[body_id, output_cursors, :, :] 58 | output_data = np.array(output_data, dtype='float32') 59 | # 计算帧差 np.diff()arr,n是计算几次差值，axis在某个维度计算差值你 60 | 61 | diff_output_data = np.diff(output_data, n=1, axis=0) 62 | # np.append()将value在维度axis=的附在arr后 63 | diff_output_data = np.array(np.append(diff_output_data, zero_metric, axis=0), dtype='float32') 64 | # 这里的label（60，1）还是（1，60）？？ 65 | label = np.array(data.attrs['label']).reshape((60,)) 66 | assert (output_data.shape == (self._frames, 25, 3)) 67 | # assert(label.shape == (60, 1)) 68 | return output_data, diff_output_data, label 69 | 70 | # 与训练集不同测试集的batch size为1所以需要扩展维度 71 | def get_tst_single_data(self, data_name, output_cursors, body_id): 72 | zero_metric = np.zeros([1, 25, 3], dtype=float) 73 | with h5py.File(self._h5file_path, 'r') as h5file: 74 | data = h5file[data_name] 75 | output_data = data[body_id, output_cursors, :, :] 76 | output_data = np.array(output_data, dtype='float32') 77 | # 计算帧差 np.diff()arr,n是计算几次差值，axis在某个维度计算差值你 78 | diff_output_data = np.diff(output_data, n=1, axis=0) 79 | # np.append()将value在维度axis=的附在arr后 80 | diff_output_data = np.array(np.append(diff_output_data, zero_metric, axis=0), dtype='float32') 81 | output_data = np.expand_dims(output_data, axis=0) 82 | diff_output_data = np.expand_dims(diff_output_data, axis=0) 83 | # 需要的是（1，60）的向量所以这里进行转置 84 | label = np.array(data.attrs['label']).T 85 | # assert(output_data.shape == (self._frames, 25, 3)) 86 | # assert(label.shape == (60, 1)) 87 | return output_data, diff_output_data, label 88 | 89 | def batch_cursors(self, data_num): 90 | # 数据需不需要打乱（个人感觉不是很需要） 91 | # 根据训练集和测试集以及batch size，生成每一个batch对应的索引编号 92 | # 这里与生成单个文件的cursors相似但不是很相同，不用考虑时序每个个体是独立的且要遍历完一遍 93 | batch_output_cursors = [] 94 | file_remainder = data_num % self._batch_size 95 | file_integer = data_num // self._batch_size 96 | file_cursors = np.arange(self._batch_size, data_num, self._batch_size) 97 | 98 | batch_output_cursors.append(list(range(0, file_cursors[0]))) 99 | for integer_index in range(1, file_integer): 100 | batch_output_cursors.append(list(range(file_cursors[integer_index - 1], file_cursors[integer_index]))) 101 | assert (len(batch_output_cursors) == file_integer) 102 | if file_remainder != 0: 103 | generate_num = self._batch_size - file_remainder 104 | generate_num_cursors = list(random.sample(range(0, data_num), generate_num)) 105 | final_cursor = list(range(file_cursors[-1], data_num)) 106 | final_cursors = final_cursor + generate_num_cursors 107 | batch_output_cursors.append(final_cursors) 108 | # 返回一个 list 含有 file_integer+1 个list 109 | return batch_output_cursors 110 | 111 | # 分两次生成不过只用一次的label 112 | def generate_batch_data(self, name_list, single_batch_cursors, body_id): 113 | batch_data = [] 114 | batch_labels = [] 115 | batch_diff_data = [] 116 | for cursor in single_batch_cursors: 117 | data_name = name_list[cursor] 118 | 119 | output_cursors = self.get_cursors(data_name=data_name) 120 | output_data, diff_output_data, label = self.get_single_data(data_name=data_name, 121 | output_cursors=output_cursors, body_id=body_id) 122 | 123 | batch_data.append(output_data) 124 | batch_diff_data.append(diff_output_data) 125 | batch_labels.append(label) 126 | 127 | batch_data = np.array(batch_data, dtype='float32').reshape((self._batch_size, self._frames, 25, 3)) 128 | batch_diff_data = np.array(batch_diff_data, dtype='float32').reshape((self._batch_size, self._frames, 25, 3)) 129 | batch_labels = np.array(batch_labels, dtype='float32') 130 | return batch_data, batch_diff_data, batch_labels 131 | 132 | 133 | 134 | if __name__ == '__main__': 135 | data = DataGenerator('F:/NTU60/data_set/cross_subject/cs_trn.hdf5',32,30) 136 | # namelist = data.get_data_name() 137 | # output=data.get_cursors(namelist[0]) 138 | # outputdata, diffoutput, label=data.get_single_data(namelist[0], output, 0) 139 | # print(outputdata) 140 | # print(diffoutput) 141 | # print(label) 142 | a = data.batch_cursors(120) 143 | print(a) -------------------------------------------------------------------------------- /function/data_generator_v2.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import h5py 3 | import random 4 | 5 | # 对数据初始话排序 6 | 7 | 8 | class DataGenerator(object): 9 | 10 | def __init__(self, h5file_path, batch_size=64, frames=30): 11 | self._h5file_path = h5file_path 12 | self._batch_size = batch_size 13 | self._frames = frames 14 | 15 | def get_data_name(self): 16 | # 获取hdf5文件中的data set名字，返回一个list 17 | with h5py.File(self._h5file_path, 'r') as h5file: 18 | return [data_name for data_name in h5file] 19 | 20 | def get_cursors(self, data_name): 21 | with h5py.File(self._h5file_path, 'r') as h5file: 22 | data = h5file[data_name] 23 | data_frames = data.shape[1] 24 | # 生成30帧随机帧 25 | output_cursors = [] 26 | add_num = 1 27 | remainder = data_frames % self._frames 28 | integer = data_frames // self._frames 29 | # 这里的+1是为了取末尾的index，-1是因为list索引从0开始 30 | max_num = integer * (self._frames + 1) - 1 31 | min_num = integer * 1 - 1 32 | # data_cursors每一段数据的结束位置 33 | data_cursors = np.arange(min_num, max_num, integer) 34 | 35 | if remainder != 0: 36 | # random.sample()生成一个在一定范围内不重复的list 37 | # extra_cursors增加的数据的索引 38 | extra_cursors = sorted(random.sample(range(self._frames), remainder)) 39 | 40 | # 改变一个位置的index其以后的所有都要改变 41 | for add_index in extra_cursors: 42 | # 这里由于data_cursors是一个numpy array所以利用broadcast性质 43 | data_cursors[add_index:] = data_cursors[add_index:] + add_num 44 | assert (data_cursors[-1] == data_frames - 1) 45 | 46 | output_cursors.append(random.randint(0, data_cursors[0])) 47 | for cursor_index in range(1, self._frames): 48 | output_cursors.append(random.randint( 49 | data_cursors[cursor_index - 1] + 1, data_cursors[cursor_index])) 50 | # print(output_cursors) 51 | # 返回帧数的选择list 52 | return output_cursors 53 | 54 | def get_single_data(self, data_name, output_cursors, body_id): 55 | # body_id是取primary和secondary的时候用的，当4维数据取出期中一维的某个则变成了三维数据 56 | zero_metric = np.zeros([1, 25, 3], dtype=float) 57 | # sequence = [24, 25, 12, 11, 10, 9, 5, 6, 7, 8, 23, 22, 20, 19, 18, 17, 13, 14, 15, 16, 4, 3, 21, 2, 1] 58 | sequence = [23, 24, 11, 10, 9, 8, 4, 5, 6, 7, 22, 21, 19, 18, 17, 16, 12, 13, 14, 15, 3, 2, 20, 1, 0] 59 | with h5py.File(self._h5file_path, 'r') as h5file: 60 | data = h5file[data_name] 61 | output_data = data[body_id, output_cursors, :, :] 62 | output_data = output_data[:, sequence, :] 63 | output_data = np.array(output_data, dtype='float32') 64 | # 计算帧差 np.diff()arr,n是计算几次差值，axis在某个维度计算差值你 65 | 66 | diff_output_data = np.diff(output_data, n=1, axis=0) 67 | # np.append()将value在维度axis=的附在arr后 68 | diff_output_data = np.array(np.append(diff_output_data, zero_metric, axis=0), dtype='float32') 69 | # 这里的label（60，1）还是（1，60）？？ 70 | label = np.array(data.attrs['label']).reshape((60,)) 71 | assert (output_data.shape == (self._frames, 25, 3)) 72 | # assert(label.shape == (60, 1)) 73 | return output_data, diff_output_data, label 74 | 75 | # 与训练集不同测试集的batch size为1所以需要扩展维度 76 | def get_tst_single_data(self, data_name, output_cursors, body_id): 77 | zero_metric = np.zeros([1, 25, 3], dtype=float) 78 | # sequence = [24, 25, 12, 11, 10, 9, 5, 6, 7, 8, 23, 22, 20, 19, 18, 17, 13, 14, 15, 16, 4, 3, 21, 2, 1] 79 | sequence = [23, 24, 11, 10, 9, 8, 4, 5, 6, 7, 22, 21, 19, 18, 17, 16, 12, 13, 14, 15, 3, 2, 20, 1, 0] 80 | with h5py.File(self._h5file_path, 'r') as h5file: 81 | data = h5file[data_name] 82 | output_data = data[body_id, output_cursors, :, :] 83 | output_data = output_data[:, sequence, :] 84 | output_data = np.array(output_data, dtype='float32') 85 | # 计算帧差 np.diff()arr,n是计算几次差值，axis在某个维度计算差值你 86 | diff_output_data = np.diff(output_data, n=1, axis=0) 87 | # np.append()将value在维度axis=的附在arr后 88 | diff_output_data = np.array(np.append(diff_output_data, zero_metric, axis=0), dtype='float32') 89 | output_data = np.expand_dims(output_data, axis=0) 90 | diff_output_data = np.expand_dims(diff_output_data, axis=0) 91 | # 需要的是（1，60）的向量所以这里进行转置 92 | label = np.array(data.attrs['label']).T 93 | # assert(output_data.shape == (self._frames, 25, 3)) 94 | # assert(label.shape == (60, 1)) 95 | return output_data, diff_output_data, label 96 | 97 | def batch_cursors(self, data_num): 98 | # 数据需不需要打乱（个人感觉不是很需要） 99 | # 根据训练集和测试集以及batch size，生成每一个batch对应的索引编号 100 | # 这里与生成单个文件的cursors相似但不是很相同，不用考虑时序每个个体是独立的且要遍历完一遍 101 | batch_output_cursors = [] 102 | file_remainder = data_num % self._batch_size 103 | file_integer = data_num // self._batch_size 104 | file_cursors = np.arange(self._batch_size, data_num, self._batch_size) 105 | 106 | batch_output_cursors.append(list(range(0, file_cursors[0]))) 107 | for integer_index in range(1, file_integer): 108 | batch_output_cursors.append(list(range(file_cursors[integer_index - 1], file_cursors[integer_index]))) 109 | assert (len(batch_output_cursors) == file_integer) 110 | if file_remainder != 0: 111 | generate_num = self._batch_size - file_remainder 112 | generate_num_cursors = list(random.sample(range(0, data_num), generate_num)) 113 | final_cursor = list(range(file_cursors[-1], data_num)) 114 | final_cursors = final_cursor + generate_num_cursors 115 | batch_output_cursors.append(final_cursors) 116 | # 返回一个 list 含有 file_integer+1 个list 117 | return batch_output_cursors 118 | 119 | # 分两次生成不过只用一次的label 120 | def generate_batch_data(self, name_list, single_batch_cursors, body_id): 121 | batch_data = [] 122 | batch_labels = [] 123 | batch_diff_data = [] 124 | for cursor in single_batch_cursors: 125 | data_name = name_list[cursor] 126 | 127 | output_cursors = self.get_cursors(data_name=data_name) 128 | output_data, diff_output_data, label = self.get_single_data(data_name=data_name, 129 | output_cursors=output_cursors, body_id=body_id) 130 | 131 | batch_data.append(output_data) 132 | batch_diff_data.append(diff_output_data) 133 | batch_labels.append(label) 134 | 135 | batch_data = np.array(batch_data, dtype='float32').reshape((self._batch_size, self._frames, 25, 3)) 136 | batch_diff_data = np.array(batch_diff_data, dtype='float32').reshape((self._batch_size, self._frames, 25, 3)) 137 | batch_labels = np.array(batch_labels, dtype='float32') 138 | return batch_data, batch_diff_data, batch_labels 139 | 140 | 141 | 142 | if __name__ == '__main__': 143 | data = DataGenerator('F:/NTU60/data_set/cross_subject/cs_trn.hdf5',32,30) 144 | # namelist = data.get_data_name() 145 | # output=data.get_cursors(namelist[0]) 146 | # outputdata, diffoutput, label=data.get_single_data(namelist[0], output, 0) 147 | # print(outputdata) 148 | # print(diffoutput) 149 | # print(label) 150 | a = data.batch_cursors(120) 151 | print(a) -------------------------------------------------------------------------------- /function/log_file.py: -------------------------------------------------------------------------------- 1 | def train_log(path, e_acc, e_loss, lr, epoch): 2 | with open(path, 'a+') as f: 3 | e_acc = str(e_acc) 4 | e_loss = str(e_loss) 5 | lr = str(lr) 6 | epoch = str(epoch + 1) 7 | f.write('Epoch:' + epoch + ' ' + 'Learning rate:' + lr + ' ' + 8 | 'Accuracy:' + e_acc + ' ' + ' Loss:' + e_loss + '\n') 9 | 10 | 11 | def eval_log(path, e_acc, epoch): 12 | with open(path, 'a+') as f: 13 | e_acc = str(e_acc) 14 | epoch = str(epoch + 1) 15 | f.write('Epoch:' + epoch + ' ' + 'Accuracy:' + e_acc + ' ' + '\n') -------------------------------------------------------------------------------- /layers/network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XiaoCode-er/Two-Stream-CNN/dce5b63d1fb8750a275816886b08124ae92809ae/layers/network.png -------------------------------------------------------------------------------- /layers/transformer.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | from keras.engine.topology import Layer 3 | import numpy as np 4 | 5 | 6 | class Transformer(Layer): 7 | 8 | def __init__(self, d_k, frames, **kwargs): 9 | # d_k是经过transformer的输出维度 10 | self.d_k = d_k 11 | # self.scalar = np.sqrt(self.d_k) 12 | self.frames = frames 13 | super().__init__(**kwargs) 14 | 15 | def build(self, input_shape): 16 | self.W = self.add_weight(name='transformer', 17 | shape=[input_shape[2], self.d_k], 18 | initializer='glorot_uniform', 19 | trainable=True) 20 | super().build(input_shape) 21 | 22 | def call(self, inputs): 23 | 24 | inputs = K.permute_dimensions(inputs, (0, 3, 1, 2)) 25 | inputs = K.reshape(inputs, (-1, self.frames*3, int(inputs.shape[3]))) 26 | inputs = K.dot(inputs, self.W) 27 | inputs = K.reshape(inputs, (-1, 3, self.frames, self.d_k)) 28 | inputs = K.permute_dimensions(inputs, (0, 2, 3, 1)) 29 | # q, k, v = inputs 30 | # A = K.batch_dot(inputs, inputs, axes=[3, 3])/self.scalar 31 | # A = K.softmax(A) 32 | # A = K.batch_dot(A, inputs, axes=[3, 3]) 33 | # A = K.permute_dimensions(A, (0, 2, 3, 1)) 34 | return inputs 35 | 36 | def compute_output_shape(self, input_shape): 37 | return (input_shape[0], input_shape[1], self.d_k, input_shape[3]) -------------------------------------------------------------------------------- /networks/basic_4_fc.py: -------------------------------------------------------------------------------- 1 | import h5py 2 | import keras 3 | import random 4 | from function.data_generator import DataGenerator 5 | from layers.attention import Attention 6 | from layers.transformer import Transformer 7 | from keras.layers import Input, Dense, Permute, Reshape 8 | from keras.models import Model 9 | from keras.layers import Conv2D, MaxPooling2D, BatchNormalization, AveragePooling2D 10 | from keras.layers import Activation, Dropout, Flatten, concatenate, Maximum 11 | from keras.optimizers import Adam 12 | from keras.regularizers import l2 13 | from keras.utils import plot_model 14 | import matplotlib.pylab as pl 15 | 16 | epochs = 200 17 | batch_size = 64 18 | frames = 30 19 | learning_rate = 0.001 20 | d_k = 30 21 | input_shape = (frames, 25, 3) 22 | use_bias = False 23 | train_path = 'F:/NTU60/data_set/cross_subject/cs_trn.hdf5' 24 | tst_path = 'F:/NTU60/data_set/cross_subject/cs_tst.hdf5' 25 | # weight_path = 'F:/NTU60/weights/basic_4_fc.h5' 26 | save_path = 'F:/NTU60/weights/basic_4_fc_bias.h5' 27 | graph_path = 'F:/NTU60/model_v1.png' 28 | 29 | 30 | # 跑通的代码 31 | # 共享层的写法Model是可以分部分来写的 32 | def share_stream(x_shape): 33 | x = Input(shape=x_shape) 34 | x_a = Transformer(d_k, frames)(x) 35 | conv1 = Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='valid', 36 | use_bias=use_bias)(x_a) 37 | # conv1 = Activation('relu')(conv1) 38 | conv1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(conv1) 39 | 40 | conv2 = Conv2D(filters=128, kernel_size=(3, 3), strides=(1, 1), padding='valid', 41 | use_bias=use_bias)(conv1) 42 | conv2 = Activation('relu')(conv2) 43 | conv2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(conv2) 44 | 45 | conv3 = Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding='valid', 46 | use_bias=use_bias)(conv2) 47 | conv3 = Activation('relu')(conv3) 48 | conv3 = Dropout(0.5)(conv3) 49 | conv3 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(conv3) 50 | 51 | shared_layer = Model(x, conv3) 52 | return shared_layer 53 | 54 | 55 | def model(): 56 | up_0 = Input(shape=input_shape, name='up_stream_0') 57 | up_1 = Input(shape=input_shape, name='up_stream_1') 58 | down_0 = Input(shape=input_shape, name='down_stream_0') 59 | down_1 = Input(shape=input_shape, name='down_stream_1') 60 | 61 | up_stream = share_stream(x_shape=input_shape) 62 | down_stream = share_stream(x_shape=input_shape) 63 | 64 | up_feature_0 = up_stream(up_0) 65 | up_feature_1 = up_stream(up_1) 66 | down_feature_0 = down_stream(down_0) 67 | down_feature_1 = down_stream(down_1) 68 | 69 | up_feature_0 = Flatten()(up_feature_0) 70 | up_feature_1 = Flatten()(up_feature_1) 71 | down_feature_0 = Flatten()(down_feature_0) 72 | down_feature_1 = Flatten()(down_feature_1) 73 | 74 | up_feature = Maximum()([up_feature_0, up_feature_1]) 75 | down_feature = Maximum()([down_feature_0, down_feature_1]) 76 | 77 | feature = concatenate([up_feature, down_feature]) 78 | 79 | fc_1 = Dense(units=256, activation='relu', use_bias=True, kernel_regularizer=l2(0.001))(feature) 80 | fc_1 = Dropout(0.5)(fc_1) 81 | 82 | fc_2 = Dense(units=128, activation='relu', use_bias=True)(fc_1) 83 | 84 | fc_3 = Dense(units=96, activation='relu', use_bias=True)(fc_2) 85 | 86 | fc_4 = Dense(units=60, activation='softmax', use_bias=True)(fc_3) 87 | 88 | network = Model(input=[up_0, up_1, down_0, down_1], outputs=fc_4) 89 | return network 90 | 91 | 92 | def train_model(network): 93 | adam = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08) 94 | network.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) 95 | network.summary() 96 | 97 | # network.load_weights(weight_path) 98 | 99 | plot_model(network, to_file=graph_path) 100 | 101 | batch_num = 0 102 | model_save_acc = 0 103 | all_train_accuracy = [] 104 | all_train_loss = [] 105 | all_tst_accuracy = [] 106 | 107 | tst_data = DataGenerator(h5file_path=tst_path, batch_size=batch_size, frames=frames) 108 | tst_data_name = tst_data.get_data_name() 109 | tst_cursors = [tst_data.get_cursors(name) for name in tst_data_name] 110 | 111 | for epoch in range(epochs): 112 | accuracy_list = [] 113 | loss_list = [] 114 | print(epoch + 1, ' epoch is beginning......') 115 | train_data = DataGenerator(h5file_path=train_path, batch_size=batch_size, frames=frames) 116 | train_data_name = train_data.get_data_name() 117 | train_data_cursors = train_data.batch_cursors(len(train_data_name)) 118 | index_num = random.sample(range(len(train_data_cursors)), len(train_data_cursors)) 119 | 120 | for ind in index_num: 121 | batch_num += 1 122 | up_data_0, down_data_0, train_labels_0 \ 123 | = train_data.generate_batch_data(train_data_name, train_data_cursors[ind], 0) 124 | up_data_1, down_data_1, train_labels_1 \ 125 | = train_data.generate_batch_data(train_data_name, train_data_cursors[ind], 1) 126 | train_loss = network.train_on_batch([up_data_0, up_data_1, down_data_0, down_data_1], train_labels_0) 127 | accuracy_list.append(train_loss[1]) 128 | loss_list.append(train_loss[0]) 129 | if batch_num % 50 == 0: 130 | print('the %r batch: loss: %r accuracy: %r' % (batch_num, train_loss[0], train_loss[1])) 131 | 132 | epoch_accuracy = sum(accuracy_list) / len(accuracy_list) 133 | epoch_loss = sum(loss_list) / len(loss_list) 134 | all_train_accuracy.append(epoch_accuracy) 135 | all_train_loss.append(epoch_loss) 136 | 137 | print('the %r epoch: mean loss: %r mean accuracy: %r' % (epoch + 1, epoch_loss, epoch_accuracy)) 138 | 139 | if epoch >= 100: 140 | tst_accuracy_list = [] 141 | for num in range(len(tst_data_name)): 142 | tst_up_0, tst_down_0, tst_labels_0 = \ 143 | tst_data.get_tst_single_data(tst_data_name[num], tst_cursors[num], 0) 144 | tst_up_1, tst_down_1, tst_labels_1 = \ 145 | tst_data.get_tst_single_data(tst_data_name[num], tst_cursors[num], 1) 146 | tst_loss = network.test_on_batch([tst_up_0, tst_up_1, tst_down_0, tst_down_1], tst_labels_0) 147 | tst_accuracy_list.append(tst_loss[1]) 148 | tst_accuracy = sum(tst_accuracy_list) / len(tst_accuracy_list) 149 | all_tst_accuracy.append(tst_accuracy) 150 | print('The test data accuracy: %r' % tst_accuracy) 151 | if tst_accuracy > model_save_acc: 152 | network.save_weights(save_path) 153 | model_save_acc = tst_accuracy 154 | 155 | pl.figure() 156 | trn_acc = pl.subplot(2, 2, 1) 157 | trn_loss = pl.subplot(2, 2, 2) 158 | tst_acc = pl.subplot(2, 1, 2) 159 | 160 | pl.sca(trn_acc) 161 | pl.plot(range(len(all_train_accuracy)), all_train_accuracy, label='train accuracy') 162 | pl.xlabel('Epoch') 163 | pl.ylabel('Accuracy') 164 | pl.ylim(0, 1.0) 165 | 166 | pl.sca(trn_loss) 167 | pl.plot(range(len(all_train_loss)), all_train_loss, label='loss') 168 | pl.xlabel('Epoch') 169 | pl.ylabel('Loss') 170 | pl.ylim(0, 5.0) 171 | 172 | pl.sca(tst_acc) 173 | pl.plot(range(len(all_tst_accuracy)), all_tst_accuracy, label='test accuracy') 174 | pl.xlabel('Epoch') 175 | pl.ylabel('Accuracy') 176 | pl.ylim(0, 1.0) 177 | 178 | pl.legend() 179 | pl.show() 180 | 181 | 182 | if __name__ == '__main__': 183 | network = model() 184 | train_model(network) 185 | -------------------------------------------------------------------------------- /networks/model_add.py: -------------------------------------------------------------------------------- 1 | import h5py 2 | import keras 3 | import random 4 | from function.data_generator import DataGenerator 5 | from layers.attentionxyz import Attention 6 | from layers.transformer import Transformer 7 | from keras.layers import Input, Dense, Permute, Reshape 8 | from keras.models import Model 9 | from keras.layers import Conv2D, MaxPooling2D, BatchNormalization, AveragePooling2D 10 | from keras.layers import Activation, Dropout, Flatten, concatenate, Maximum, Add 11 | from keras.optimizers import Adam 12 | from keras.regularizers import l2 13 | from keras.utils import plot_model 14 | import matplotlib.pylab as pl 15 | 16 | ''' 17 | 本代码将Maximum换成Add 18 | 将Maxout变为对应项相加 19 | ''' 20 | 21 | epochs = 100 22 | batch_size = 64 23 | frames = 30 24 | learning_rate = 0.001 25 | d_k = 30 26 | input_shape = (frames, 25, 3) 27 | use_bias = True 28 | train_path = 'F:/NTU60/data_set/cross_subject/cs_trn.hdf5' 29 | tst_path = 'F:/NTU60/data_set/cross_subject/cs_tst.hdf5' 30 | weight_path = 'F:/NTU60/weights/second/add.h5' 31 | graph_path = 'F:/NTU60/model_v1.png' 32 | 33 | 34 | # 跑通的代码 35 | # 共享层的写法Model是可以分部分来写的 36 | def share_stream(x_shape): 37 | x = Input(shape=x_shape) 38 | x_a = Transformer(d_k, frames)(x) 39 | conv1 = Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='valid', 40 | use_bias=use_bias)(x_a) 41 | conv1 = Activation('relu')(conv1) 42 | conv1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(conv1) 43 | 44 | conv2 = Conv2D(filters=128, kernel_size=(3, 3), strides=(1, 1), padding='valid', 45 | use_bias=use_bias)(conv1) 46 | conv2 = Activation('relu')(conv2) 47 | conv2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(conv2) 48 | 49 | conv3 = Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding='valid', 50 | use_bias=use_bias)(conv2) 51 | conv3 = Activation('relu')(conv3) 52 | conv3 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(conv3) 53 | 54 | shared_layer = Model(x, conv3) 55 | return shared_layer 56 | 57 | 58 | def model(): 59 | up_0 = Input(shape=input_shape, name='up_stream_0') 60 | up_1 = Input(shape=input_shape, name='up_stream_1') 61 | down_0 = Input(shape=input_shape, name='down_stream_0') 62 | down_1 = Input(shape=input_shape, name='down_stream_1') 63 | 64 | up_stream = share_stream(x_shape=input_shape) 65 | down_stream = share_stream(x_shape=input_shape) 66 | 67 | up_feature_0 = up_stream(up_0) 68 | up_feature_1 = up_stream(up_1) 69 | down_feature_0 = down_stream(down_0) 70 | down_feature_1 = down_stream(down_1) 71 | 72 | up_feature_0 = Flatten()(up_feature_0) 73 | up_feature_1 = Flatten()(up_feature_1) 74 | down_feature_0 = Flatten()(down_feature_0) 75 | down_feature_1 = Flatten()(down_feature_1) 76 | 77 | up_feature = Add()([up_feature_0, up_feature_1]) 78 | down_feature = Add()([down_feature_0, down_feature_1]) 79 | 80 | feature = concatenate([up_feature, down_feature]) 81 | 82 | fc_1 = Dense(units=256, activation='relu', kernel_regularizer=l2(0.001))(feature) 83 | fc_1 = Dropout(0.5)(fc_1) 84 | 85 | fc_2 = Dense(units=60, activation='softmax')(fc_1) 86 | 87 | network = Model(input=[up_0, up_1, down_0, down_1], outputs=fc_2) 88 | return network 89 | 90 | 91 | def train_model(network): 92 | adam = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08) 93 | network.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) 94 | network.summary() 95 | plot_model(network, to_file=graph_path) 96 | 97 | batch_num = 0 98 | model_save_acc = 0 99 | all_train_accuracy = [] 100 | all_train_loss = [] 101 | all_tst_accuracy = [] 102 | 103 | tst_data = DataGenerator(h5file_path=tst_path, batch_size=batch_size, frames=frames) 104 | tst_data_name = tst_data.get_data_name() 105 | tst_cursors = [tst_data.get_cursors(name) for name in tst_data_name] 106 | 107 | for epoch in range(epochs): 108 | accuracy_list = [] 109 | loss_list = [] 110 | print(epoch + 1, ' epoch is beginning......') 111 | train_data = DataGenerator(h5file_path=train_path, batch_size=batch_size, frames=frames) 112 | train_data_name = train_data.get_data_name() 113 | train_data_cursors = train_data.batch_cursors(len(train_data_name)) 114 | index_num = random.sample(range(len(train_data_cursors)), len(train_data_cursors)) 115 | 116 | for ind in index_num: 117 | batch_num += 1 118 | up_data_0, down_data_0, train_labels_0 \ 119 | = train_data.generate_batch_data(train_data_name, train_data_cursors[ind], 0) 120 | up_data_1, down_data_1, train_labels_1 \ 121 | = train_data.generate_batch_data(train_data_name, train_data_cursors[ind], 1) 122 | train_loss = network.train_on_batch([up_data_0, up_data_1, down_data_0, down_data_1], train_labels_0) 123 | accuracy_list.append(train_loss[1]) 124 | loss_list.append(train_loss[0]) 125 | if batch_num % 50 == 0: 126 | print('the %r batch: loss: %r accuracy: %r' % (batch_num, train_loss[0], train_loss[1])) 127 | 128 | epoch_accuracy = sum(accuracy_list) / len(accuracy_list) 129 | epoch_loss = sum(loss_list) / len(loss_list) 130 | all_train_accuracy.append(epoch_accuracy) 131 | all_train_loss.append(epoch_loss) 132 | 133 | print('the %r epoch: mean loss: %r mean accuracy: %r' % (epoch + 1, epoch_loss, epoch_accuracy)) 134 | 135 | if epoch >= 0: 136 | tst_accuracy_list = [] 137 | for num in range(len(tst_data_name)): 138 | tst_up_0, tst_down_0, tst_labels_0 = \ 139 | tst_data.get_tst_single_data(tst_data_name[num], tst_cursors[num], 0) 140 | tst_up_1, tst_down_1, tst_labels_1 = \ 141 | tst_data.get_tst_single_data(tst_data_name[num], tst_cursors[num], 1) 142 | tst_loss = network.test_on_batch([tst_up_0, tst_up_1, tst_down_0, tst_down_1], tst_labels_0) 143 | tst_accuracy_list.append(tst_loss[1]) 144 | tst_accuracy = sum(tst_accuracy_list) / len(tst_accuracy_list) 145 | all_tst_accuracy.append(tst_accuracy) 146 | print('The test data accuracy: %r' % tst_accuracy) 147 | if tst_accuracy > model_save_acc: 148 | network.save_weights(weight_path) 149 | model_save_acc = tst_accuracy 150 | 151 | pl.figure() 152 | trn_acc = pl.subplot(2, 2, 1) 153 | trn_loss = pl.subplot(2, 2, 2) 154 | tst_acc = pl.subplot(2, 1, 2) 155 | 156 | pl.sca(trn_acc) 157 | pl.plot(range(len(all_train_accuracy)), all_train_accuracy, label='train accuracy') 158 | pl.xlabel('Epoch') 159 | pl.ylabel('Accuracy') 160 | pl.ylim(0, 1.0) 161 | 162 | pl.sca(trn_loss) 163 | pl.plot(range(len(all_train_loss)), all_train_loss, label='loss') 164 | pl.xlabel('Epoch') 165 | pl.ylabel('Loss') 166 | pl.ylim(0, 5.0) 167 | 168 | pl.sca(tst_acc) 169 | pl.plot(range(len(all_tst_accuracy)), all_tst_accuracy, label='test accuracy') 170 | pl.xlabel('Epoch') 171 | pl.ylabel('Accuracy') 172 | pl.ylim(0, 1.0) 173 | 174 | pl.legend() 175 | pl.show() 176 | 177 | 178 | if __name__ == '__main__': 179 | network = model() 180 | train_model(network) 181 | -------------------------------------------------------------------------------- /networks/model_average.py: -------------------------------------------------------------------------------- 1 | import h5py 2 | import keras 3 | import random 4 | from function.data_generator import DataGenerator 5 | from layers.attention import Attention 6 | from layers.transformer import Transformer 7 | from keras.layers import Input, Dense, Permute, Reshape 8 | from keras.models import Model 9 | from keras.layers import Conv2D, MaxPooling2D, BatchNormalization, AveragePooling2D 10 | from keras.layers import Activation, Dropout, Flatten, concatenate, Maximum, Add, Average 11 | from keras.optimizers import Adam 12 | from keras.regularizers import l2 13 | from keras.utils import plot_model 14 | import matplotlib.pylab as pl 15 | 16 | ''' 17 | 本代码将Maximum换成Average 18 | 将Maxout变为对应项相加再平均 19 | ''' 20 | 21 | epochs = 100 22 | batch_size = 64 23 | frames = 30 24 | learning_rate = 0.001 25 | d_k = 30 26 | input_shape = (frames, 25, 3) 27 | use_bias = True 28 | train_path = 'F:/NTU60/data_set/cross_subject/cs_trn.hdf5' 29 | tst_path = 'F:/NTU60/data_set/cross_subject/cs_tst.hdf5' 30 | weight_path = 'F:/NTU60/weights/second/average.h5' 31 | graph_path = 'F:/NTU60/model_v1.png' 32 | 33 | 34 | # 跑通的代码 35 | # 共享层的写法Model是可以分部分来写的 36 | def share_stream(x_shape): 37 | x = Input(shape=x_shape) 38 | x_a = Transformer(d_k, frames)(x) 39 | conv1 = Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='valid', 40 | use_bias=use_bias)(x_a) 41 | conv1 = Activation('relu')(conv1) 42 | conv1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(conv1) 43 | 44 | conv2 = Conv2D(filters=128, kernel_size=(3, 3), strides=(1, 1), padding='valid', 45 | use_bias=use_bias)(conv1) 46 | conv2 = Activation('relu')(conv2) 47 | conv2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(conv2) 48 | 49 | conv3 = Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding='valid', 50 | use_bias=use_bias)(conv2) 51 | conv3 = Activation('relu')(conv3) 52 | conv3 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(conv3) 53 | 54 | shared_layer = Model(x, conv3) 55 | return shared_layer 56 | 57 | 58 | def model(): 59 | up_0 = Input(shape=input_shape, name='up_stream_0') 60 | up_1 = Input(shape=input_shape, name='up_stream_1') 61 | down_0 = Input(shape=input_shape, name='down_stream_0') 62 | down_1 = Input(shape=input_shape, name='down_stream_1') 63 | 64 | up_stream = share_stream(x_shape=input_shape) 65 | down_stream = share_stream(x_shape=input_shape) 66 | 67 | up_feature_0 = up_stream(up_0) 68 | up_feature_1 = up_stream(up_1) 69 | down_feature_0 = down_stream(down_0) 70 | down_feature_1 = down_stream(down_1) 71 | 72 | up_feature_0 = Flatten()(up_feature_0) 73 | up_feature_1 = Flatten()(up_feature_1) 74 | down_feature_0 = Flatten()(down_feature_0) 75 | down_feature_1 = Flatten()(down_feature_1) 76 | 77 | up_feature = Average()([up_feature_0, up_feature_1]) 78 | down_feature = Average()([down_feature_0, down_feature_1]) 79 | 80 | feature = concatenate([up_feature, down_feature]) 81 | 82 | fc_1 = Dense(units=256, activation='relu', kernel_regularizer=l2(0.001))(feature) 83 | fc_1 = Dropout(0.5)(fc_1) 84 | 85 | fc_2 = Dense(units=60, activation='softmax')(fc_1) 86 | 87 | network = Model(input=[up_0, up_1, down_0, down_1], outputs=fc_2) 88 | return network 89 | 90 | 91 | def train_model(network): 92 | adam = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08) 93 | network.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) 94 | network.summary() 95 | plot_model(network, to_file=graph_path) 96 | 97 | batch_num = 0 98 | model_save_acc = 0 99 | all_train_accuracy = [] 100 | all_train_loss = [] 101 | all_tst_accuracy = [] 102 | 103 | tst_data = DataGenerator(h5file_path=tst_path, batch_size=batch_size, frames=frames) 104 | tst_data_name = tst_data.get_data_name() 105 | tst_cursors = [tst_data.get_cursors(name) for name in tst_data_name] 106 | 107 | for epoch in range(epochs): 108 | accuracy_list = [] 109 | loss_list = [] 110 | print(epoch + 1, ' epoch is beginning......') 111 | train_data = DataGenerator(h5file_path=train_path, batch_size=batch_size, frames=frames) 112 | train_data_name = train_data.get_data_name() 113 | train_data_cursors = train_data.batch_cursors(len(train_data_name)) 114 | index_num = random.sample(range(len(train_data_cursors)), len(train_data_cursors)) 115 | 116 | for ind in index_num: 117 | batch_num += 1 118 | up_data_0, down_data_0, train_labels_0 \ 119 | = train_data.generate_batch_data(train_data_name, train_data_cursors[ind], 0) 120 | up_data_1, down_data_1, train_labels_1 \ 121 | = train_data.generate_batch_data(train_data_name, train_data_cursors[ind], 1) 122 | train_loss = network.train_on_batch([up_data_0, up_data_1, down_data_0, down_data_1], train_labels_0) 123 | accuracy_list.append(train_loss[1]) 124 | loss_list.append(train_loss[0]) 125 | if batch_num % 50 == 0: 126 | print('the %r batch: loss: %r accuracy: %r' % (batch_num, train_loss[0], train_loss[1])) 127 | 128 | epoch_accuracy = sum(accuracy_list) / len(accuracy_list) 129 | epoch_loss = sum(loss_list) / len(loss_list) 130 | all_train_accuracy.append(epoch_accuracy) 131 | all_train_loss.append(epoch_loss) 132 | 133 | print('the %r epoch: mean loss: %r mean accuracy: %r' % (epoch + 1, epoch_loss, epoch_accuracy)) 134 | 135 | if epoch >= 0: 136 | tst_accuracy_list = [] 137 | for num in range(len(tst_data_name)): 138 | tst_up_0, tst_down_0, tst_labels_0 = \ 139 | tst_data.get_tst_single_data(tst_data_name[num], tst_cursors[num], 0) 140 | tst_up_1, tst_down_1, tst_labels_1 = \ 141 | tst_data.get_tst_single_data(tst_data_name[num], tst_cursors[num], 1) 142 | tst_loss = network.test_on_batch([tst_up_0, tst_up_1, tst_down_0, tst_down_1], tst_labels_0) 143 | tst_accuracy_list.append(tst_loss[1]) 144 | tst_accuracy = sum(tst_accuracy_list) / len(tst_accuracy_list) 145 | all_tst_accuracy.append(tst_accuracy) 146 | print('The test data accuracy: %r' % tst_accuracy) 147 | if tst_accuracy > model_save_acc: 148 | network.save_weights(weight_path) 149 | model_save_acc = tst_accuracy 150 | 151 | pl.figure() 152 | trn_acc = pl.subplot(2, 2, 1) 153 | trn_loss = pl.subplot(2, 2, 2) 154 | tst_acc = pl.subplot(2, 1, 2) 155 | 156 | pl.sca(trn_acc) 157 | pl.plot(range(len(all_train_accuracy)), all_train_accuracy, label='train accuracy') 158 | pl.xlabel('Epoch') 159 | pl.ylabel('Accuracy') 160 | pl.ylim(0, 1.0) 161 | 162 | pl.sca(trn_loss) 163 | pl.plot(range(len(all_train_loss)), all_train_loss, label='loss') 164 | pl.xlabel('Epoch') 165 | pl.ylabel('Loss') 166 | pl.ylim(0, 5.0) 167 | 168 | pl.sca(tst_acc) 169 | pl.plot(range(len(all_tst_accuracy)), all_tst_accuracy, label='test accuracy') 170 | pl.xlabel('Epoch') 171 | pl.ylabel('Accuracy') 172 | pl.ylim(0, 1.0) 173 | 174 | pl.legend() 175 | pl.show() 176 | 177 | 178 | if __name__ == '__main__': 179 | network = model() 180 | train_model(network) 181 | -------------------------------------------------------------------------------- /networks/model_multiply.py: -------------------------------------------------------------------------------- 1 | import h5py 2 | import keras 3 | import random 4 | from function.data_generator import DataGenerator 5 | from layers.attention import Attention 6 | from layers.transformer import Transformer 7 | from keras.layers import Input, Dense, Permute, Reshape 8 | from keras.models import Model 9 | from keras.layers import Conv2D, MaxPooling2D, BatchNormalization, AveragePooling2D 10 | from keras.layers import Activation, Dropout, Flatten, concatenate, Maximum, Add, Average, Multiply 11 | from keras.optimizers import Adam 12 | from keras.regularizers import l2 13 | from keras.utils import plot_model 14 | import matplotlib.pylab as pl 15 | 16 | ''' 17 | 本代码将Maximum换成Average 18 | 将Maxout变为对应项相加再平均 19 | ''' 20 | 21 | epochs = 100 22 | batch_size = 64 23 | frames = 30 24 | learning_rate = 0.001 25 | d_k = 30 26 | input_shape = (frames, 25, 3) 27 | use_bias = True 28 | train_path = 'F:/NTU60/data_set/cross_subject/cs_trn.hdf5' 29 | tst_path = 'F:/NTU60/data_set/cross_subject/cs_tst.hdf5' 30 | weight_path = 'F:/NTU60/weights/second/multiply.h5' 31 | graph_path = 'F:/NTU60/model_v1.png' 32 | 33 | 34 | # 跑通的代码 35 | # 共享层的写法Model是可以分部分来写的 36 | def share_stream(x_shape): 37 | x = Input(shape=x_shape) 38 | x_a = Transformer(d_k, frames)(x) 39 | conv1 = Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='valid', 40 | use_bias=use_bias)(x_a) 41 | conv1 = Activation('relu')(conv1) 42 | conv1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(conv1) 43 | 44 | conv2 = Conv2D(filters=128, kernel_size=(3, 3), strides=(1, 1), padding='valid', 45 | use_bias=use_bias)(conv1) 46 | conv2 = Activation('relu')(conv2) 47 | conv2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(conv2) 48 | 49 | conv3 = Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding='valid', 50 | use_bias=use_bias)(conv2) 51 | conv3 = Activation('relu')(conv3) 52 | conv3 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(conv3) 53 | 54 | shared_layer = Model(x, conv3) 55 | return shared_layer 56 | 57 | 58 | def model(): 59 | up_0 = Input(shape=input_shape, name='up_stream_0') 60 | up_1 = Input(shape=input_shape, name='up_stream_1') 61 | down_0 = Input(shape=input_shape, name='down_stream_0') 62 | down_1 = Input(shape=input_shape, name='down_stream_1') 63 | 64 | up_stream = share_stream(x_shape=input_shape) 65 | down_stream = share_stream(x_shape=input_shape) 66 | 67 | up_feature_0 = up_stream(up_0) 68 | up_feature_1 = up_stream(up_1) 69 | down_feature_0 = down_stream(down_0) 70 | down_feature_1 = down_stream(down_1) 71 | 72 | up_feature_0 = Flatten()(up_feature_0) 73 | up_feature_1 = Flatten()(up_feature_1) 74 | down_feature_0 = Flatten()(down_feature_0) 75 | down_feature_1 = Flatten()(down_feature_1) 76 | 77 | up_feature = Multiply()([up_feature_0, up_feature_1]) 78 | down_feature = Multiply()([down_feature_0, down_feature_1]) 79 | 80 | feature = concatenate([up_feature, down_feature]) 81 | 82 | fc_1 = Dense(units=256, activation='relu', kernel_regularizer=l2(0.001))(feature) 83 | fc_1 = Dropout(0.5)(fc_1) 84 | 85 | fc_2 = Dense(units=60, activation='softmax')(fc_1) 86 | 87 | network = Model(input=[up_0, up_1, down_0, down_1], outputs=fc_2) 88 | return network 89 | 90 | 91 | def train_model(network): 92 | adam = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08) 93 | network.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) 94 | network.summary() 95 | plot_model(network, to_file=graph_path) 96 | 97 | batch_num = 0 98 | model_save_acc = 0 99 | all_train_accuracy = [] 100 | all_train_loss = [] 101 | all_tst_accuracy = [] 102 | 103 | tst_data = DataGenerator(h5file_path=tst_path, batch_size=batch_size, frames=frames) 104 | tst_data_name = tst_data.get_data_name() 105 | tst_cursors = [tst_data.get_cursors(name) for name in tst_data_name] 106 | 107 | for epoch in range(epochs): 108 | accuracy_list = [] 109 | loss_list = [] 110 | print(epoch + 1, ' epoch is beginning......') 111 | train_data = DataGenerator(h5file_path=train_path, batch_size=batch_size, frames=frames) 112 | train_data_name = train_data.get_data_name() 113 | train_data_cursors = train_data.batch_cursors(len(train_data_name)) 114 | index_num = random.sample(range(len(train_data_cursors)), len(train_data_cursors)) 115 | 116 | for ind in index_num: 117 | batch_num += 1 118 | up_data_0, down_data_0, train_labels_0 \ 119 | = train_data.generate_batch_data(train_data_name, train_data_cursors[ind], 0) 120 | up_data_1, down_data_1, train_labels_1 \ 121 | = train_data.generate_batch_data(train_data_name, train_data_cursors[ind], 1) 122 | train_loss = network.train_on_batch([up_data_0, up_data_1, down_data_0, down_data_1], train_labels_0) 123 | accuracy_list.append(train_loss[1]) 124 | loss_list.append(train_loss[0]) 125 | if batch_num % 50 == 0: 126 | print('the %r batch: loss: %r accuracy: %r' % (batch_num, train_loss[0], train_loss[1])) 127 | 128 | epoch_accuracy = sum(accuracy_list) / len(accuracy_list) 129 | epoch_loss = sum(loss_list) / len(loss_list) 130 | all_train_accuracy.append(epoch_accuracy) 131 | all_train_loss.append(epoch_loss) 132 | 133 | print('the %r epoch: mean loss: %r mean accuracy: %r' % (epoch + 1, epoch_loss, epoch_accuracy)) 134 | 135 | if epoch >= 0: 136 | tst_accuracy_list = [] 137 | for num in range(len(tst_data_name)): 138 | tst_up_0, tst_down_0, tst_labels_0 = \ 139 | tst_data.get_tst_single_data(tst_data_name[num], tst_cursors[num], 0) 140 | tst_up_1, tst_down_1, tst_labels_1 = \ 141 | tst_data.get_tst_single_data(tst_data_name[num], tst_cursors[num], 1) 142 | tst_loss = network.test_on_batch([tst_up_0, tst_up_1, tst_down_0, tst_down_1], tst_labels_0) 143 | tst_accuracy_list.append(tst_loss[1]) 144 | tst_accuracy = sum(tst_accuracy_list) / len(tst_accuracy_list) 145 | all_tst_accuracy.append(tst_accuracy) 146 | print('The test data accuracy: %r' % tst_accuracy) 147 | if tst_accuracy > model_save_acc: 148 | network.save_weights(weight_path) 149 | model_save_acc = tst_accuracy 150 | 151 | pl.figure() 152 | trn_acc = pl.subplot(2, 2, 1) 153 | trn_loss = pl.subplot(2, 2, 2) 154 | tst_acc = pl.subplot(2, 1, 2) 155 | 156 | pl.sca(trn_acc) 157 | pl.plot(range(len(all_train_accuracy)), all_train_accuracy, label='train accuracy') 158 | pl.xlabel('Epoch') 159 | pl.ylabel('Accuracy') 160 | pl.ylim(0, 1.0) 161 | 162 | pl.sca(trn_loss) 163 | pl.plot(range(len(all_train_loss)), all_train_loss, label='loss') 164 | pl.xlabel('Epoch') 165 | pl.ylabel('Loss') 166 | pl.ylim(0, 5.0) 167 | 168 | pl.sca(tst_acc) 169 | pl.plot(range(len(all_tst_accuracy)), all_tst_accuracy, label='test accuracy') 170 | pl.xlabel('Epoch') 171 | pl.ylabel('Accuracy') 172 | pl.ylim(0, 1.0) 173 | 174 | pl.legend() 175 | pl.show() 176 | 177 | 178 | if __name__ == '__main__': 179 | network = model() 180 | train_model(network) 181 | --------------------------------------------------------------------------------