├── app └── PLSTM_xsub.ipynb ├── datas └── sample_mat_file.mat ├── imgs ├── lstm.png └── plstm.png ├── network ├── DataLoader.py ├── PLSTM.py └── PartAwareLSTMCell.py ├── ntu_rgbd_parser ├── .gitignore ├── ntu_rgb120_missings.txt ├── readme.md └── txt2npy.py ├── readme.md └── utils └── gendata.py /datas/sample_mat_file.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FesianXu/PLSTM/2f40b9dbe315e0d8ed967eba5649f3fc25a57cab/datas/sample_mat_file.mat -------------------------------------------------------------------------------- /imgs/lstm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FesianXu/PLSTM/2f40b9dbe315e0d8ed967eba5649f3fc25a57cab/imgs/lstm.png -------------------------------------------------------------------------------- /imgs/plstm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FesianXu/PLSTM/2f40b9dbe315e0d8ed967eba5649f3fc25a57cab/imgs/plstm.png -------------------------------------------------------------------------------- /network/DataLoader.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | __author__ = 'FesianXu' 5 | __date__ = '2018/1/28' 6 | __version__ = '' 7 | 8 | import tensorflow as tf 9 | import os 10 | import numpy as np 11 | import random 12 | from keras.utils import to_categorical 13 | 14 | g_feat_dim = 75 15 | g_num_actions = 40 16 | 17 | class DataLoader(object): 18 | __skel_train_dataset = None 19 | __skel_test_dataset = None 20 | __train_ids = None 21 | __valid_ids = None 22 | __mode = None 23 | 24 | def __init__(self, skel_train, skel_test, mode, lstm_tim_step=8): 25 | self.__mode = mode 26 | # divide the origin training set into train part and validation part 27 | 28 | if mode == 'train': 29 | self.__skel_train_dataset = skel_train 30 | elif mode == 'test': 31 | self.__skel_test_dataset = skel_test 32 | elif mode == 'both': 33 | self.__skel_train_dataset = skel_train 34 | self.__skel_test_dataset = skel_test 35 | else: 36 | self.__skel_test_dataset = None 37 | self.__skel_train_dataset = None 38 | 39 | self.lstm_time_step = lstm_tim_step 40 | 41 | if self.__skel_train_dataset is not None: 42 | self.train_index = list(range(self.__skel_train_dataset.shape[0])) 43 | random.shuffle(self.train_index) 44 | self.__skel_train_dataset = self.__skel_train_dataset[self.train_index] 45 | 46 | def _y_transmat(self, thetas): 47 | tms = np.zeros((0, 3, 3)) 48 | thetas = thetas*np.pi/180 49 | for theta in thetas: 50 | tm = np.zeros((3, 3)) 51 | tm[0, 0] = np.cos(theta) 52 | tm[0, 2] = -np.sin(theta) 53 | tm[1, 1] = 1 54 | tm[2, 0] = np.sin(theta) 55 | tm[2, 2] = np.cos(theta) 56 | tm = tm[np.newaxis, :, :] 57 | tms = np.concatenate((tms, tm), axis=0) 58 | return tms 59 | 60 | def _pararell_skeleton(self, raw_mat): 61 | ''' 62 | raw_mat with the shape of (nframes, 25*3) 63 | ''' 64 | joints_list = [] 65 | 66 | for each_joints in range(25): 67 | joints_list.append(raw_mat[:, each_joints*3:each_joints*3+3]) 68 | 69 | right_shoulder = joints_list[8] # 9th joint 70 | left_shoulder = joints_list[4] # 5tf joint 71 | vec = right_shoulder-left_shoulder 72 | vec[:, 1] = 0 73 | l2_norm = np.sqrt(np.sum(np.square(vec), axis=1)) 74 | theta = vec[:, 0]/(l2_norm+0.0001) 75 | # print(l2_norm) 76 | thetas = np.arccos(theta)*(180/np.pi) 77 | isv = np.sum(vec[:, 2]) 78 | if isv >= 0: 79 | thetas = -thetas 80 | y_tms = self._y_transmat(thetas) 81 | 82 | new_skel = np.zeros(shape=(0, 25*3)) 83 | for ind, each_s in enumerate(raw_mat): 84 | r = np.reshape(each_s, newshape=(25, 3)) 85 | r = np.transpose(r) 86 | r = np.dot(y_tms[ind], r) 87 | r_t = np.transpose(r) 88 | r_t = np.reshape(r_t, newshape=(1, -1)) 89 | new_skel = np.concatenate((new_skel, r_t), axis=0) 90 | return new_skel 91 | 92 | 93 | def __T_clips(self, mat, T_size): 94 | samples_num = mat.shape[0] 95 | each_clip_size = int(samples_num/T_size) 96 | index_list = [] 97 | begin = 0 98 | end = 0 99 | for each in range(T_size): 100 | end += each_clip_size 101 | index_list.append((begin, end)) 102 | begin = end 103 | random_list = [] 104 | for each_index in index_list: 105 | random_id = random.sample(list(range(each_index[0], each_index[1])), 1)[0] 106 | random_list.append(random_id) 107 | sample_mat = mat[random_list] 108 | return sample_mat 109 | 110 | def get_samples(self, batch_size, anchor, mode='train', is_rotate=True): 111 | 112 | mat_batch = np.zeros(shape=(batch_size, self.lstm_time_step, g_feat_dim)) 113 | label_batch = np.zeros(shape=(batch_size, g_num_actions)) 114 | view_batch = np.zeros(shape=(batch_size, 1)) 115 | for each in range(batch_size): 116 | data = self.__skel_train_dataset[anchor+each] 117 | mat = data['mat'] 118 | label = data['action'] 119 | view = data['view'] 120 | 121 | mat = self.__T_clips(mat, T_size=self.lstm_time_step) 122 | mat = np.reshape(mat, newshape=(-1, g_feat_dim)) 123 | if is_rotate: 124 | mat = self._pararell_skeleton(mat) 125 | label = to_categorical(label, num_classes=g_num_actions) 126 | 127 | mat_batch[each] = mat 128 | label_batch[each] = label 129 | view_batch[each] = view 130 | return mat_batch, label_batch, view_batch 131 | 132 | def get_test_sample(self, batch_size, ind, is_rotate): 133 | mat_batch = np.zeros(shape=(batch_size, self.lstm_time_step, g_feat_dim)) 134 | label_batch = np.zeros(shape=(batch_size, 1)) 135 | view_batch = np.zeros(shape=(batch_size, 1)) 136 | for each in range(batch_size): 137 | data = self.__skel_test_dataset[ind+each] 138 | mat = data['mat'] 139 | label = data['action'] 140 | view = data['clip_ind'] 141 | mat = self.__T_clips(mat, T_size=self.lstm_time_step) 142 | mat = np.reshape(mat, newshape=(-1, g_feat_dim)) 143 | if is_rotate: 144 | mat = self._pararell_skeleton(mat) 145 | mat_batch[each] = mat 146 | label_batch[each] = label 147 | view_batch[each] = view 148 | return mat_batch, label_batch, view_batch 149 | 150 | 151 | 152 | 153 | 154 | if __name__ == '__main__': 155 | pass 156 | 157 | 158 | -------------------------------------------------------------------------------- /network/PLSTM.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | __author__ = 'FesianXu' 5 | __date__ = 2018 / 3 / 19 6 | __version__ = '' 7 | 8 | import PartAwareLSTMCell as PartAwareLSTMCell 9 | import tensorflow as tf 10 | import os 11 | import pandas as pd 12 | import numpy as np 13 | 14 | os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152 15 | os.environ["CUDA_VISIBLE_DEVICES"] = "0" 16 | config = tf.ConfigProto() 17 | config.gpu_options.allow_growth = True 18 | 19 | g_feat_dim = 25*3 20 | g_T_size = 8 21 | g_num_action = 40 22 | 23 | 24 | class PLSTM(object): 25 | _linear_layer_counter = 0 26 | _reg_scope = 'reg_col' 27 | _reg_ratio = 0.01 28 | 29 | def __init__(self, 30 | batch_size=32, 31 | lr=0.001, 32 | mode='train', 33 | n_layers=3, 34 | time_step=g_T_size): 35 | self._batch_size = batch_size 36 | self._lr = lr 37 | self._time_step = time_step 38 | self._n_layers = n_layers 39 | self._mode = mode 40 | 41 | with tf.variable_scope('net'): 42 | self.skel_input = tf.placeholder(dtype=tf.float32, shape=(batch_size, g_T_size, g_feat_dim), name='skel_input') 43 | self.action_gt = tf.placeholder(dtype=tf.float32, shape=(batch_size, g_num_action), name='skel_action_gt') 44 | self.plstm_keep_prob = tf.placeholder(dtype=tf.float32) 45 | 46 | self._build_graph() 47 | 48 | 49 | action_reg_loss_s = tf.summary.scalar(name='action_reg_loss', tensor=self.action_reg_loss) 50 | action_raw_loss_s = tf.summary.scalar(name='action_raw_loss', tensor=self.action_raw_loss) 51 | action_pred_acc_s = tf.summary.scalar(name='action_pred_acc', tensor=self.action_pred_acc) 52 | 53 | loss_summary_list = [action_reg_loss_s, action_raw_loss_s, action_pred_acc_s] 54 | self.loss_merged_op = tf.summary.merge(loss_summary_list) 55 | # tran summary 56 | 57 | pred_summary_list = [action_pred_acc_s] 58 | self.pred_mergerd_op = tf.summary.merge(pred_summary_list) 59 | 60 | 61 | def _build_graph(self): 62 | 63 | def _get_PLSTM_cells(hidden_layer): 64 | return PartAwareLSTMCell.PartAwareLSTMCell(num_units=hidden_layer) 65 | 66 | cells_list = [] 67 | hidden_shape = int(self.skel_input.get_shape()[2]) 68 | for num in range(self._n_layers): 69 | cells_list.append(_get_PLSTM_cells(hidden_layer=hidden_shape)) 70 | for ind_lstm, each_lstm in enumerate(cells_list): 71 | cells_list[ind_lstm] = tf.contrib.rnn.DropoutWrapper(each_lstm, output_keep_prob=self.plstm_keep_prob) 72 | cells = tf.contrib.rnn.MultiRNNCell(cells_list) 73 | cells_state = cells.zero_state(self._batch_size, tf.float32) 74 | encoder_outputs, fin_state = tf.nn.dynamic_rnn(cells, self.skel_input, initial_state=cells_state) 75 | output_final = fin_state[-1].h 76 | 77 | with tf.variable_scope('fcs'): 78 | output_final = self._linear_connect(output_final, output_s=75, activation=tf.nn.relu) 79 | output_logit = self._linear_connect(output_final, output_s=g_num_action, activation=None) 80 | 81 | self.action_cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=output_logit, 82 | labels=self.action_gt) 83 | self.action_raw_loss = tf.reduce_mean(self.action_cross_entropy, name='action_raw_loss') 84 | tf.add_to_collection(self._reg_scope, self.action_raw_loss) 85 | self.action_reg_loss = tf.add_n(tf.get_collection(self._reg_scope), name='action_reg_loss') 86 | 87 | adam_optimizer = tf.train.AdamOptimizer(learning_rate=self._lr) 88 | train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) 89 | 90 | raw_grad = tf.gradients(self.action_reg_loss, train_vars) 91 | clip_grad, _ = tf.clip_by_global_norm(raw_grad, 10) 92 | self.adam_train_op = adam_optimizer.apply_gradients(zip(clip_grad, train_vars)) 93 | 94 | 95 | self.action_distribution = tf.nn.softmax(logits=output_logit) 96 | self.action_argmax_target = tf.argmax(self.action_distribution, axis=1) 97 | self.action_predict_label = tf.one_hot(self.action_argmax_target, on_value=1.0, off_value=0.0, depth=g_num_action, name='predict_label') 98 | self.action_pred_acc = tf.div(x=tf.reduce_sum(self.action_predict_label*self.action_gt), 99 | y=self._batch_size) 100 | 101 | 102 | def _linear_connect(self, input_v, output_s, activation, is_reg=True): 103 | self._linear_layer_counter += 1 104 | weight_name = 'fc_w_%d' % self._linear_layer_counter 105 | bias_name = 'fc_b_%d' % self._linear_layer_counter 106 | input_shape = int(input_v.shape[1]) 107 | 108 | weight = tf.get_variable(weight_name, shape=(input_shape, output_s), dtype=tf.float32) 109 | bias = tf.get_variable(bias_name, shape=(output_s), dtype=tf.float32, initializer=tf.zeros_initializer()) 110 | 111 | # regularization collection 112 | if is_reg: 113 | tf.add_to_collection(self._reg_scope, tf.contrib.layers.l2_regularizer(self._reg_ratio)(weight)) 114 | 115 | logits = tf.matmul(input_v, weight)+bias 116 | if activation is None: 117 | return logits 118 | else: 119 | return activation(logits) 120 | 121 | def generate_report(self, data, path): 122 | data_df = pd.DataFrame(data) 123 | writer = pd.ExcelWriter(path) 124 | data_df.to_excel(writer,'page_1',float_format='%.5f') # float_format 控制精度 125 | data_df.to_excel(writer,'page_2',float_format='%.5f') # float_format 控制精度 126 | writer.save() 127 | 128 | if __name__ == '__main__': 129 | model = PLSTM() -------------------------------------------------------------------------------- /network/PartAwareLSTMCell.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | __author__ = 'FesianXu' 5 | __date__ = 2018 / 3 / 20 6 | __version__ = '' 7 | 8 | from tensorflow.python.ops.rnn_cell_impl import RNNCell, LSTMStateTuple 9 | from tensorflow.python.framework import ops 10 | from tensorflow.python.ops import array_ops, math_ops 11 | from tensorflow.python.platform import tf_logging as logging 12 | from tensorflow.python.framework import dtypes 13 | 14 | class PartAwareLSTMCell(RNNCell): 15 | ''' 16 | The implement of paper 17 | Part-Aware LSTM Cell 18 | ''' 19 | def __init__(self,num_units,forget_bias=1.0,state_is_tuple=True,activation=None,reuse=None): 20 | # here num_units has to be times of 5 21 | assert num_units % 5 == 0 22 | 23 | super(PartAwareLSTMCell, self).__init__(_reuse=reuse) 24 | if not state_is_tuple: 25 | logging.warn("%s: Using a concatenated state is slower and will soon be " 26 | "deprecated. Use state_is_tuple=True.", self) 27 | self._num_units = int(num_units/5) 28 | self._forget_bias = forget_bias 29 | self._state_is_tuple = state_is_tuple 30 | self._activation = activation or math_ops.tanh 31 | 32 | @property 33 | def state_size(self): 34 | cs_size = self._num_units * 5 35 | return (LSTMStateTuple(cs_size, 5*self._num_units) 36 | if self._state_is_tuple else 2 * self._num_units) 37 | 38 | @property 39 | def output_size(self): 40 | return self._num_units * 5 41 | 42 | 43 | def call(self, skel_inputs, state): 44 | ''' 45 | here inputs with the shape of (batch_size, feat_dim) 46 | in kinect 2.0, feat_dim is 25*3 = 75 47 | for five part of a skeleton body. 48 | (head, r_arm, l_arm, r_leg, l_leg) 49 | divide config: 50 | head: [ 3, 4, 1,2,21] 51 | r_arm: [ 5, 6, 7, 8,22,23, 1,2,21] 52 | l_arm: [ 9,10,11,12,24,25, 1,2,21] 53 | r_leg: [13,14,15,16, 1,2,21] 54 | l_leg: [17,18,19,20, 1,2,21] 55 | 56 | state: LSTMStateTuple with the format of (Tensor(c1, c2, ..., c5), Tensor(h)) 57 | ''' 58 | 59 | sigmoid = math_ops.sigmoid 60 | tanh = math_ops.tanh 61 | 62 | if self._state_is_tuple: 63 | cs, h = state 64 | else: 65 | cs, h = array_ops.split(value=state, num_or_size_splits=2, axis=1) 66 | # split he state into c and h 67 | # here cs mean c1 to c5, where each part means a part of body, cs is also a list or turple 68 | 69 | # split the cs into 5 parts 70 | cs = array_ops.split(cs, num_or_size_splits=5, axis=1) 71 | 72 | divide_config = { 73 | 'head': ( 3, 4, 1,2,21), 74 | 'r_arm': ( 5, 6, 7, 8,22,23, 1,2,21), 75 | 'l_arm': ( 9,10,11,12,24,25, 1,2,21), 76 | 'r_leg': ( 13,14,15,16, 1,2,21), 77 | 'l_leg': ( 17,18,19,20, 1,2,21) 78 | } 79 | # assert skel_inputs.shape[1] == 75 80 | 81 | reshaped_input = array_ops.reshape(skel_inputs, shape=(-1, 25, 3)) 82 | head_joints = [reshaped_input[:, each-1, :] for each in divide_config['head']] 83 | r_arm_joints = [reshaped_input[:, each-1, :] for each in divide_config['r_arm']] 84 | l_arm_joints = [reshaped_input[:, each-1, :] for each in divide_config['l_arm']] 85 | r_leg_joints = [reshaped_input[:, each-1, :] for each in divide_config['r_leg']] 86 | l_leg_joints = [reshaped_input[:, each-1, :] for each in divide_config['l_leg']] 87 | 88 | body_list = [head_joints, r_arm_joints, l_arm_joints, r_leg_joints, l_leg_joints] 89 | 90 | body_list = ops.convert_n_to_tensor(body_list) 91 | 92 | for ind, each in enumerate(body_list): 93 | tmp = array_ops.transpose(each, perm=(1,0,2)) 94 | batch_size = int(tmp.shape[0]) 95 | body_list[ind] = array_ops.reshape(tmp, shape=(batch_size, -1)) 96 | 97 | o_all_skel = _linear([body_list[0], 98 | body_list[1], 99 | body_list[2], 100 | body_list[3], 101 | body_list[4], 102 | h], # here 111 + h_size 103 | 5 * self._num_units, True) 104 | o_all_skel = sigmoid(o_all_skel) 105 | new_c_list = [] 106 | for ind, each_part in enumerate(body_list): 107 | concat_p = _linear([each_part, h], 108 | 3 * self._num_units, 109 | weight_name='weight_%d' % ind, 110 | bias_name='bias_%d' % ind, 111 | bias=True) 112 | ip, fp, gp = array_ops.split(value=concat_p, num_or_size_splits=3, axis=1) 113 | ip, fp, gp = sigmoid(ip), sigmoid(fp), tanh(gp) 114 | new_c = cs[ind] * (fp+self._forget_bias) + ip * gp 115 | new_c_list.append(new_c) 116 | 117 | 118 | new_c_tensors = array_ops.concat(new_c_list, axis=1) 119 | new_h = o_all_skel * tanh(array_ops.concat(new_c_list, 1)) 120 | 121 | if self._state_is_tuple: 122 | new_state = LSTMStateTuple(new_c_tensors, new_h) 123 | else: 124 | new_state = array_ops.concat([new_c_tensors, new_h], 1) 125 | 126 | return new_h, new_state 127 | 128 | 129 | from tensorflow.python.util import nest 130 | from tensorflow.python.ops import variable_scope as vs 131 | from tensorflow.python.ops import nn_ops 132 | from tensorflow.python.ops import init_ops 133 | 134 | _BIAS_VARIABLE_NAME = "bias" 135 | _WEIGHTS_VARIABLE_NAME = "kernel" 136 | 137 | 138 | def _linear(args, 139 | output_size, 140 | bias, 141 | weight_name=_WEIGHTS_VARIABLE_NAME, 142 | bias_name=_BIAS_VARIABLE_NAME, 143 | bias_initializer=None, 144 | kernel_initializer=None): 145 | """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable. 146 | 147 | Args: 148 | args: a 2D Tensor or a list of 2D, batch x n, Tensors. 149 | output_size: int, second dimension of W[i]. 150 | bias: boolean, whether to add a bias term or not. 151 | bias_initializer: starting value to initialize the bias 152 | (default is all zeros). 153 | kernel_initializer: starting value to initialize the weight. 154 | 155 | Returns: 156 | A 2D Tensor with shape [batch x output_size] equal to 157 | sum_i(args[i] * W[i]), where W[i]s are newly created matrices. 158 | 159 | Raises: 160 | ValueError: if some of the arguments has unspecified or wrong shape. 161 | """ 162 | if args is None or (nest.is_sequence(args) and not args): 163 | raise ValueError("`args` must be specified") 164 | if not nest.is_sequence(args): 165 | args = [args] 166 | 167 | # Calculate the total size of arguments on dimension 1. 168 | total_arg_size = 0 169 | shapes = [a.get_shape() for a in args] 170 | for shape in shapes: 171 | if shape.ndims != 2: 172 | raise ValueError("linear is expecting 2D arguments: %s" % shapes) 173 | if shape[1].value is None: 174 | raise ValueError("linear expects shape[1] to be provided for shape %s, " 175 | "but saw %s" % (shape, shape[1])) 176 | else: 177 | total_arg_size += shape[1].value 178 | 179 | dtype = [a.dtype for a in args][0] 180 | 181 | # Now the computation. 182 | scope = vs.get_variable_scope() 183 | with vs.variable_scope(scope) as outer_scope: 184 | weights = vs.get_variable( 185 | weight_name, [total_arg_size, output_size], 186 | dtype=dtype, 187 | initializer=kernel_initializer) 188 | 189 | # if the args is a single tensor then matmul it with weight 190 | # if the args is a list of tensors then concat them in axis of 1 and matmul 191 | if len(args) == 1: 192 | res = math_ops.matmul(args[0], weights) 193 | else: 194 | res = math_ops.matmul(array_ops.concat(args, 1), weights) 195 | if not bias: 196 | return res 197 | with vs.variable_scope(outer_scope) as inner_scope: 198 | inner_scope.set_partitioner(None) 199 | if bias_initializer is None: 200 | bias_initializer = init_ops.constant_initializer(0.0, dtype=dtype) 201 | biases = vs.get_variable( 202 | bias_name, [output_size], 203 | dtype=dtype, 204 | initializer=bias_initializer) 205 | return nn_ops.bias_add(res, biases) -------------------------------------------------------------------------------- /ntu_rgbd_parser/.gitignore: -------------------------------------------------------------------------------- 1 | ./raw_npy/ 2 | -------------------------------------------------------------------------------- /ntu_rgbd_parser/ntu_rgb120_missings.txt: -------------------------------------------------------------------------------- 1 | S001C002P005R002A008 2 | S001C002P006R001A008 3 | S001C003P002R001A055 4 | S001C003P002R002A012 5 | S001C003P005R002A004 6 | S001C003P005R002A005 7 | S001C003P005R002A006 8 | S001C003P006R002A008 9 | S002C002P011R002A030 10 | S002C003P008R001A020 11 | S002C003P010R002A010 12 | S002C003P011R002A007 13 | S002C003P011R002A011 14 | S002C003P014R002A007 15 | S003C001P019R001A055 16 | S003C002P002R002A055 17 | S003C002P018R002A055 18 | S003C003P002R001A055 19 | S003C003P016R001A055 20 | S003C003P018R002A024 21 | S004C002P003R001A013 22 | S004C002P008R001A009 23 | S004C002P020R001A003 24 | S004C002P020R001A004 25 | S004C002P020R001A012 26 | S004C002P020R001A020 27 | S004C002P020R001A021 28 | S004C002P020R001A036 29 | S005C002P004R001A001 30 | S005C002P004R001A003 31 | S005C002P010R001A016 32 | S005C002P010R001A017 33 | S005C002P010R001A048 34 | S005C002P010R001A049 35 | S005C002P016R001A009 36 | S005C002P016R001A010 37 | S005C002P018R001A003 38 | S005C002P018R001A028 39 | S005C002P018R001A029 40 | S005C003P016R002A009 41 | S005C003P018R002A013 42 | S005C003P021R002A057 43 | S006C001P001R002A055 44 | S006C002P007R001A005 45 | S006C002P007R001A006 46 | S006C002P016R001A043 47 | S006C002P016R001A051 48 | S006C002P016R001A052 49 | S006C002P022R001A012 50 | S006C002P023R001A020 51 | S006C002P023R001A021 52 | S006C002P023R001A022 53 | S006C002P023R001A023 54 | S006C002P024R001A018 55 | S006C002P024R001A019 56 | S006C003P001R002A013 57 | S006C003P007R002A009 58 | S006C003P007R002A010 59 | S006C003P007R002A025 60 | S006C003P016R001A060 61 | S006C003P017R001A055 62 | S006C003P017R002A013 63 | S006C003P017R002A014 64 | S006C003P017R002A015 65 | S006C003P022R002A013 66 | S007C001P018R002A050 67 | S007C001P025R002A051 68 | S007C001P028R001A050 69 | S007C001P028R001A051 70 | S007C001P028R001A052 71 | S007C002P008R002A008 72 | S007C002P015R002A055 73 | S007C002P026R001A008 74 | S007C002P026R001A009 75 | S007C002P026R001A010 76 | S007C002P026R001A011 77 | S007C002P026R001A012 78 | S007C002P026R001A050 79 | S007C002P027R001A011 80 | S007C002P027R001A013 81 | S007C002P028R002A055 82 | S007C003P007R001A002 83 | S007C003P007R001A004 84 | S007C003P019R001A060 85 | S007C003P027R002A001 86 | S007C003P027R002A002 87 | S007C003P027R002A003 88 | S007C003P027R002A004 89 | S007C003P027R002A005 90 | S007C003P027R002A006 91 | S007C003P027R002A007 92 | S007C003P027R002A008 93 | S007C003P027R002A009 94 | S007C003P027R002A010 95 | S007C003P027R002A011 96 | S007C003P027R002A012 97 | S007C003P027R002A013 98 | S008C002P001R001A009 99 | S008C002P001R001A010 100 | S008C002P001R001A014 101 | S008C002P001R001A015 102 | S008C002P001R001A016 103 | S008C002P001R001A018 104 | S008C002P001R001A019 105 | S008C002P008R002A059 106 | S008C002P025R001A060 107 | S008C002P029R001A004 108 | S008C002P031R001A005 109 | S008C002P031R001A006 110 | S008C002P032R001A018 111 | S008C002P034R001A018 112 | S008C002P034R001A019 113 | S008C002P035R001A059 114 | S008C002P035R002A002 115 | S008C002P035R002A005 116 | S008C003P007R001A009 117 | S008C003P007R001A016 118 | S008C003P007R001A017 119 | S008C003P007R001A018 120 | S008C003P007R001A019 121 | S008C003P007R001A020 122 | S008C003P007R001A021 123 | S008C003P007R001A022 124 | S008C003P007R001A023 125 | S008C003P007R001A025 126 | S008C003P007R001A026 127 | S008C003P007R001A028 128 | S008C003P007R001A029 129 | S008C003P007R002A003 130 | S008C003P008R002A050 131 | S008C003P025R002A002 132 | S008C003P025R002A011 133 | S008C003P025R002A012 134 | S008C003P025R002A016 135 | S008C003P025R002A020 136 | S008C003P025R002A022 137 | S008C003P025R002A023 138 | S008C003P025R002A030 139 | S008C003P025R002A031 140 | S008C003P025R002A032 141 | S008C003P025R002A033 142 | S008C003P025R002A049 143 | S008C003P025R002A060 144 | S008C003P031R001A001 145 | S008C003P031R002A004 146 | S008C003P031R002A014 147 | S008C003P031R002A015 148 | S008C003P031R002A016 149 | S008C003P031R002A017 150 | S008C003P032R002A013 151 | S008C003P033R002A001 152 | S008C003P033R002A011 153 | S008C003P033R002A012 154 | S008C003P034R002A001 155 | S008C003P034R002A012 156 | S008C003P034R002A022 157 | S008C003P034R002A023 158 | S008C003P034R002A024 159 | S008C003P034R002A044 160 | S008C003P034R002A045 161 | S008C003P035R002A016 162 | S008C003P035R002A017 163 | S008C003P035R002A018 164 | S008C003P035R002A019 165 | S008C003P035R002A020 166 | S008C003P035R002A021 167 | S009C002P007R001A001 168 | S009C002P007R001A003 169 | S009C002P007R001A014 170 | S009C002P008R001A014 171 | S009C002P015R002A050 172 | S009C002P016R001A002 173 | S009C002P017R001A028 174 | S009C002P017R001A029 175 | S009C003P017R002A030 176 | S009C003P025R002A054 177 | S010C001P007R002A020 178 | S010C002P016R002A055 179 | S010C002P017R001A005 180 | S010C002P017R001A018 181 | S010C002P017R001A019 182 | S010C002P019R001A001 183 | S010C002P025R001A012 184 | S010C003P007R002A043 185 | S010C003P008R002A003 186 | S010C003P016R001A055 187 | S010C003P017R002A055 188 | S011C001P002R001A008 189 | S011C001P018R002A050 190 | S011C002P008R002A059 191 | S011C002P016R002A055 192 | S011C002P017R001A020 193 | S011C002P017R001A021 194 | S011C002P018R002A055 195 | S011C002P027R001A009 196 | S011C002P027R001A010 197 | S011C002P027R001A037 198 | S011C003P001R001A055 199 | S011C003P002R001A055 200 | S011C003P008R002A012 201 | S011C003P015R001A055 202 | S011C003P016R001A055 203 | S011C003P019R001A055 204 | S011C003P025R001A055 205 | S011C003P028R002A055 206 | S012C001P019R001A060 207 | S012C001P019R002A060 208 | S012C002P015R001A055 209 | S012C002P017R002A012 210 | S012C002P025R001A060 211 | S012C003P008R001A057 212 | S012C003P015R001A055 213 | S012C003P015R002A055 214 | S012C003P016R001A055 215 | S012C003P017R002A055 216 | S012C003P018R001A055 217 | S012C003P018R001A057 218 | S012C003P019R002A011 219 | S012C003P019R002A012 220 | S012C003P025R001A055 221 | S012C003P027R001A055 222 | S012C003P027R002A009 223 | S012C003P028R001A035 224 | S012C003P028R002A055 225 | S013C001P015R001A054 226 | S013C001P017R002A054 227 | S013C001P018R001A016 228 | S013C001P028R001A040 229 | S013C002P015R001A054 230 | S013C002P017R002A054 231 | S013C002P028R001A040 232 | S013C003P008R002A059 233 | S013C003P015R001A054 234 | S013C003P017R002A054 235 | S013C003P025R002A022 236 | S013C003P027R001A055 237 | S013C003P028R001A040 238 | S014C001P027R002A040 239 | S014C002P015R001A003 240 | S014C002P019R001A029 241 | S014C002P025R002A059 242 | S014C002P027R002A040 243 | S014C002P039R001A050 244 | S014C003P007R002A059 245 | S014C003P015R002A055 246 | S014C003P019R002A055 247 | S014C003P025R001A048 248 | S014C003P027R002A040 249 | S015C001P008R002A040 250 | S015C001P016R001A055 251 | S015C001P017R001A055 252 | S015C001P017R002A055 253 | S015C002P007R001A059 254 | S015C002P008R001A003 255 | S015C002P008R001A004 256 | S015C002P008R002A040 257 | S015C002P015R001A002 258 | S015C002P016R001A001 259 | S015C002P016R002A055 260 | S015C003P008R002A007 261 | S015C003P008R002A011 262 | S015C003P008R002A012 263 | S015C003P008R002A028 264 | S015C003P008R002A040 265 | S015C003P025R002A012 266 | S015C003P025R002A017 267 | S015C003P025R002A020 268 | S015C003P025R002A021 269 | S015C003P025R002A030 270 | S015C003P025R002A033 271 | S015C003P025R002A034 272 | S015C003P025R002A036 273 | S015C003P025R002A037 274 | S015C003P025R002A044 275 | S016C001P019R002A040 276 | S016C001P025R001A011 277 | S016C001P025R001A012 278 | S016C001P025R001A060 279 | S016C001P040R001A055 280 | S016C001P040R002A055 281 | S016C002P008R001A011 282 | S016C002P019R002A040 283 | S016C002P025R002A012 284 | S016C003P008R001A011 285 | S016C003P008R002A002 286 | S016C003P008R002A003 287 | S016C003P008R002A004 288 | S016C003P008R002A006 289 | S016C003P008R002A009 290 | S016C003P019R002A040 291 | S016C003P039R002A016 292 | S017C001P016R002A031 293 | S017C002P007R001A013 294 | S017C002P008R001A009 295 | S017C002P015R001A042 296 | S017C002P016R002A031 297 | S017C002P016R002A055 298 | S017C003P007R002A013 299 | S017C003P008R001A059 300 | S017C003P016R002A031 301 | S017C003P017R001A055 302 | S017C003P020R001A059 303 | S019C001P046R001A075 304 | S019C002P042R001A094 305 | S019C002P042R001A095 306 | S019C002P042R001A096 307 | S019C002P042R001A097 308 | S019C002P042R001A098 309 | S019C002P042R001A099 310 | S019C002P042R001A100 311 | S019C002P042R001A101 312 | S019C002P042R001A102 313 | S019C002P049R002A074 314 | S019C002P049R002A079 315 | S019C002P051R001A061 316 | S019C003P046R001A061 317 | S019C003P046R002A061 318 | S019C003P046R002A062 319 | S020C002P041R001A063 320 | S020C002P041R001A064 321 | S020C002P044R001A063 322 | S020C002P044R001A064 323 | S020C002P044R001A066 324 | S020C002P044R001A084 325 | S020C002P054R001A081 326 | S021C001P059R001A108 327 | S021C002P055R001A065 328 | S021C002P055R001A092 329 | S021C002P055R001A093 330 | S021C002P057R001A064 331 | S021C002P058R001A063 332 | S021C002P058R001A064 333 | S021C002P059R001A074 334 | S021C002P059R001A075 335 | S021C002P059R001A076 336 | S021C002P059R001A077 337 | S021C002P059R001A078 338 | S021C002P059R001A079 339 | S021C003P057R002A078 340 | S021C003P057R002A079 341 | S021C003P057R002A094 342 | S022C002P061R001A113 343 | S022C003P061R002A061 344 | S022C003P061R002A062 345 | S022C003P063R002A061 346 | S022C003P063R002A062 347 | S022C003P063R002A063 348 | S022C003P063R002A064 349 | S022C003P063R002A078 350 | S022C003P064R002A061 351 | S022C003P064R002A062 352 | S022C003P065R002A061 353 | S022C003P065R002A062 354 | S022C003P065R002A119 355 | S022C003P067R002A064 356 | S023C002P055R001A114 357 | S023C002P055R002A092 358 | S023C002P059R001A075 359 | S023C002P063R001A075 360 | S023C003P055R002A093 361 | S023C003P055R002A094 362 | S023C003P061R002A061 363 | S023C003P064R001A092 364 | S024C001P063R001A109 365 | S024C002P062R002A074 366 | S024C002P067R001A100 367 | S024C002P067R001A101 368 | S024C002P067R001A102 369 | S024C002P067R001A103 370 | S024C003P062R002A074 371 | S024C003P063R002A061 372 | S024C003P063R002A062 373 | S025C001P055R002A119 374 | S025C003P056R002A119 375 | S025C003P059R002A115 376 | S026C002P044R001A061 377 | S026C002P044R001A062 378 | S026C002P070R001A092 379 | S026C003P069R002A075 380 | S026C003P074R002A061 381 | S026C003P074R002A062 382 | S026C003P075R001A117 383 | S026C003P075R001A118 384 | S027C001P082R001A063 385 | S027C002P044R002A092 386 | S027C002P079R001A061 387 | S027C002P079R001A062 388 | S027C002P079R001A063 389 | S027C002P079R001A064 390 | S027C002P082R001A092 391 | S027C002P084R001A061 392 | S027C002P084R001A062 393 | S027C002P086R001A061 394 | S027C003P041R002A087 395 | S027C003P080R002A061 396 | S027C003P082R002A061 397 | S027C003P082R002A062 398 | S027C003P086R002A061 399 | S027C003P086R002A062 400 | S028C001P087R001A061 401 | S028C002P041R001A091 402 | S028C002P087R001A061 403 | S028C003P042R002A064 404 | S028C003P046R002A063 405 | S028C003P046R002A066 406 | S028C003P046R002A067 407 | S028C003P046R002A068 408 | S028C003P046R002A069 409 | S028C003P046R002A070 410 | S028C003P046R002A071 411 | S028C003P046R002A072 412 | S028C003P046R002A074 413 | S028C003P046R002A075 414 | S028C003P046R002A077 415 | S028C003P046R002A081 416 | S028C003P046R002A082 417 | S028C003P046R002A083 418 | S028C003P046R002A084 419 | S028C003P048R002A061 420 | S028C003P048R002A062 421 | S028C003P048R002A073 422 | S028C003P073R002A073 423 | S028C003P087R001A061 424 | S028C003P087R002A061 425 | S028C003P087R002A062 426 | S029C001P043R002A092 427 | S029C001P044R002A092 428 | S029C001P048R001A073 429 | S029C001P089R001A063 430 | S029C002P041R001A074 431 | S029C002P041R001A084 432 | S029C002P044R001A091 433 | S029C002P048R001A075 434 | S029C002P048R001A081 435 | S029C002P074R001A081 436 | S029C002P074R001A095 437 | S029C002P074R001A096 438 | S029C002P080R001A091 439 | S029C002P088R001A066 440 | S029C002P089R001A065 441 | S029C002P090R001A067 442 | S029C003P008R002A065 443 | S029C003P008R002A067 444 | S029C003P041R001A089 445 | S029C003P043R001A080 446 | S029C003P043R001A092 447 | S029C003P043R001A105 448 | S029C003P043R002A085 449 | S029C003P043R002A086 450 | S029C003P044R002A106 451 | S029C003P048R001A065 452 | S029C003P048R002A073 453 | S029C003P048R002A074 454 | S029C003P048R002A075 455 | S029C003P048R002A076 456 | S029C003P048R002A092 457 | S029C003P048R002A094 458 | S029C003P051R002A073 459 | S029C003P051R002A074 460 | S029C003P051R002A075 461 | S029C003P051R002A076 462 | S029C003P051R002A077 463 | S029C003P051R002A078 464 | S029C003P051R002A079 465 | S029C003P051R002A080 466 | S029C003P051R002A081 467 | S029C003P051R002A082 468 | S029C003P051R002A083 469 | S029C003P051R002A084 470 | S029C003P051R002A085 471 | S029C003P051R002A086 472 | S029C003P051R002A110 473 | S029C003P067R001A098 474 | S029C003P074R002A110 475 | S029C003P080R002A066 476 | S029C003P088R002A078 477 | S029C003P089R001A075 478 | S029C003P089R002A061 479 | S029C003P089R002A062 480 | S029C003P089R002A063 481 | S029C003P090R002A092 482 | S029C003P090R002A095 483 | S030C002P091R002A091 484 | S030C002P091R002A092 485 | S030C002P091R002A093 486 | S030C002P091R002A094 487 | S030C002P091R002A095 488 | S030C002P091R002A096 489 | S030C002P091R002A097 490 | S030C002P091R002A098 491 | S030C002P091R002A099 492 | S030C002P091R002A100 493 | S030C002P091R002A101 494 | S030C002P091R002A102 495 | S030C002P091R002A103 496 | S030C002P091R002A104 497 | S030C002P091R002A105 498 | S030C003P044R002A065 499 | S030C003P044R002A081 500 | S030C003P044R002A084 501 | S031C002P042R001A111 502 | S031C002P051R001A061 503 | S031C002P051R001A062 504 | S031C002P067R001A067 505 | S031C002P067R001A068 506 | S031C002P067R001A069 507 | S031C002P067R001A070 508 | S031C002P067R001A071 509 | S031C002P067R001A072 510 | S031C002P082R001A075 511 | S031C002P082R002A117 512 | S031C002P097R001A061 513 | S031C002P097R001A062 514 | S031C003P043R002A074 515 | S031C003P043R002A075 516 | S031C003P044R002A094 517 | S031C003P082R002A067 518 | S031C003P082R002A068 519 | S031C003P082R002A069 520 | S031C003P082R002A070 521 | S031C003P082R002A071 522 | S031C003P082R002A072 523 | S031C003P082R002A073 524 | S031C003P082R002A075 525 | S031C003P082R002A076 526 | S031C003P082R002A077 527 | S031C003P082R002A084 528 | S031C003P082R002A085 529 | S031C003P082R002A086 530 | S032C002P067R001A092 531 | S032C003P067R002A066 532 | S032C003P067R002A067 533 | S032C003P067R002A075 534 | S032C003P067R002A076 535 | S032C003P067R002A077 -------------------------------------------------------------------------------- /ntu_rgbd_parser/readme.md: -------------------------------------------------------------------------------- 1 | # NTU RGBD 120 database skeleton data parser in python 2 | 3 | 4 | 5 | This repository is to parse the skeleton data in NTU RGB+D 120 database[1]. Although the database already contains the skeleton data parser, It's written in MATLAB which is dependent on the large, commercial platform MATLAB. Also, the script in MATLAB is quite slow and hard to be parallel, making the process of extract the valid data in txt files tedious and time-consuming. To address those problems and make it easy to embed in the whole python project, I write the skeleton data parser in python. 6 | 7 | # Files in Project 8 | 9 | The structure of the project looks like: 10 | 11 | ```shell 12 | read_ntu_rgbd/ 13 | ├── ntu_rgb120_missings.txt # The missing files list, each item in this list needs to be ignored 14 | ├── raw_txt/ # the folder contains the original .txt skeleton data 15 | ├── raw_npy/ # the folder contains the parsed numpy arrays 16 | ├── readme.md # it's me :) 17 | └── txt2npy.py # the main script to parse the data 18 | ``` 19 | 20 | You need to move your original txt data in the `raw_txt/` folder first. 21 | 22 | 23 | 24 | # Run the script 25 | 26 | Before running the script, there are some configs you may need to modify to fit your system. Edit the `txt2npy.py` 27 | 28 | You need to specify the `save_npy_path`, `load_txt_path` to load and save the data. 29 | 30 | After the prepare, just run: 31 | 32 | ```shell 33 | python txt2npy.py 34 | ``` 35 | 36 | and wait for the ending. 37 | 38 | 39 | 40 | # the data structure in array 41 | 42 | In the data parsing process, I mainly concern some but not all items, there are what I most concern: 43 | 44 | 1. skeleton X, Y, Z coordinate 45 | 2. the skeleton coordinate projection on RGBs, it's a 2D coordinate X,Y 46 | 3. the skeleton coordinate projection on Depth, it's a 2D coordinate X,Y 47 | 48 | There are still other items left, like the ` handRightConfidence `, ` handLeftState `, ` trackingState `, ` handLeftConfidence ` etc. But I just ignore them in this parser, if you need them you need to modify the code. Don't worry, it's rather easy. 49 | 50 | Each sample is saved as the array in numpy with name like `SxxxCxxxPxxxRxxxAxxx.skeleton.npy` . You can read the data by: 51 | 52 | ```python 53 | data = np.load('./SxxxCxxxPxxxRxxxAxxx.skeleton.npy',allow_pickle=True).item() 54 | ``` 55 | 56 | Inside the array, it's actually a dictionary with some keys and each keys is a certain item you need. I list the item as following: 57 | 58 | - `file_name`: file's name 59 | - `nbodys`: it's a list with same length of the sequence. it represents the number of the actors in each frame. 60 | - `njoints`: the number of the joint node in the skeleton, it's a constant here 61 | - `skel_bodyx`: the skeleton coordinate with the shape of `(nframe, njoint, 3)`, the x denotes the id of the acting person in each frame. 62 | - `rgb_bodyx`: the projection of the skeleton coordinate in RGBs. 63 | - `depth_bodyx`: the projection of the skeleton coordinate in Depths 64 | 65 | 66 | 67 | # Reference 68 | 69 | [1]. https://github.com/shahroudy/NTURGB-D -------------------------------------------------------------------------------- /ntu_rgbd_parser/txt2npy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | ''' 5 | transform the skeleton data in NTU RGB+D dataset into the numpy arrays for a more efficient data loading 6 | ''' 7 | 8 | import numpy as np 9 | import os 10 | import sys 11 | 12 | user_name = 'user' 13 | save_npy_path = '/home/user/AI_workspace/datasets/NTU/raw_npy120/' 14 | load_txt_path = '/home/user/AI_workspace/datasets/NTU/raw_txt120/' 15 | missing_file_path = './ntu_rgb120_missings.txt' 16 | step_ranges = list(range(0,100)) # just parse range, for the purpose of paralle running. 17 | 18 | 19 | toolbar_width = 50 20 | def _print_toolbar(rate, annotation=''): 21 | sys.stdout.write("{}[".format(annotation)) 22 | for i in range(toolbar_width): 23 | if i * 1.0 / toolbar_width > rate: 24 | sys.stdout.write(' ') 25 | else: 26 | sys.stdout.write('-') 27 | sys.stdout.flush() 28 | sys.stdout.write(']\r') 29 | 30 | def _end_toolbar(): 31 | sys.stdout.write('\n') 32 | 33 | def _load_missing_file(path): 34 | missing_files = dict() 35 | with open(path, 'r') as f: 36 | lines = f.readlines() 37 | for line in lines: 38 | line = line[:-1] 39 | if line not in missing_files: 40 | missing_files[line] = True 41 | return missing_files 42 | 43 | def _read_skeleton(file_path, save_skelxyz=True, save_rgbxy=True, save_depthxy=True): 44 | f = open(file_path, 'r') 45 | datas = f.readlines() 46 | f.close() 47 | max_body = 4 48 | njoints = 25 49 | 50 | # specify the maximum number of the body shown in the sequence, according to the certain sequence, need to pune the 51 | # abundant bodys. 52 | # read all lines into the pool to speed up, less io operation. 53 | nframe = int(datas[0][:-1]) 54 | bodymat = dict() 55 | bodymat['file_name'] = file_path[-29:-9] 56 | nbody = int(datas[1][:-1]) 57 | bodymat['nbodys'] = [] 58 | bodymat['njoints'] = njoints 59 | for body in range(max_body): 60 | if save_skelxyz: 61 | bodymat['skel_body{}'.format(body)] = np.zeros(shape=(nframe, njoints, 3)) 62 | if save_rgbxy: 63 | bodymat['rgb_body{}'.format(body)] = np.zeros(shape=(nframe, njoints, 2)) 64 | if save_depthxy: 65 | bodymat['depth_body{}'.format(body)] = np.zeros(shape=(nframe, njoints, 2)) 66 | # above prepare the data holder 67 | cursor = 0 68 | for frame in range(nframe): 69 | cursor += 1 70 | bodycount = int(datas[cursor][:-1]) 71 | if bodycount == 0: 72 | continue 73 | # skip the empty frame 74 | bodymat['nbodys'].append(bodycount) 75 | for body in range(bodycount): 76 | cursor += 1 77 | skel_body = 'skel_body{}'.format(body) 78 | rgb_body = 'rgb_body{}'.format(body) 79 | depth_body = 'depth_body{}'.format(body) 80 | 81 | bodyinfo = datas[cursor][:-1].split(' ') 82 | cursor += 1 83 | 84 | njoints = int(datas[cursor][:-1]) 85 | for joint in range(njoints): 86 | cursor += 1 87 | jointinfo = datas[cursor][:-1].split(' ') 88 | jointinfo = np.array(list(map(float, jointinfo))) 89 | if save_skelxyz: 90 | bodymat[skel_body][frame,joint] = jointinfo[:3] 91 | if save_depthxy: 92 | bodymat[depth_body][frame,joint] = jointinfo[3:5] 93 | if save_rgbxy: 94 | bodymat[rgb_body][frame,joint] = jointinfo[5:7] 95 | # prune the abundant bodys 96 | for each in range(max_body): 97 | if each >= max(bodymat['nbodys']): 98 | if save_skelxyz: 99 | del bodymat['skel_body{}'.format(each)] 100 | if save_rgbxy: 101 | del bodymat['rgb_body{}'.format(each)] 102 | if save_depthxy: 103 | del bodymat['depth_body{}'.format(each)] 104 | return bodymat 105 | 106 | 107 | 108 | 109 | if __name__ == '__main__': 110 | missing_files = _load_missing_file(missing_file_path) 111 | datalist = os.listdir(load_txt_path) 112 | alread_exist = os.listdir(save_npy_path) 113 | alread_exist_dict = dict(zip(alread_exist, len(alread_exist) * [True])) 114 | 115 | for ind, each in enumerate(datalist): 116 | _print_toolbar(ind * 1.0 / len(datalist), 117 | '({:>5}/{:<5})'.format( 118 | ind + 1, len(datalist) 119 | )) 120 | S = int(each[1:4]) 121 | if S not in step_ranges: 122 | continue 123 | if each+'.skeleton.npy' in alread_exist_dict: 124 | print('file already existed !') 125 | continue 126 | if each[:20] in missing_files: 127 | print('file missing') 128 | continue 129 | loadname = load_txt_path+each 130 | print(each) 131 | mat = _read_skeleton(loadname) 132 | mat = np.array(mat) 133 | save_path = save_npy_path+'{}.npy'.format(each) 134 | np.save(save_path, mat) 135 | # raise ValueError() 136 | _end_toolbar() 137 | 138 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Part-Aware LSTM implemented in TensorFlow 2 | **part-aware lstm** is proposed in [《NTU RGB+D: A Large Scale Dataset for 3D Human Activity Analysis》](https://arxiv.org/abs/1604.02808), which is used for skeleton-based action recognition. It splits the whole body into 5 body part context. The specially designed lstm cell could better extract the context features than the normal one. The cell diagram is shown following: 3 | ![plstm][plstm] 4 | 5 | Comparing with the normal lstm cell, it is not difficult to find that plstm has seperated i, g and g gate corresponding to different body part. 6 | 7 | It is inspired. So let us implement it in TensorFlow. 8 | 9 | Folder `network` includes three files, `PartAwareLSTMCell.py`,`DataLoader.py` and `PLSTM.py`. 10 | 1. `PartAwareLSTMCell.py` is the Part-Aware LSTM cell. 11 | 2. `DataLoader.py` is the data loader used to load and preprocess the skeleton datas. Note that all skeleton data are formatted like **Array [Number_clips, {'mat':data, 'view': v, 'class':c, 'actor':a}] data with shape of [Frames, 25, 3]** 12 | 3. `PLSTM.py` is the main train and evaluation entry. 13 | 14 | Folder `utils` include one file, `gendata.py` which uses to generate the formatted data in numpy arrays. Note that all raw skeleton is stored in txt file and i transform them to mat file in MATLAB and i only save the (x,y,z) information. 15 | 16 | ## update 17 | **2018 4.13**, add a jupyter notebook script in `app` folder used for training and evaluation. The code has not clear yet but could be a reference. 18 | 19 | **2018 5.7**, add a .mat file sample in `datas` folder to give a typical example of the .mat data formation. 20 | 21 | **2020 1.11**, add the NTU RGBD 120 skeleton data parser with python in folder `ntu_rgbd_parser`, for more detail, go to my another repository [2]. 22 | 23 | 24 | 25 | 26 | [plstm]: ./imgs/plstm.png 27 | 28 | 29 | 30 | # Reference 31 | 32 | [1]. Shahroudy A, Liu J, Ng T T, et al. Ntu rgb+ d: A large scale dataset for 3d human activity analysis[C]//Proceedings of the IEEE conference on computer vision and pattern recognition. 2016: 1010-1019. 33 | 34 | [2]. https://github.com/FesianXu/NTU_RGBD120_Parser_python 35 | 36 | [3]. Liu J, Shahroudy A, Perez M L, et al. NTU RGB+ D 120: A Large-Scale Benchmark for 3D Human Activity Understanding[J]. IEEE transactions on pattern analysis and machine intelligence, 2019. -------------------------------------------------------------------------------- /utils/gendata.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | __author__ = 'FesianXu' 5 | __date__ = 2018 / 3 / 22 6 | __version__ = '' 7 | 8 | ''' 9 | test and train dataset generator based on x-sub or x-view 10 | generated data formation: 11 | Array [Number_clips, {'mat':data, 'view': v, 'class':c, 'actor':a}] 12 | data [Frames, 25, 3] 13 | ''' 14 | 15 | import numpy as np 16 | import scipy.io as sio 17 | 18 | x_mode = 'x-view' 19 | mode = 'train' 20 | # only consider the x-view not loop 21 | total_views = [0, 1, 2, 3, 4, 5, 6, 7] # 0 is front view and 1-7 is 45d ti -45d, 8 is loop samples, not exist in this list 22 | train_views_config = [7] # modify this list to select the training views 23 | test_views_config = [i for i in total_views if i not in train_views_config] 24 | 25 | total_actors = range(1, 119) # actor id begin with 1, the total actor is 118 persons 26 | # train_actors_config = range(1, 51) 27 | train_actors_config = np.array([ 1, 2, 6, 12, 13, 16, 21, 24, 28, 29, 30, 31, 33, 28 | 35, 39, 41, 42, 45, 47, 50, 52, 54, 55, 57, 59, 61, 29 | 63, 64, 67, 69, 70, 71, 73, 77, 81, 84, 86, 87, 88, 30 | 90, 91, 93, 96, 99, 102, 103, 104, 107, 108, 112, 113]) 31 | test_actors_config = [i for i in total_actors if i not in train_actors_config] 32 | 33 | cameras_list = (1, 2) 34 | num_actions = 40 35 | num_camera = 2 36 | num_views = len(total_views) 37 | num_actors = len(total_actors) 38 | 39 | #### config above ##### 40 | 41 | def _get_file_name(action_id, camera_id, view_id, actor_id): 42 | # need to modify the file name formation 43 | # return path+'a{}_c{}_d{}_p{:0>2d}_choose_skelton.mat'.format(action_id, camera_id, view_id, actor_id) 44 | return 'a{:0>2}_d{}_p{:0>3d}_c{}.mat'.format(action_id, view_id, actor_id, camera_id) 45 | 46 | 47 | def _check_view(camera_id, view_id): 48 | # clips in camera 1 is all front view i.e. 0 view 49 | # but i eliminate all loop samples no matter it is in #1 or #2 host 50 | # loop sample will process individually 51 | if view_id == 7: 52 | return 'loop' 53 | if camera_id == 1: 54 | return 0 55 | elif camera_id == 2: 56 | return view_id+1 57 | else: 58 | raise ValueError() 59 | 60 | def _check_mode(view_id, actor_id, x_mode, mode): 61 | if x_mode == 'x-view': 62 | if mode == 'train': 63 | return (view_id in train_views_config) 64 | elif mode == 'test': 65 | return (view_id in test_views_config) 66 | else: 67 | raise ValueError() 68 | 69 | elif x_mode == 'x-sub': 70 | if mode == 'train': 71 | return (actor_id in train_actors_config) 72 | elif mode == 'test': 73 | return (actor_id in test_actors_config) 74 | else: 75 | raise ValueError() 76 | 77 | else: 78 | raise ValueError() 79 | 80 | 81 | def _read_data(file_path): 82 | # [frames, 25, 3] 83 | mat = sio.loadmat(file_path) 84 | mat = mat['skel'] 85 | mat = mat.astype(dtype=np.float32) 86 | return mat 87 | 88 | 89 | def _x_view_gendata(raw_path, x_mode, mode): 90 | datasets = [] 91 | counter = 0 92 | missing_list = [] 93 | 94 | for each_action in range(num_actions): 95 | for each_camera in cameras_list: 96 | for each_view in total_views: # (0, 7) 97 | for each_actor in total_actors: 98 | view = _check_view(view_id=each_view, camera_id=each_camera) # 1 - 8 99 | if view == 'loop': 100 | continue 101 | # ignore loop samples, leave (1, 7) 102 | if not _check_mode(view_id=view, actor_id=each_actor, x_mode=x_mode, mode=mode): 103 | continue 104 | file_name = raw_path+_get_file_name(each_action, each_camera, each_view+1, each_actor) 105 | try: 106 | data_clip = _read_data(file_name) # return x,y,z 107 | except FileNotFoundError: 108 | # print('file not exist %s' % file_name) 109 | missing_list.append(file_name) 110 | continue 111 | block = [] 112 | if view in (0, 1, 2): 113 | block += [0] 114 | if view in (2, 3, 4): 115 | block += [1] 116 | if view in (4, 5, 6): 117 | block += [2] 118 | if view in (6, 7, 0): 119 | block += [3] 120 | 121 | data_dict = { 122 | 'id': counter, # start from 0 123 | 'mat': data_clip, 124 | 'view': view, # from 0 to 7 125 | 'block': block, # block id, divide 8 views into 4 blocks 126 | 'action': each_action, # start from 0 127 | 'actor': each_actor # start from 0 128 | } 129 | datasets.append(data_dict) 130 | counter += 1 131 | print(counter) 132 | datasets = np.array(datasets) 133 | return datasets 134 | 135 | def _x_sub_gendata(raw_path, x_mode, mode): 136 | datasets = [] 137 | counter = 0 138 | for each_action in range(num_actions): 139 | for each_camera in cameras_list: 140 | for each_view in total_views: # (0, 7) 141 | if mode == 'train': 142 | actor_list = train_actors_config 143 | elif mode == 'test': 144 | actor_list = test_actors_config 145 | # select valid actors 146 | 147 | for each_actor in actor_list: 148 | view = _check_view(view_id=each_view, camera_id=each_camera) # (0, 7) here each_view need to be (0 - 7) 149 | if view == 'loop': 150 | continue 151 | # ignore the loop samples 152 | file_name = raw_path+_get_file_name(each_action, each_camera, each_view+1, each_actor) 153 | try: 154 | data_clip = _read_data(file_name) # return x,y,z 155 | except FileNotFoundError: 156 | continue 157 | if data_clip is None: 158 | raise ValueError() 159 | 160 | block = [] 161 | if view in (0, 1, 2): 162 | block += [0] 163 | if view in (2, 3, 4): 164 | block += [1] 165 | if view in (4, 5, 6): 166 | block += [2] 167 | if view in (6, 7, 0): 168 | block += [3] 169 | 170 | data_dict = { 171 | 'id': counter, # start from 0 172 | 'mat': data_clip, 173 | 'view': view, # from 0 to 7 174 | 'block': block, # block id, divide 8 views into 4 blocks 175 | 'action': each_action, # start from 0 176 | 'actor': each_actor # start from 0 177 | } 178 | datasets.append(data_dict) 179 | counter += 1 180 | print(counter) 181 | datasets = np.array(datasets) 182 | return datasets 183 | 184 | 185 | 186 | def gendata(raw_path, x_mode='x-view', mode='train'): 187 | ## this gendata is only used on DeepLSTM,ResTCN,SkelCNN 188 | ## and not used on ST-GCN ! 189 | if x_mode == 'x-view': 190 | dataset = _x_view_gendata(raw_path, x_mode, mode) 191 | elif x_mode == 'x-sub': 192 | dataset = _x_sub_gendata(raw_path, x_mode, mode) 193 | else: 194 | raise ValueError() 195 | return dataset 196 | 197 | 198 | 199 | raw_path = '/home/fesian/AI_workspace/datasets/HRI40/raw/mats/' 200 | # 201 | view_save_path = '/home/fesian/AI_workspace/datasets/HRI40/hri40_new_skel/x_view/' 202 | sub_save_path = '/home/fesian/AI_workspace/datasets/HRI40/hri40_new_skel/x_sub/' 203 | 204 | is_view = True 205 | 206 | if is_view: 207 | dat = gendata(raw_path, mode='test', x_mode='x-view') 208 | np.save(view_save_path+'/x_view_test_v7.npy', dat) 209 | 210 | dat = gendata(raw_path, mode='train', x_mode='x-view') 211 | np.save(view_save_path+'/x_view_train_v7.npy', dat) 212 | else: 213 | dat = gendata(raw_path, mode='test', x_mode='x-sub') 214 | np.save(sub_save_path+'/x_sub_test.npy', dat) 215 | 216 | dat = gendata(raw_path, mode='train', x_mode='x-sub') 217 | np.save(sub_save_path+'/x_sub_train.npy', dat) 218 | 219 | 220 | --------------------------------------------------------------------------------