├── models ├── __init__.py ├── __pycache__ │ ├── gat.cpython-35.pyc │ ├── gat.cpython-36.pyc │ ├── sp_gat.cpython-35.pyc │ ├── sp_gat.cpython-36.pyc │ ├── __init__.cpython-35.pyc │ ├── __init__.cpython-36.pyc │ ├── base_gattn.cpython-35.pyc │ └── base_gattn.cpython-36.pyc ├── sp_gat.py ├── gat.py └── base_gattn.py ├── img ├── overview.png └── SM-SGE-results.png ├── LICENSE ├── utils ├── layers.py ├── process_cme.py └── process_i.py ├── evaluate.py ├── README.md └── SM-SGE.py /models/__init__.py: -------------------------------------------------------------------------------- 1 | from .gat import MGRN_S 2 | from .sp_gat import SpGAT 3 | -------------------------------------------------------------------------------- /img/overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kali-Hac/SM-SGE/HEAD/img/overview.png -------------------------------------------------------------------------------- /img/SM-SGE-results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kali-Hac/SM-SGE/HEAD/img/SM-SGE-results.png -------------------------------------------------------------------------------- /models/__pycache__/gat.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kali-Hac/SM-SGE/HEAD/models/__pycache__/gat.cpython-35.pyc -------------------------------------------------------------------------------- /models/__pycache__/gat.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kali-Hac/SM-SGE/HEAD/models/__pycache__/gat.cpython-36.pyc -------------------------------------------------------------------------------- /models/__pycache__/sp_gat.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kali-Hac/SM-SGE/HEAD/models/__pycache__/sp_gat.cpython-35.pyc -------------------------------------------------------------------------------- /models/__pycache__/sp_gat.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kali-Hac/SM-SGE/HEAD/models/__pycache__/sp_gat.cpython-36.pyc -------------------------------------------------------------------------------- /models/__pycache__/__init__.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kali-Hac/SM-SGE/HEAD/models/__pycache__/__init__.cpython-35.pyc -------------------------------------------------------------------------------- /models/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kali-Hac/SM-SGE/HEAD/models/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /models/__pycache__/base_gattn.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kali-Hac/SM-SGE/HEAD/models/__pycache__/base_gattn.cpython-35.pyc -------------------------------------------------------------------------------- /models/__pycache__/base_gattn.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kali-Hac/SM-SGE/HEAD/models/__pycache__/base_gattn.cpython-36.pyc -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Anonymous-926 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /models/sp_gat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | from utils import layers 5 | from models.base_gattn import BaseGAttN 6 | 7 | class SpGAT(BaseGAttN): 8 | def inference(inputs, nb_classes, nb_nodes, training, attn_drop, ffd_drop, 9 | bias_mat, hid_units, n_heads, activation=tf.nn.elu, 10 | residual=False): 11 | attns = [] 12 | for _ in range(n_heads[0]): 13 | attns.append(layers.sp_attn_head(inputs, 14 | adj_mat=bias_mat, 15 | out_sz=hid_units[0], activation=activation, nb_nodes=nb_nodes, 16 | in_drop=ffd_drop, coef_drop=attn_drop, residual=False)) 17 | h_1 = tf.concat(attns, axis=-1) 18 | for i in range(1, len(hid_units)): 19 | h_old = h_1 20 | attns = [] 21 | for _ in range(n_heads[i]): 22 | attns.append(layers.sp_attn_head(h_1, 23 | adj_mat=bias_mat, 24 | out_sz=hid_units[i], activation=activation, nb_nodes=nb_nodes, 25 | in_drop=ffd_drop, coef_drop=attn_drop, residual=residual)) 26 | h_1 = tf.concat(attns, axis=-1) 27 | out = [] 28 | for i in range(n_heads[-1]): 29 | out.append(layers.sp_attn_head(h_1, adj_mat=bias_mat, 30 | out_sz=nb_classes, activation=lambda x: x, nb_nodes=nb_nodes, 31 | in_drop=ffd_drop, coef_drop=attn_drop, residual=False)) 32 | logits = tf.add_n(out) / n_heads[-1] 33 | 34 | return logits 35 | -------------------------------------------------------------------------------- /models/gat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | from utils import layers 5 | from models.base_gattn import BaseGAttN 6 | 7 | class MGRN_S(BaseGAttN): 8 | def inference(inputs, nb_classes, nb_nodes, training, attn_drop, ffd_drop, 9 | bias_mat, hid_units, n_heads, activation=tf.nn.elu, residual=False, r_pool=True): 10 | attns = [] 11 | for _ in range(n_heads[0]): 12 | attns.append(layers.attn_head(inputs, bias_mat=bias_mat, 13 | out_sz=hid_units[0], activation=activation, 14 | in_drop=ffd_drop, coef_drop=attn_drop, residual=False)) 15 | h_1 = tf.concat(attns, axis=-1) 16 | for i in range(1, len(hid_units)): 17 | h_old = h_1 18 | attns = [] 19 | for _ in range(n_heads[i]): 20 | attns.append(layers.attn_head(h_1, bias_mat=bias_mat, 21 | out_sz=hid_units[i], activation=activation, 22 | in_drop=ffd_drop, coef_drop=attn_drop, residual=residual)) 23 | h_1 = tf.concat(attns, axis=-1) 24 | 25 | if nb_classes == 0: 26 | if r_pool: 27 | return tf.add_n(attns) / n_heads[0] 28 | else: 29 | return h_1 30 | out = [] 31 | for i in range(n_heads[-1]): 32 | out.append(layers.attn_head(h_1, bias_mat=bias_mat, 33 | out_sz=nb_classes, activation=lambda x: x, 34 | in_drop=ffd_drop, coef_drop=attn_drop, residual=False)) 35 | logits = tf.add_n(out) / n_heads[-1] 36 | return logits 37 | -------------------------------------------------------------------------------- /utils/layers.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | conv1d = tf.layers.conv1d 5 | 6 | def attn_head(seq, out_sz, bias_mat, activation, in_drop=0.0, coef_drop=0.0, residual=False): 7 | with tf.name_scope('my_attn'): 8 | if in_drop != 0.0: 9 | seq = tf.nn.dropout(seq, 1.0 - in_drop) 10 | 11 | seq_fts = tf.layers.conv1d(seq, out_sz, 1, use_bias=False) 12 | 13 | # simplest self-attention possible 14 | f_1 = tf.layers.conv1d(seq_fts, 1, 1) 15 | f_2 = tf.layers.conv1d(seq_fts, 1, 1) 16 | logits = f_1 + tf.transpose(f_2, [0, 2, 1]) 17 | 18 | coefs = tf.nn.softmax(tf.nn.leaky_relu(logits) + bias_mat) 19 | # coefs = tf.nn.softmax(tf.nn.leaky_relu(logits) / 0.1 + bias_mat) 20 | # print(coefs) 21 | # exit(1) 22 | 23 | if coef_drop != 0.0: 24 | coefs = tf.nn.dropout(coefs, 1.0 - coef_drop) 25 | if in_drop != 0.0: 26 | seq_fts = tf.nn.dropout(seq_fts, 1.0 - in_drop) 27 | 28 | vals = tf.matmul(coefs, seq_fts) 29 | ret = tf.contrib.layers.bias_add(vals) 30 | 31 | # residual connection 32 | if residual: 33 | if seq.shape[-1] != ret.shape[-1]: 34 | ret = ret + conv1d(seq, ret.shape[-1], 1) # activation 35 | else: 36 | ret = ret + seq 37 | 38 | return activation(ret) # activation 39 | 40 | # Experimental sparse attention head (for running on datasets such as Pubmed) 41 | # N.B. Because of limitations of current TF implementation, will work _only_ if batch_size = 1! 42 | def sp_attn_head(seq, out_sz, adj_mat, activation, nb_nodes, in_drop=0.0, coef_drop=0.0, residual=False): 43 | with tf.name_scope('sp_attn'): 44 | if in_drop != 0.0: 45 | seq = tf.nn.dropout(seq, 1.0 - in_drop) 46 | 47 | seq_fts = tf.layers.conv1d(seq, out_sz, 1, use_bias=False) 48 | 49 | # simplest self-attention possible 50 | f_1 = tf.layers.conv1d(seq_fts, 1, 1) 51 | f_2 = tf.layers.conv1d(seq_fts, 1, 1) 52 | 53 | f_1 = tf.reshape(f_1, (nb_nodes, 1)) 54 | f_2 = tf.reshape(f_2, (nb_nodes, 1)) 55 | 56 | f_1 = adj_mat*f_1 57 | f_2 = adj_mat * tf.transpose(f_2, [1,0]) 58 | 59 | logits = tf.sparse_add(f_1, f_2) 60 | lrelu = tf.SparseTensor(indices=logits.indices, 61 | values=tf.nn.leaky_relu(logits.values), 62 | dense_shape=logits.dense_shape) 63 | coefs = tf.sparse_softmax(lrelu) 64 | 65 | if coef_drop != 0.0: 66 | coefs = tf.SparseTensor(indices=coefs.indices, 67 | values=tf.nn.dropout(coefs.values, 1.0 - coef_drop), 68 | dense_shape=coefs.dense_shape) 69 | if in_drop != 0.0: 70 | seq_fts = tf.nn.dropout(seq_fts, 1.0 - in_drop) 71 | 72 | # As tf.sparse_tensor_dense_matmul expects its arguments to have rank-2, 73 | # here we make an assumption that our input is of batch size 1, and reshape appropriately. 74 | # The method will fail in all other cases! 75 | coefs = tf.sparse_reshape(coefs, [nb_nodes, nb_nodes]) 76 | seq_fts = tf.squeeze(seq_fts) 77 | vals = tf.sparse_tensor_dense_matmul(coefs, seq_fts) 78 | vals = tf.expand_dims(vals, axis=0) 79 | vals.set_shape([1, nb_nodes, out_sz]) 80 | ret = tf.contrib.layers.bias_add(vals) 81 | 82 | # residual connection 83 | if residual: 84 | if seq.shape[-1] != ret.shape[-1]: 85 | ret = ret + conv1d(seq, ret.shape[-1], 1) # activation 86 | else: 87 | ret = ret + seq 88 | 89 | return activation(ret) # activation 90 | 91 | -------------------------------------------------------------------------------- /models/base_gattn.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | class BaseGAttN: 4 | def loss(logits, labels, nb_classes, class_weights): 5 | sample_wts = tf.reduce_sum(tf.multiply(tf.one_hot(labels, nb_classes), class_weights), axis=-1) 6 | xentropy = tf.multiply(tf.nn.sparse_softmax_cross_entropy_with_logits( 7 | labels=labels, logits=logits), sample_wts) 8 | return tf.reduce_mean(xentropy, name='xentropy_mean') 9 | def training(loss, lr, l2_coef): 10 | # weight decay 11 | vars = tf.trainable_variables() 12 | lossL2 = tf.add_n([tf.nn.l2_loss(v) for v in vars if v.name not 13 | in ['bias', 'gamma', 'b', 'g', 'beta']]) * l2_coef 14 | 15 | # optimizer 16 | opt = tf.train.AdamOptimizer(learning_rate=lr) 17 | # train_op = opt.minimize(loss) 18 | # training op 19 | train_op = opt.minimize(loss+lossL2) 20 | 21 | return train_op 22 | 23 | def preshape(logits, labels, nb_classes): 24 | new_sh_lab = [-1] 25 | new_sh_log = [-1, nb_classes] 26 | log_resh = tf.reshape(logits, new_sh_log) 27 | lab_resh = tf.reshape(labels, new_sh_lab) 28 | return log_resh, lab_resh 29 | 30 | def confmat(logits, labels): 31 | preds = tf.argmax(logits, axis=1) 32 | return tf.confusion_matrix(labels, preds) 33 | 34 | ########################## 35 | # Adapted from tkipf/gcn # 36 | ########################## 37 | 38 | def masked_softmax_cross_entropy(logits, labels, mask): 39 | """Softmax cross-entropy loss with masking.""" 40 | loss = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels) 41 | mask = tf.cast(mask, dtype=tf.float32) 42 | mask /= tf.reduce_mean(mask) 43 | loss *= mask 44 | return tf.reduce_mean(loss) 45 | 46 | def masked_sigmoid_cross_entropy(logits, labels, mask): 47 | """Softmax cross-entropy loss with masking.""" 48 | labels = tf.cast(labels, dtype=tf.float32) 49 | loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=labels) 50 | loss=tf.reduce_mean(loss,axis=1) 51 | mask = tf.cast(mask, dtype=tf.float32) 52 | mask /= tf.reduce_mean(mask) 53 | loss *= mask 54 | return tf.reduce_mean(loss) 55 | 56 | def masked_accuracy(logits, labels, mask): 57 | """Accuracy with masking.""" 58 | correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(labels, 1)) 59 | accuracy_all = tf.cast(correct_prediction, tf.float32) 60 | mask = tf.cast(mask, dtype=tf.float32) 61 | mask /= tf.reduce_mean(mask) 62 | accuracy_all *= mask 63 | return tf.reduce_mean(accuracy_all) 64 | 65 | def micro_f1(logits, labels, mask): 66 | """Accuracy with masking.""" 67 | predicted = tf.round(tf.nn.sigmoid(logits)) 68 | 69 | # Use integers to avoid any nasty FP behaviour 70 | predicted = tf.cast(predicted, dtype=tf.int32) 71 | labels = tf.cast(labels, dtype=tf.int32) 72 | mask = tf.cast(mask, dtype=tf.int32) 73 | 74 | # expand the mask so that broadcasting works ([nb_nodes, 1]) 75 | mask = tf.expand_dims(mask, -1) 76 | 77 | # Count true positives, true negatives, false positives and false negatives. 78 | tp = tf.count_nonzero(predicted * labels * mask) 79 | tn = tf.count_nonzero((predicted - 1) * (labels - 1) * mask) 80 | fp = tf.count_nonzero(predicted * (labels - 1) * mask) 81 | fn = tf.count_nonzero((predicted - 1) * labels * mask) 82 | 83 | # Calculate accuracy, precision, recall and F1 score. 84 | precision = tp / (tp + fp) 85 | recall = tp / (tp + fn) 86 | fmeasure = (2 * precision * recall) / (precision + recall) 87 | fmeasure = tf.cast(fmeasure, tf.float32) 88 | return fmeasure 89 | -------------------------------------------------------------------------------- /evaluate.py: -------------------------------------------------------------------------------- 1 | from sklearn.preprocessing import label_binarize 2 | from sklearn.metrics import roc_curve, auc, confusion_matrix 3 | import numpy as np 4 | import tensorflow as tf 5 | import os, sys 6 | from utils import process_i as process 7 | 8 | config = tf.ConfigProto() 9 | config.gpu_options.allow_growth = True 10 | 11 | nb_nodes = 20 12 | nhood = 1 13 | ft_size = 3 14 | time_step = 6 15 | 16 | batch_size = 256 17 | lr = 0.005 # learning rate 18 | 19 | tf.app.flags.DEFINE_string('dataset', 'KS20', "Dataset: IAS, KS20 or KGBD") 20 | tf.app.flags.DEFINE_string('length', '6', "4, 6, 8 or 10") 21 | tf.app.flags.DEFINE_string('split', '', "for IAS-Lab testing splits (A or B)") 22 | tf.app.flags.DEFINE_string('gpu', '0', "GPU number") 23 | tf.app.flags.DEFINE_string('model_dir', 'best', "model directory") # 'best' will test the best model in current directory 24 | 25 | FLAGS = tf.app.flags.FLAGS 26 | 27 | # check parameters 28 | if FLAGS.dataset not in ['IAS', 'KGBD', 'KS20']: 29 | raise Exception('Dataset must be IAS, KGBD, or KS20.') 30 | if not FLAGS.gpu.isdigit() or int(FLAGS.gpu) < 0: 31 | raise Exception('GPU number must be a positive integer.') 32 | if FLAGS.length not in ['4', '6', '8', '10']: 33 | raise Exception('Length number must be 4, 6, 8 or 10.') 34 | if FLAGS.split not in ['', 'A', 'B']: 35 | raise Exception('Datset split must be "A" (for IAS-A), "B" (for IAS-B), "" (for other datasets).') 36 | 37 | os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu 38 | dataset = FLAGS.dataset 39 | time_step = int(FLAGS.length) 40 | split = FLAGS.split 41 | model_dir = FLAGS.model_dir 42 | 43 | 44 | def evaluate_reid(model_dir, dataset): 45 | if dataset == 'IAS': 46 | classes = list(range(11)) 47 | elif dataset == 'KS20': 48 | classes = list(range(20)) 49 | elif dataset == 'KGBD': 50 | classes = list(range(164)) 51 | checkpoint = model_dir + ".ckpt" 52 | print('Evaluating the model saved in ' + model_dir) 53 | loaded_graph = tf.get_default_graph() 54 | 55 | with tf.Session(graph=loaded_graph, config=config) as sess: 56 | loader = tf.train.import_meta_graph(checkpoint + '.meta') 57 | loader.restore(sess, checkpoint) 58 | # loader = tf.train.import_meta_graph(checkpt_file + '.meta') 59 | # loader.restore(sess, checkpt_file) 60 | J_in = loaded_graph.get_tensor_by_name("Input/Placeholder_1:0") 61 | P_in = loaded_graph.get_tensor_by_name("Input/Placeholder_2:0") 62 | B_in = loaded_graph.get_tensor_by_name("Input/Placeholder_3:0") 63 | I_in = loaded_graph.get_tensor_by_name("Input/Placeholder_4:0") 64 | J_bias_in = loaded_graph.get_tensor_by_name("Input/Placeholder_5:0") 65 | P_bias_in = loaded_graph.get_tensor_by_name("Input/Placeholder_6:0") 66 | B_bias_in = loaded_graph.get_tensor_by_name("Input/Placeholder_7:0") 67 | I_bias_in = loaded_graph.get_tensor_by_name("Input/Placeholder_8:0") 68 | lbl_in = loaded_graph.get_tensor_by_name("Input/Placeholder:0") 69 | is_train = loaded_graph.get_tensor_by_name("Input/Placeholder_11:0") 70 | attn_drop = loaded_graph.get_tensor_by_name("Input/Placeholder_9:0") 71 | ffd_drop = loaded_graph.get_tensor_by_name("Input/Placeholder_10:0") 72 | aver_pre = loaded_graph.get_tensor_by_name('Recognition/Recognition/add_30:0') 73 | accuracy = loaded_graph.get_tensor_by_name('Recognition/Recognition/Mean_4:0') 74 | loss = loaded_graph.get_tensor_by_name('Recognition/Recognition/Mean_5:0') 75 | rank_acc = {} 76 | en_to_pred = loaded_graph.get_tensor_by_name("Recognition/Recognition/StopGradient:0") 77 | 78 | X_train_J, X_train_P, X_train_B, X_train_I, y_train, X_test_J, X_test_P, X_test_B, X_test_I, y_test, \ 79 | adj_J, biases_J, adj_P, biases_P, adj_B, biases_B, adj_I, biases_I, nb_classes = \ 80 | process.gen_train_data(dataset=dataset, split=split, time_step=time_step, 81 | nb_nodes=nb_nodes, nhood=nhood, global_att=False, batch_size=batch_size, view='', 82 | reverse='0') 83 | # print(batch_size) 84 | X_train = X_train_J 85 | X_test = X_test_J 86 | vl_step = 0 87 | vl_size = X_test.shape[0] 88 | logits_all = [] 89 | labels_all = [] 90 | 91 | vl_step = 0 92 | vl_loss = 0.0 93 | vl_acc = 0.0 94 | while vl_step * batch_size < vl_size: 95 | if (vl_step + 1) * batch_size > vl_size: 96 | break 97 | X_input_J = X_test_J[vl_step * batch_size:(vl_step + 1) * batch_size] 98 | X_input_J = X_input_J.reshape([-1, nb_nodes, 3]) 99 | X_input_P = X_test_P[vl_step * batch_size:(vl_step + 1) * batch_size] 100 | X_input_P = X_input_P.reshape([-1, 10, 3]) 101 | X_input_B = X_test_B[vl_step * batch_size:(vl_step + 1) * batch_size] 102 | X_input_B = X_input_B.reshape([-1, 5, 3]) 103 | X_input_I = X_test_I[vl_step * batch_size:(vl_step + 1) * batch_size] 104 | X_input_I = X_input_I.reshape([-1, I_nodes, 3]) 105 | y_input = y_test[vl_step * batch_size:(vl_step + 1) * batch_size] 106 | loss_value_vl, acc_vl, pred = sess.run([loss, accuracy, aver_pre], 107 | feed_dict={ 108 | J_in: X_input_J, 109 | P_in: X_input_P, 110 | B_in: X_input_B, 111 | I_in: X_input_I, 112 | J_bias_in: biases_J, 113 | P_bias_in: biases_P, 114 | B_bias_in: biases_B, 115 | I_bias_in: biases_I, 116 | lbl_in: y_test[vl_step * batch_size:(vl_step + 1) * batch_size], 117 | is_train: False, 118 | attn_drop: 0.0, ffd_drop: 0.0}) 119 | for i in range(y_input.shape[0]): 120 | for K in range(1, len(classes) + 1): 121 | if K not in rank_acc.keys(): 122 | rank_acc[K] = 0 123 | t = np.argpartition(pred[i], -K)[-K:] 124 | if np.argmax(y_input[i]) in t: 125 | rank_acc[K] += 1 126 | logits_all.extend(pred.tolist()) 127 | labels_all.extend(y_input.tolist()) 128 | vl_loss += loss_value_vl 129 | vl_acc += acc_vl 130 | vl_step += 1 131 | for K in rank_acc.keys(): 132 | rank_acc[K] /= (vl_step * batch_size) 133 | rank_acc[K] = round(rank_acc[K], 4) 134 | val_nAUC = process.cal_nAUC(scores=np.array(logits_all), labels=np.array(labels_all)) 135 | from sklearn.metrics import roc_curve, auc, confusion_matrix 136 | y_true = np.argmax(np.array(labels_all), axis=-1) 137 | y_pred = np.argmax(np.array(logits_all), axis=-1) 138 | print('\n### Re-ID Confusion Matrix: ') 139 | print(confusion_matrix(y_true, y_pred)) 140 | print('### Rank-N Accuracy: ') 141 | print(rank_acc) 142 | print('### Test loss:', round(vl_loss / vl_step, 4), '; Test accuracy:', round(vl_acc / vl_step, 4), 143 | '; Test nAUC:', round(val_nAUC, 4)) 144 | exit() 145 | 146 | if dataset == 'KS20': 147 | nb_nodes = 25 148 | I_nodes = 49 149 | batch_size = 64 150 | elif dataset == 'IAS' or dataset == 'KGBD': 151 | nb_nodes = 20 152 | I_nodes = 39 153 | elif dataset == 'CASIA_B': 154 | nb_nodes = 14 155 | I_nodes = 27 156 | batch_size = 128 157 | if split == 'A': 158 | batch_size = 128 159 | elif split == 'B': 160 | batch_size = 64 161 | if dataset == 'KGBD': 162 | batch_size = 256 163 | 164 | 165 | if model_dir == 'best': 166 | if dataset == 'IAS' and split == 'A': 167 | batch_size = 64 168 | model_dir = 'RN/IAS-A_59.4_86.7_formal' 169 | elif dataset == 'IAS' and split == 'B': 170 | model_dir = 'RN/IAS-B_69.8_90.4_formal' 171 | elif dataset == 'KS20': 172 | model_dir = 'RN/KS20_87.5_95.8_formal' 173 | elif dataset == 'KGBD': 174 | model_dir = 'RN/KGBD_99.5_99.6_formal' 175 | 176 | evaluate_reid(model_dir, dataset) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Python >=3.5](https://img.shields.io/badge/Python->=3.5-blue.svg) 2 | ![PyTorch >=1.6](https://img.shields.io/badge/Tensorflow->=1.10-yellow.svg) 3 | # SM-SGE: A Self-Supervised Multi-Scale Skeleton Graph Encoding Framework for Person Re-Identification 4 | By Haocong Rao, Shihao Xu, Xiping Hu, Jun Cheng, Bin Hu. In [ACMMM 2021](https://dl.acm.org/doi/10.1145/3474085.3475330). 5 | 6 | To the best of our knowledge, this is the *first* skeleton-based person re-identification work accepted by ACMMM. 7 | 8 | 9 | ## Introduction 10 | This is the official implementation of SM-SGE framework presented by "SM-SGE: A Self-Supervised Multi-Scale Skeleton Graph Encoding Framework for Person Re-Identification". The codes are used to reproduce experimental results in the [**paper**](https://dl.acm.org/doi/pdf/10.1145/3474085.3475330) ([Appendix](https://dl.acm.org/action/downloadSupplement?doi=10.1145%2F3474085.3475330&file=mfp0926aux.zip&download=true)). 11 | 12 | ![image](https://github.com/Kali-Hac/SM-SGE/blob/main/img/overview.png) 13 | 14 | Abstract: Person re-identification via 3D skeletons is an emerging topic with great potential in security-critical applications. Existing methods typically learn body and motion features from the body-joint trajectory, whereas they lack a systematic way to model body structure and underlying relations of body components beyond the scale of body joints. In this paper, we for the first time propose a Self-supervised Multi-scale Skeleton Graph Encoding (SM-SGE) framework that comprehensively models human body, component relations, and skeleton dynamics from unlabeled skeleton graphs of various scales to learn an effective skeleton representation for person Re-ID. Specifically, we first devise multi-scale skeleton graphs with coarse-to-fine human body partitions, which enables us to model body structure and skeleton dynamics at multiple levels. Second, to mine inherent correlations between body components in skeletal motion, we propose a multi-scale graph relation network to learn structural relations between adjacent body-component nodes and collaborative relations among nodes of different scales, so as to capture more discriminative skeleton graph features. Last, we propose a novel multi-scale skeleton reconstruction mechanism to enable our framework to encode skeleton dynamics and high-level semantics from unlabeled skeleton graphs, which encourages learning a discriminative skeleton representation for person Re-ID. Extensive experiments show that SM-SGE outperforms most state-of-the-art skeleton-based methods. 15 | We further demonstrate its effectiveness on 3D skeleton data estimated from large-scale RGB videos. 16 | 17 | ## Requirements 18 | - Python 3.5 19 | - Tensorflow 1.10.0 (GPU) 20 | 21 | ## Datasets and Models 22 | We provide two already pre-processed datasets (IAS-Lab, KGBD) with various sequence lengths on
23 | https://pan.baidu.com/s/1nuFH2EENyrMZbTnKGYssTw       password：  hyo7
24 | 25 | Note: The access to the Vislab Multi-view KS20 dataset is available upon request. If you have signed the license agreement and been granted the right to use it, please contact me and I will share the pre-processed KS20 data. 26 | 27 | All the best models reported in our paper can be acquired on
28 | https://pan.baidu.com/s/1AIn7Iyfn7B-w2Eh3ZfHIZA       password：  sd4v
29 | Please download the pre-processed datasets ``Datasets/`` and model files ``RN/`` into the current directory.
30 | 31 | We also provide the pre-trained multi-scale skeleton graph encoding models on
32 | https://pan.baidu.com/s/1ergZ7oJyo6fDkF2WGDNL9A       password：  vx48
33 | Please download the pre-trained graph encoding models into the directory ``Pre-Trained/``. 34 |
35 | 36 | The original datasets can be downloaded here: [IAS-Lab](http://robotics.dei.unipd.it/reid/index.php/downloads), [KGBD](https://www.researchgate.net/publication/275023745_Kinect_Gait_Biometry_Dataset_-_data_from_164_individuals_walking_in_front_of_a_X-Box_360_Kinect_Sensor), [KS20.](http://vislab.isr.ist.utl.pt/datasets/#ks20)
37 | 38 | 39 | 40 | ## Usage 41 | 42 | To (1) train the SM-SGE framework to obtain skeleton representations and (2) validate their effectiveness on the downstream task of person Re-ID on a specific dataset, simply run the following command: 43 | 44 | ```bash 45 | python SM-SGE.py --dataset KS20 46 | 47 | # Default options: --dataset KS20 --split '' --length 6 --fusion_lambda 1 --P 8 --sample_num 1 --pre_train 1 --save_flag 1 --gpu 0 48 | # --dataset [IAS, KGBD, KS20] 49 | # --split ['', 'A' (for IAS-A), 'B' (for IAS-B)] 50 | # --length [4, 6, 8, 10] 51 | # --pre_train ['1' (train from scratch), '0' (reload pre-trained model)] 52 | # --save_flag ['1' (save MLP models), '0' (do not save MLP models)] 53 | # --gpu [0, 1, ...] 54 | 55 | ``` 56 | Please see ```SM-SGE.py``` for more details. 57 | 58 | To print evaluation results (Re-ID Confusion Matrix / Rank-n Accuracy / Rank-1 Accuracy / nAUC) of the trained model, run: 59 | 60 | ```bash 61 | python evaluate.py --dataset KS20 --model_dir RN/xx/xx 62 | 63 | # Default options: --dataset KS20 --model_dir best --length 6 --gpu 0 64 | # --dataset [IAS, KGBD, KS20] 65 | # --model_dir [best (load the best models), RN/xx/xx (directory of model files, e.g., RN/best_models/KS20_87.5_95.8_formal)] 66 | ``` 67 | 68 | Please see ```evaluate.py``` for more details. 69 | 70 | ## Application to Model-Estimated Skeleton Data 71 | To extend our SM-SGE framework to a large RGB-based gait dataset (CASIA B), we exploit pose estimation methods to extract 3D skeletons from RGB videos of CASIA B as follows: 72 | - Step 1: Download [CASIA-B Dataset](http://www.cbsr.ia.ac.cn/english/Gait%20Databases.asp) 73 | - Step 2: Extract the 2D human body joints by using [OpenPose](https://github.com/CMU-Perceptual-Computing-Lab/openpose) 74 | - Step 3: Estimate the 3D human body joints by using [3DHumanPose](https://github.com/flyawaychase/3DHumanPose) 75 | 76 | We provide already pre-processed skeleton data of CASIA B for Cross-View Evaluation (**CVE**) (f=20/30/40) on       https://pan.baidu.com/s/1gDekBzf-3bBVdd0MGL0CvA       password：  x3e5
77 | Please download the pre-processed datasets into the directory ``Datasets/``.
78 | 79 | ## Usage 80 | To (1) train the SM-SGE framework to obtain skeleton representations and (2) validate their effectiveness on the downstream task of person Re-ID on CASIA B under **CVE** setup, simply run the following command: 81 | 82 | ```bash 83 | python SM-SGE.py --dataset CASIA_B --length 20 --view 0 84 | 85 | # Default options: --split '' --fusion_lambda 1 --P 8 --sample_num 1 --pre_train 1 --save_flag 1 --gpu 0 86 | # --length [20, 30, 40] 87 | # --view [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] 88 | # --pre_train ['1' (train from scratch), '0' (reload pre-trained model)] 89 | # --save_flag ['1' (save MLP models), '0' (do not save MLP models)] 90 | # --gpu [0, 1, ...] 91 | 92 | ``` 93 | Please see ```SM-SGE.py``` for more details.
94 | 95 | 96 | ## Results 97 | 98 | ![results](img/SM-SGE-results.png) 99 | 100 | ## Model Size & Computational Complexity 101 | | Methods | # Params | GFLOPs | 102 | |----------|----------|--------| 103 | | PoseGait | 26.80M | 364.8 | 104 | | [AGE](https://github.com/Kali-Hac/SGE-LA) | 7.15M | 37.4 | 105 | | [CAGE](https://github.com/Kali-Hac/Locality-Awareness-SGE) | 8.47M | 7.5 | 106 | | [SM-SGE (Ours)](https://github.com/Kali-Hac/SM-SGE) | 5.58M | 22.6 | 107 | 108 | ## Citation 109 | If you find this code useful for your research, please cite our paper 110 | ```bash 111 | @inproceedings{rao2021sm, 112 | title={SM-SGE: A Self-Supervised Multi-Scale Skeleton Graph Encoding Framework for Person Re-Identification}, 113 | author={Rao, Haocong and Hu, Xiping and Cheng, Jun and Hu, Bin}, 114 | booktitle={Proceedings of the 29th ACM International Conference on Multimedia}, 115 | pages={1812--1820}, 116 | year={2021} 117 | } 118 | 119 | ``` 120 | More awesome skeleton-based models are collected in our [Awesome-Skeleton-Based-Models](https://github.com/Kali-Hac/Awesome-Skeleton-Based-Models). 121 | 122 | # Acknowledgements 123 | 124 | Thanks to Veličković *et al.* for opening source of their excellent works [GAT](https://github.com/PetarV-/GAT). 125 | 126 | ## License 127 | 128 | SM-SGE is released under the MIT License. 129 | -------------------------------------------------------------------------------- /utils/process_cme.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pickle as pkl 3 | import networkx as nx 4 | import scipy.sparse as sp 5 | import copy 6 | from scipy.sparse.linalg.eigen.arpack import eigsh 7 | import sys 8 | from sklearn.preprocessing import label_binarize 9 | from sklearn.metrics import roc_curve, auc 10 | 11 | """ 12 | Generate training data for each dataset. 13 | """ 14 | 15 | def gen_train_data(dataset, split, time_step, nb_nodes, nhood, global_att, batch_size, reverse, PG_type): 16 | def get_data(dimension, fr): 17 | if reverse == '1': 18 | used_data = 'target' 19 | else: 20 | used_data = 'source' 21 | input_data = np.load( 22 | 'Datasets/' + frames_ps + dataset + '_train_npy_data/' + used_data + '_' + dimension + '_' + dataset + '_' + str( 23 | fr) + '.npy') 24 | input_data = input_data.reshape([-1, time_step, nb_nodes]) 25 | spine_pos = input_data[:, :, 0] 26 | spine_pos = np.expand_dims(spine_pos, -1) 27 | input_data = input_data - spine_pos 28 | if dataset == 'IAS': 29 | t_input_data = np.load( 30 | 'Datasets/' + frames_ps + dataset + '_test_npy_data/t_' + used_data + '_' + dimension + '_IAS-' + split + '_' + str( 31 | fr) + '.npy') 32 | else: 33 | t_input_data = np.load( 34 | 'Datasets/' + frames_ps + dataset + '_test_npy_data/' + PG_type + '/t_' + used_data + '_' + dimension + '_' + dataset + '_' + str( 35 | fr) + '.npy') 36 | t_input_data = t_input_data.reshape([-1, time_step, nb_nodes]) 37 | # Normalize 38 | t_spine_pos = t_input_data[:, :, 0] 39 | t_spine_pos = np.expand_dims(t_spine_pos, -1) 40 | t_input_data = t_input_data - t_spine_pos 41 | 42 | return input_data, t_input_data 43 | 44 | # For CASIA B CME 45 | frames_ps = dataset + '_match/' + str(time_step) + '/' 46 | input_data_x, t_input_data_x = get_data('x', fr=time_step) 47 | input_data_y, t_input_data_y = get_data('y', fr=time_step) 48 | input_data_z, t_input_data_z = get_data('z', fr=time_step) 49 | 50 | X_train = np.concatenate([input_data_x, input_data_y, input_data_z], axis=-1) 51 | X_test = np.concatenate([t_input_data_x, t_input_data_y, t_input_data_z], axis=-1) 52 | 53 | ids = np.load( 54 | 'Datasets/' + frames_ps + dataset + '_train_npy_data/ids_' + dataset + '_' + str(time_step) + '.npy') 55 | ids = ids.item() 56 | if dataset == 'IAS': 57 | t_ids = np.load( 58 | 'Datasets/' + frames_ps + 'IAS_test_npy_data/ids_IAS-' + split + '_' + str(time_step) + '.npy') 59 | else: 60 | t_ids = np.load( 61 | 'Datasets/' + frames_ps + dataset + '_test_npy_data/' + PG_type + '/ids_' + dataset + '_' + str(time_step) + '.npy') 62 | t_ids = t_ids.item() 63 | 64 | y_train = np.load( 65 | 'Datasets/' + frames_ps + dataset + '_train_npy_data/frame_id_' + dataset + '_' + str(time_step) + '.npy') 66 | if dataset == 'IAS': 67 | y_test = np.load( 68 | 'Datasets/' + frames_ps + 'IAS_test_npy_data/frame_id_IAS-' + split + '_' + str(time_step) + '.npy') 69 | else: 70 | y_test = np.load( 71 | 'Datasets/' + frames_ps + dataset + '_test_npy_data/' + PG_type + '/frame_id_' + dataset + '_' + str(time_step) + '.npy') 72 | 73 | X_train, y_train = class_samp_gen(X_train, y_train, ids, batch_size) 74 | # print(X_train.shape, y_train.shape) 75 | 76 | ids_keys = sorted(list(ids.keys())) 77 | classes = [i for i in ids_keys] 78 | y_train = label_binarize(y_train, classes=classes) 79 | t_ids_keys = sorted(list(t_ids.keys())) 80 | classes = [i for i in t_ids_keys] 81 | y_test = label_binarize(y_test, classes=classes) 82 | 83 | X_train_J = X_train.reshape([-1, time_step, 3, nb_nodes]) 84 | X_train_J = np.transpose(X_train_J, [0, 1, 3, 2]) 85 | X_train_P = reduce2part(X_train_J, nb_nodes) 86 | X_train_B = reduce2body(X_train_J, nb_nodes) 87 | # if dataset == 'KS20': 88 | # X_train_In = interpolation(X_train_J, nb_nodes) 89 | 90 | X_test_J = X_test.reshape([-1, time_step, 3, nb_nodes]) 91 | X_test_J = np.transpose(X_test_J, [0, 1, 3, 2]) 92 | X_test_P = reduce2part(X_test_J, nb_nodes) 93 | X_test_B = reduce2body(X_test_J, nb_nodes) 94 | # if dataset == 'KS20': 95 | # X_test_In = interpolation(X_test_J, nb_nodes) 96 | 97 | def generate_denser_adj(adj): 98 | adj_temp = copy.deepcopy(adj).tolist() 99 | node_num = len(adj_temp) 100 | new_adj = np.zeros([node_num * 2 - 1, node_num * 2 - 1]) 101 | cnt = node_num 102 | for i in range(node_num): 103 | for j in range(node_num): 104 | if adj_temp[i][j] == 1: 105 | new_adj[i, cnt] = new_adj[cnt, i] = new_adj[j, cnt] = new_adj[cnt, j] = 1 106 | adj_temp[i][j] = adj_temp[j][i] = 0 107 | # print(i, j, cnt) 108 | cnt += 1 109 | for i in range(node_num): 110 | for j in range(node_num): 111 | if adj_temp[i][j] == 1: 112 | assert new_adj[i, j] == new_adj[j, i] == 0 113 | if global_att: 114 | new_adj = np.ones([node_num*2-1, node_num*2-1]) 115 | # print(cnt) 116 | # print(new_adj) 117 | return new_adj 118 | import scipy.sparse 119 | if dataset == 'KS20': 120 | # Joint-Level adjacent matrix 121 | j_pair_1 = np.array([3, 2, 20, 8, 8, 9, 10, 9, 11, 10, 4, 20, 4, 5, 5, 6, 6, 7, 1, 20, 1, 0, 16, 0, 122 | 12, 0, 16, 17, 12, 13, 17, 18, 19, 18, 13, 14, 14, 15, 2, 20, 11, 23, 10, 24, 7, 21, 6, 22]) 123 | j_pair_2 = np.array([2, 3, 8, 20, 9, 8, 9, 10, 10, 11, 20, 4, 5, 4, 6, 5, 7, 6, 20, 1, 0, 1, 0, 16, 124 | 0, 12, 17, 16, 13, 12, 18, 17, 18, 19, 14, 13, 15, 14, 20, 2, 23, 11, 24, 10, 21, 7, 22, 6]) 125 | con_matrix = np.ones([48]) 126 | adj_joint = scipy.sparse.coo_matrix((con_matrix, (j_pair_1, j_pair_2)), shape=(nb_nodes, nb_nodes)).toarray() 127 | if global_att: 128 | adj_joint = np.ones([25, 25]) 129 | # hyper-Level (interpolation) adjacent matrix, NOT used, replaced by generate_denser_adj 130 | # i_pair_1 = np.array( 131 | # [3, 31, 31, 2, 2, 32, 32, 20, 20, 42, 42, 8, 8, 37, 37, 9, 9, 44, 44, 10, 10, 38, 38, 11, 11, 132 | # 39, 39, 23, 10, 40, 40, 24, 20, 41, 41, 4, 4, 33, 33, 5, 5, 43, 43, 6, 6, 34, 34, 7, 7, 35, 133 | # 35, 21, 6, 36, 36, 22, 134 | # 20, 45, 45, 1, 1, 30, 30, 0, 0, 46, 46, 12, 12, 26, 26, 13, 13, 48, 48, 14, 14, 27, 27, 15, 135 | # 0, 47, 47, 16, 16, 28, 28, 17, 17, 49, 49, 18, 18, 29, 29, 19]) 136 | # # miss 25 137 | # i_pair_1 = i_pair_1.tolist() 138 | # for i in range(len(i_pair_1)): 139 | # if i_pair_1[i] > 24: 140 | # i_pair_1[i] -= 1 141 | # i_pair_1 = np.array(i_pair_1) 142 | # i_pair_2 = np.array( 143 | # [31, 3, 2, 31, 32, 2, 20, 32, 42, 20, 8, 42, 37, 8, 9, 37, 44, 9, 10, 44, 38, 10, 11, 38, 39, 144 | # 11, 23, 39, 40, 10, 24, 40, 41, 20, 4, 41, 33, 4, 5, 33, 43, 5, 6, 43, 34, 6, 7, 34, 35, 7, 145 | # 21, 35, 36, 6, 22, 36, 146 | # 45, 20, 1, 45, 30, 1, 0, 30, 46, 0, 12, 46, 26, 12, 13, 26, 48, 13, 14, 48, 27, 14, 15, 27, 147 | # 47, 0, 16, 47, 28, 16, 17, 28, 49, 17, 18, 49, 29, 18, 19, 29]) 148 | # i_pair_2 = i_pair_2.tolist() 149 | # for i in range(len(i_pair_2)): 150 | # if i_pair_2[i] > 24: 151 | # i_pair_2[i] -= 1 152 | # i_pair_2 = np.array(i_pair_2) 153 | # # print(i_pair_1.shape, i_pair_2.shape) 154 | # con_matrix = np.ones([96]) 155 | # adj_interp = scipy.sparse.coo_matrix((con_matrix, (i_pair_1, i_pair_2)), 156 | # shape=(49, 49)).toarray() 157 | adj_interp = generate_denser_adj(adj_joint) 158 | elif dataset == 'CASIA_B': 159 | # Joint-Level adjacent matrix 160 | j_pair_1 = np.array([0, 1, 1, 2, 2, 3, 3, 4, 1, 5, 5, 6, 6, 7, 1, 8, 8, 9, 9, 10, 1, 11, 11, 12, 12, 13]) 161 | j_pair_2 = np.array([1, 0, 2, 1, 3, 2, 4, 3, 5, 1, 6, 5, 7, 6, 8, 1, 9, 8, 10, 9, 11, 1, 12, 11, 13, 12]) 162 | con_matrix = np.ones([26]) 163 | adj_joint = scipy.sparse.coo_matrix((con_matrix, (j_pair_1, j_pair_2)), shape=(nb_nodes, nb_nodes)).toarray() 164 | if global_att: 165 | adj_joint = np.ones([14, 14]) 166 | adj_interp = generate_denser_adj(adj_joint) 167 | else: 168 | # Joint-Level adjacent matrix 169 | j_pair_1 = np.array([3, 2, 2, 8, 8, 9, 10, 9, 11, 10, 4, 2, 4, 5, 5, 6, 6, 7, 1, 2, 1, 0, 16, 0, 170 | 12, 0, 16, 17, 12, 13, 17, 18, 19, 18, 13, 14, 14, 15]) 171 | j_pair_2 = np.array([2, 3, 8, 2, 9, 8, 9, 10, 10, 11, 2, 4, 5, 4, 6, 5, 7, 6, 2, 1, 0, 1, 0, 16, 172 | 0, 12, 17, 16, 13, 12, 18, 17, 18, 19, 14, 13, 15, 14]) 173 | con_matrix = np.ones([38]) 174 | adj_joint = scipy.sparse.coo_matrix((con_matrix, (j_pair_1, j_pair_2)), shape=(nb_nodes, nb_nodes)).toarray() 175 | if global_att: 176 | adj_joint = np.ones([20, 20]) 177 | adj_interp = generate_denser_adj(adj_joint) 178 | 179 | # Part-Level adjacent matrix 180 | p_pair_1 = np.array([5, 6, 5, 8, 6, 7, 8, 9, 5, 4, 4, 2, 4, 0, 2, 3, 1, 0]) 181 | p_pair_2 = np.array([6, 5, 8, 5, 7, 6, 9, 8, 4, 5, 2, 4, 0, 4, 3, 2, 0, 1]) 182 | con_matrix = np.ones([18]) 183 | adj_part = scipy.sparse.coo_matrix((con_matrix, (p_pair_1, p_pair_2)), shape=(10, 10)).toarray() 184 | 185 | # Body-Level adjacent matrix 186 | b_pair_1 = np.array([2, 3, 2, 4, 2, 1, 2, 0]) 187 | b_pair_2 = np.array([3, 2, 4, 2, 1, 2, 0, 2]) 188 | con_matrix = np.ones([8]) 189 | adj_body = scipy.sparse.coo_matrix((con_matrix, (b_pair_1, b_pair_2)), shape=(5, 5)).toarray() 190 | 191 | if global_att: 192 | adj_part = np.ones([10, 10]) 193 | adj_body = np.ones([5, 5]) 194 | 195 | # if dataset != 'KS20': 196 | X_train_In = generate_denser_graph_data(X_train_J, adj_joint, nb_nodes) 197 | X_test_In = generate_denser_graph_data(X_test_J, adj_joint, nb_nodes) 198 | 199 | if dataset == 'IAS': 200 | nb_classes = 11 201 | elif dataset == 'KGBD': 202 | nb_classes = 164 203 | elif dataset == 'BIWI': 204 | nb_classes = 28 205 | elif dataset == 'KS20': 206 | nb_classes = 20 207 | elif dataset == 'CASIA_B': 208 | nb_classes = 62 209 | 210 | adj_joint = adj_joint[np.newaxis] 211 | biases_joint = adj_to_bias(adj_joint, [nb_nodes], nhood=nhood) 212 | 213 | adj_part = adj_part[np.newaxis] 214 | biases_part = adj_to_bias(adj_part, [10], nhood=1) 215 | 216 | adj_body = adj_body[np.newaxis] 217 | biases_body = adj_to_bias(adj_body, [5], nhood=1) 218 | 219 | adj_interp = adj_interp[np.newaxis] 220 | biases_interp = adj_to_bias(adj_interp, [nb_nodes*2-1], nhood=1) 221 | 222 | return X_train_J, X_train_P, X_train_B, X_train_In, y_train, X_test_J, X_test_P, X_test_B, X_test_In, y_test, \ 223 | adj_joint, biases_joint, adj_part, biases_part, adj_body, biases_body, adj_interp, biases_interp, nb_classes 224 | 225 | """ 226 | Generate part-level skeleton graphs. 227 | """ 228 | 229 | def reduce2part(X, joint_num=20): 230 | if joint_num == 25: 231 | left_leg_up = [12, 13] 232 | left_leg_down = [14, 15] 233 | right_leg_up = [16, 17] 234 | right_leg_down = [18, 19] 235 | torso = [0, 1] 236 | head = [2, 3, 20] 237 | left_arm_up = [4, 5] 238 | left_arm_down = [6, 7, 21, 22] 239 | right_arm_up = [8, 9] 240 | right_arm_down = [10, 11, 23, 24] 241 | elif joint_num == 20: 242 | left_leg_up = [12, 13] 243 | left_leg_down = [14, 15] 244 | right_leg_up = [16, 17] 245 | right_leg_down = [18, 19] 246 | torso = [0, 1] 247 | head = [2, 3] 248 | left_arm_up = [4, 5] 249 | left_arm_down = [6, 7] 250 | right_arm_up = [8, 9] 251 | right_arm_down = [10, 11] 252 | elif joint_num == 14: 253 | left_leg_up = [11] 254 | left_leg_down = [12, 13] 255 | right_leg_up = [8] 256 | right_leg_down = [9, 10] 257 | torso = [1] 258 | head = [0] 259 | left_arm_up = [5] 260 | left_arm_down = [6, 7] 261 | right_arm_up = [2] 262 | right_arm_down = [3, 4] 263 | 264 | x_torso = np.mean(X[:, :, torso, :], axis=2) # [N * T, V=1] 265 | x_leftlegup = np.mean(X[:, :, left_leg_up, :], axis=2) 266 | x_leftlegdown = np.mean(X[:, :, left_leg_down, :], axis=2) 267 | x_rightlegup = np.mean(X[:, :, right_leg_up, :], axis=2) 268 | x_rightlegdown = np.mean(X[:, :, right_leg_down, :], axis=2) 269 | x_head = np.mean(X[:, :, head, :], axis=2) 270 | x_leftarmup = np.mean(X[:, :, left_arm_up, :], axis=2) 271 | x_leftarmdown = np.mean(X[:, :, left_arm_down, :], axis=2) 272 | x_rightarmup = np.mean(X[:, :, right_arm_up, :], axis=2) 273 | x_rightarmdown = np.mean(X[:, :, right_arm_down, :], axis=2) 274 | X_part = np.concatenate((x_leftlegup, x_leftlegdown, x_rightlegup, x_rightlegdown, x_torso, x_head, x_leftarmup, 275 | x_leftarmdown, x_rightarmup, x_rightarmdown), axis=-1) \ 276 | .reshape([X.shape[0], X.shape[1], 10, 3]) 277 | return X_part 278 | 279 | """ 280 | Generate body-level skeleton graphs. 281 | """ 282 | 283 | def reduce2body(X, joint_num=20): 284 | if joint_num == 25: 285 | left_leg = [12, 13, 14, 15] 286 | right_leg = [16, 17, 18, 19] 287 | torso = [0, 1, 2, 3, 20] 288 | left_arm = [4, 5, 6, 7, 21, 22] 289 | right_arm = [8, 9, 10, 11, 23, 24] 290 | elif joint_num == 20: 291 | left_leg = [12, 13, 14, 15] 292 | right_leg = [16, 17, 18, 19] 293 | torso = [0, 1, 2, 3] 294 | left_arm = [4, 5, 6, 7] 295 | right_arm = [8, 9, 10, 11] 296 | elif joint_num == 14: 297 | left_leg = [11, 12, 13] 298 | right_leg = [8, 9, 10] 299 | torso = [0, 1] 300 | left_arm = [5, 6, 7] 301 | right_arm = [2, 3, 4] 302 | 303 | x_torso = np.mean(X[:, :, torso, :], axis=2) # [N * T, V=1] 304 | x_leftleg = np.mean(X[:, :, left_leg, :], axis=2) 305 | x_rightleg = np.mean(X[:, :, right_leg, :], axis=2) 306 | x_leftarm = np.mean(X[:, :, left_arm, :], axis=2) 307 | x_rightarm = np.mean(X[:, :, right_arm, :], axis=2) 308 | X_body = np.concatenate((x_leftleg, x_rightleg, x_torso, x_leftarm, x_rightarm), axis=-1)\ 309 | .reshape([X.shape[0], X.shape[1], 5, 3]) 310 | return X_body 311 | 312 | """ 313 | Linear interpolation 314 | """ 315 | 316 | def interpolation(X, joint_num=20): 317 | if joint_num == 25: 318 | left_leg_up = [12, 13] 319 | left_leg_down = [14, 15] 320 | right_leg_up = [16, 17] 321 | right_leg_down = [18, 19] 322 | torso = [0, 1] 323 | head_1 = [2, 3] 324 | head_2 = [2, 20] 325 | left_arm_up = [4, 5] 326 | left_arm_down_1 = [6, 7] 327 | left_arm_down_2 = [7, 21] 328 | left_arm_down_3 = [6, 22] 329 | right_arm_up = [8, 9] 330 | right_arm_down_1 = [10, 11] 331 | right_arm_down_2 = [11, 23] 332 | right_arm_down_3 = [10, 24] 333 | shoulder_1 = [4, 20] 334 | shoulder_2 = [8, 20] 335 | elbow_1 = [5, 6] 336 | elbow_2 = [9, 10] 337 | spine_mm = [20, 1] 338 | hip_1 = [0, 12] 339 | hip_2 = [0, 16] 340 | knee_1 = [13, 14] 341 | knee_2 = [17, 18] 342 | x_torso = np.mean(X[:, :, torso, :], axis=2) # [N * T, V=1] 343 | x_leftlegup = np.mean(X[:, :, left_leg_up, :], axis=2) 344 | x_leftlegdown = np.mean(X[:, :, left_leg_down, :], axis=2) 345 | x_rightlegup = np.mean(X[:, :, right_leg_up, :], axis=2) 346 | x_rightlegdown = np.mean(X[:, :, right_leg_down, :], axis=2) 347 | x_head_1 = np.mean(X[:, :, head_1, :], axis=2) 348 | x_head_2 = np.mean(X[:, :, head_2, :], axis=2) 349 | x_leftarmup = np.mean(X[:, :, left_arm_up, :], axis=2) 350 | x_leftarmdown_1 = np.mean(X[:, :, left_arm_down_1, :], axis=2) 351 | x_leftarmdown_2 = np.mean(X[:, :, left_arm_down_2, :], axis=2) 352 | x_leftarmdown_3 = np.mean(X[:, :, left_arm_down_3, :], axis=2) 353 | x_rightarmup = np.mean(X[:, :, right_arm_up, :], axis=2) 354 | x_rightarmdown_1 = np.mean(X[:, :, right_arm_down_1, :], axis=2) 355 | x_rightarmdown_2 = np.mean(X[:, :, right_arm_down_2, :], axis=2) 356 | x_rightarmdown_3 = np.mean(X[:, :, right_arm_down_3, :], axis=2) 357 | shoulder_1 = np.mean(X[:, :, shoulder_1, :], axis=2) 358 | shoulder_2 = np.mean(X[:, :, shoulder_2, :], axis=2) 359 | elbow_1 = np.mean(X[:, :, elbow_1, :], axis=2) 360 | elbow_2 = np.mean(X[:, :, elbow_2, :], axis=2) 361 | spine_mm = np.mean(X[:, :, spine_mm, :], axis=2) 362 | hip_1 = np.mean(X[:, :, hip_1, :], axis=2) 363 | hip_2 = np.mean(X[:, :, hip_2, :], axis=2) 364 | knee_1 = np.mean(X[:, :, knee_1, :], axis=2) 365 | knee_2 = np.mean(X[:, :, knee_2, :], axis=2) 366 | X_part = np.concatenate((x_leftlegup, x_leftlegdown, x_rightlegup, 367 | x_rightlegdown, x_torso, x_head_1, x_head_2, x_leftarmup, 368 | x_leftarmdown_1, x_leftarmdown_2, x_leftarmdown_3, 369 | x_rightarmup, x_rightarmdown_1, x_rightarmdown_2, x_rightarmdown_3, 370 | shoulder_1, shoulder_2, elbow_1, elbow_2, spine_mm, 371 | hip_1, hip_2, knee_1, knee_2), axis=-1) \ 372 | .reshape([X.shape[0], X.shape[1], 24, 3]) 373 | # 25+24 374 | X_interp = np.concatenate((X, X_part), axis=-2) 375 | # Changed 376 | elif joint_num == 20: 377 | # left_leg_up = [12, 13] 378 | # left_leg_down = [14, 15] 379 | # right_leg_up = [16, 17] 380 | # right_leg_down = [18, 19] 381 | # torso = [0, 1] 382 | # head = [2, 3] 383 | # left_arm_up = [4, 5] 384 | # left_arm_down = [6, 7] 385 | # right_arm_up = [8, 9] 386 | # right_arm_down = [10, 11] 387 | # 388 | left_leg_up = [12, 13] 389 | left_leg_down = [14, 15] 390 | right_leg_up = [16, 17] 391 | right_leg_down = [18, 19] 392 | torso = [0, 1] 393 | head_1 = [2, 3] 394 | # head_2 = [2, 20] 395 | left_arm_up = [4, 5] 396 | left_arm_down_1 = [6, 7] 397 | # left_arm_down_2 = [7, 21] 398 | # left_arm_down_3 = [6, 22] 399 | right_arm_up = [8, 9] 400 | right_arm_down_1 = [10, 11] 401 | # right_arm_down_2 = [11, 23] 402 | # right_arm_down_3 = [10, 24] 403 | # shoulder_1 = [4, 20] 404 | # shoulder_2 = [8, 20] 405 | shoulder_1 = [4, 2] 406 | shoulder_2 = [8, 2] 407 | elbow_1 = [5, 6] 408 | elbow_2 = [9, 10] 409 | # spine_mm = [20, 1] 410 | spine_mm = [2, 1] 411 | hip_1 = [0, 12] 412 | hip_2 = [0, 16] 413 | knee_1 = [13, 14] 414 | knee_2 = [17, 18] 415 | 416 | x_torso = np.mean(X[:, :, torso, :], axis=2) # [N * T, V=1] 417 | x_leftlegup = np.mean(X[:, :, left_leg_up, :], axis=2) 418 | x_leftlegdown = np.mean(X[:, :, left_leg_down, :], axis=2) 419 | x_rightlegup = np.mean(X[:, :, right_leg_up, :], axis=2) 420 | x_rightlegdown = np.mean(X[:, :, right_leg_down, :], axis=2) 421 | x_head_1 = np.mean(X[:, :, head_1, :], axis=2) 422 | # x_head_2 = np.mean(X[:, :, head_2, :], axis=2) 423 | x_leftarmup = np.mean(X[:, :, left_arm_up, :], axis=2) 424 | x_leftarmdown_1 = np.mean(X[:, :, left_arm_down_1, :], axis=2) 425 | # x_leftarmdown_2 = np.mean(X[:, :, left_arm_down_2, :], axis=2) 426 | # x_leftarmdown_3 = np.mean(X[:, :, left_arm_down_3, :], axis=2) 427 | x_rightarmup = np.mean(X[:, :, right_arm_up, :], axis=2) 428 | x_rightarmdown_1 = np.mean(X[:, :, right_arm_down_1, :], axis=2) 429 | # x_rightarmdown_2 = np.mean(X[:, :, right_arm_down_2, :], axis=2) 430 | # x_rightarmdown_3 = np.mean(X[:, :, right_arm_down_3, :], axis=2) 431 | shoulder_1 = np.mean(X[:, :, shoulder_1, :], axis=2) 432 | shoulder_2 = np.mean(X[:, :, shoulder_2, :], axis=2) 433 | elbow_1 = np.mean(X[:, :, elbow_1, :], axis=2) 434 | elbow_2 = np.mean(X[:, :, elbow_2, :], axis=2) 435 | spine_mm = np.mean(X[:, :, spine_mm, :], axis=2) 436 | hip_1 = np.mean(X[:, :, hip_1, :], axis=2) 437 | hip_2 = np.mean(X[:, :, hip_2, :], axis=2) 438 | knee_1 = np.mean(X[:, :, knee_1, :], axis=2) 439 | knee_2 = np.mean(X[:, :, knee_2, :], axis=2) 440 | X_part = np.concatenate((x_leftlegup, x_leftlegdown, x_rightlegup, 441 | x_rightlegdown, x_torso, x_head_1, x_leftarmup, 442 | x_leftarmdown_1, 443 | x_rightarmup, x_rightarmdown_1, 444 | shoulder_1, shoulder_2, elbow_1, elbow_2, spine_mm, 445 | hip_1, hip_2, knee_1, knee_2), axis=-1) \ 446 | .reshape([X.shape[0], X.shape[1], 19, 3]) 447 | # 25+24 448 | X_interp = np.concatenate((X, X_part), axis=-2) 449 | return X_interp 450 | 451 | def generate_denser_graph_data(X, adj, joint_num=20): 452 | adj_temp = copy.deepcopy(adj) 453 | adj_temp = adj_temp.tolist() 454 | node_num = len(adj_temp) 455 | cnt = node_num 456 | for i in range(node_num): 457 | for j in range(node_num): 458 | if adj_temp[i][j] == 1: 459 | adj_temp[i][j] = adj_temp[j][i] = 0 460 | new_node = np.mean(X[:, :, [i, j], :], axis=2) 461 | # print(new_node.shape) 462 | if cnt == node_num: 463 | X_interp = new_node 464 | else: 465 | X_interp = np.concatenate((X_interp, new_node), axis=-1) 466 | # print(X_interp.shape) 467 | # print(i, j) 468 | # print(i, j, cnt) 469 | cnt += 1 470 | # print(X_interp.shape) 471 | X_interp = np.reshape(X_interp, [X.shape[0], X.shape[1], node_num-1, 3]) 472 | X_interp = np.concatenate((X, X_interp), axis=-2) 473 | return X_interp 474 | 475 | """ 476 | Calculate normalized area under curves. 477 | """ 478 | def cal_nAUC(scores, labels): 479 | scores = np.array(scores) 480 | labels = np.array(labels) 481 | # Compute micro-average ROC curve and ROC area 482 | fpr, tpr, thresholds = roc_curve(labels.ravel(), scores.ravel()) 483 | roc_auc = auc(fpr, tpr) 484 | return roc_auc 485 | 486 | """ 487 | Generate training data with evenly distributed classes. 488 | """ 489 | def class_samp_gen(X, y, ids_, batch_size): 490 | class_num = len(ids_.keys()) 491 | ids_ = sorted(ids_.items(), key=lambda item: item[0]) 492 | cnt = 0 493 | all_batch_X = [] 494 | all_batch_y = [] 495 | total = y.shape[0] 496 | batch_num = total // batch_size * 2 497 | batch_num = total // batch_size * 2 498 | class_in_bacth = class_num 499 | batch_per_class = batch_size // class_in_bacth 500 | class_cnt = class_in_bacth 501 | # print(total, batch_num, batch_per_class) 502 | for i in range(batch_num): 503 | batch_X = [] 504 | batch_y = [] 505 | for k, v in ids_[class_cnt-class_in_bacth:class_cnt]: 506 | # print(k, len(v)) 507 | # cnt += len(v) 508 | if len(v[batch_per_class*i:batch_per_class*(i+1)]) < batch_per_class: 509 | rand_ind = np.random.choice(len(v), batch_per_class) 510 | v_array = np.array(v) 511 | samp_per_class = v_array[rand_ind].tolist() 512 | batch_X.extend(samp_per_class) 513 | else: 514 | batch_X.extend(v[batch_per_class*i:batch_per_class*(i+1)]) 515 | batch_y.extend(batch_per_class * [k]) 516 | if class_cnt + class_in_bacth > class_num and class_cnt <= class_num: 517 | class_cnt = class_num 518 | else: 519 | class_cnt = class_cnt + class_in_bacth 520 | all_batch_X.extend(batch_X) 521 | all_batch_y.extend(batch_y) 522 | # print(len(all_batch_X), len(all_batch_y)) 523 | X_train = X[all_batch_X] 524 | y_train = np.array(all_batch_y) 525 | return X_train, y_train 526 | 527 | """ 528 | Prepare adjacency matrix by expanding up to a given neighbourhood. 529 | This will insert loops on every node. 530 | Finally, the matrix is converted to bias vectors. 531 | Expected shape: [graph, nodes, nodes] 532 | """ 533 | 534 | 535 | def adj_to_bias(adj, sizes, nhood=1): 536 | nb_graphs = adj.shape[0] 537 | mt = np.empty(adj.shape) 538 | for g in range(nb_graphs): 539 | mt[g] = np.eye(adj.shape[1]) 540 | for _ in range(nhood): 541 | mt[g] = np.matmul(mt[g], (adj[g] + np.eye(adj.shape[1]))) 542 | for i in range(sizes[g]): 543 | for j in range(sizes[g]): 544 | if mt[g][i][j] > 0.0: 545 | mt[g][i][j] = 1.0 546 | return -1e9 * (1.0 - mt) 547 | 548 | 549 | ############################################### 550 | # This section of code adapted from tkipf/gcn # 551 | ############################################### 552 | 553 | def parse_index_file(filename): 554 | """Parse index file.""" 555 | index = [] 556 | for line in open(filename): 557 | index.append(int(line.strip())) 558 | return index 559 | 560 | 561 | def sample_mask(idx, l): 562 | """Create mask.""" 563 | mask = np.zeros(l) 564 | mask[idx] = 1 565 | return np.array(mask, dtype=np.bool) 566 | 567 | 568 | def load_data(dataset_str): # {'pubmed', 'citeseer', 'cora'} 569 | """Load data.""" 570 | names = ['x', 'y', 'tx', 'ty', 'allx', 'ally', 'graph'] 571 | objects = [] 572 | for i in range(len(names)): 573 | with open("data/ind.{}.{}".format(dataset_str, names[i]), 'rb') as f: 574 | if sys.version_info > (3, 0): 575 | objects.append(pkl.load(f, encoding='latin1')) 576 | else: 577 | objects.append(pkl.load(f)) 578 | 579 | x, y, tx, ty, allx, ally, graph = tuple(objects) 580 | test_idx_reorder = parse_index_file("data/ind.{}.test.index".format(dataset_str)) 581 | test_idx_range = np.sort(test_idx_reorder) 582 | 583 | if dataset_str == 'citeseer': 584 | # Fix citeseer dataset (there are some isolated nodes in the graph) 585 | # Find isolated nodes, add them as zero-vecs into the right position 586 | test_idx_range_full = range(min(test_idx_reorder), max(test_idx_reorder) + 1) 587 | tx_extended = sp.lil_matrix((len(test_idx_range_full), x.shape[1])) 588 | tx_extended[test_idx_range - min(test_idx_range), :] = tx 589 | tx = tx_extended 590 | ty_extended = np.zeros((len(test_idx_range_full), y.shape[1])) 591 | ty_extended[test_idx_range - min(test_idx_range), :] = ty 592 | ty = ty_extended 593 | 594 | features = sp.vstack((allx, tx)).tolil() 595 | features[test_idx_reorder, :] = features[test_idx_range, :] 596 | adj = nx.adjacency_matrix(nx.from_dict_of_lists(graph)) 597 | 598 | labels = np.vstack((ally, ty)) 599 | labels[test_idx_reorder, :] = labels[test_idx_range, :] 600 | 601 | idx_test = test_idx_range.tolist() 602 | idx_train = range(len(y)) 603 | idx_val = range(len(y), len(y) + 500) 604 | 605 | train_mask = sample_mask(idx_train, labels.shape[0]) 606 | val_mask = sample_mask(idx_val, labels.shape[0]) 607 | test_mask = sample_mask(idx_test, labels.shape[0]) 608 | 609 | y_train = np.zeros(labels.shape) 610 | y_val = np.zeros(labels.shape) 611 | y_test = np.zeros(labels.shape) 612 | y_train[train_mask, :] = labels[train_mask, :] 613 | y_val[val_mask, :] = labels[val_mask, :] 614 | y_test[test_mask, :] = labels[test_mask, :] 615 | 616 | # print(adj.shape) 617 | # print(features.shape) 618 | 619 | return adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask 620 | 621 | 622 | def load_random_data(size): 623 | adj = sp.random(size, size, density=0.002) # density similar to cora 624 | features = sp.random(size, 1000, density=0.015) 625 | int_labels = np.random.randint(7, size=(size)) 626 | labels = np.zeros((size, 7)) # Nx7 627 | labels[np.arange(size), int_labels] = 1 628 | 629 | train_mask = np.zeros((size,)).astype(bool) 630 | train_mask[np.arange(size)[0:int(size / 2)]] = 1 631 | 632 | val_mask = np.zeros((size,)).astype(bool) 633 | val_mask[np.arange(size)[int(size / 2):]] = 1 634 | 635 | test_mask = np.zeros((size,)).astype(bool) 636 | test_mask[np.arange(size)[int(size / 2):]] = 1 637 | 638 | y_train = np.zeros(labels.shape) 639 | y_val = np.zeros(labels.shape) 640 | y_test = np.zeros(labels.shape) 641 | y_train[train_mask, :] = labels[train_mask, :] 642 | y_val[val_mask, :] = labels[val_mask, :] 643 | y_test[test_mask, :] = labels[test_mask, :] 644 | 645 | # sparse NxN, sparse NxF, norm NxC, ..., norm Nx1, ... 646 | return adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask 647 | 648 | 649 | def sparse_to_tuple(sparse_mx): 650 | """Convert sparse matrix to tuple representation.""" 651 | 652 | def to_tuple(mx): 653 | if not sp.isspmatrix_coo(mx): 654 | mx = mx.tocoo() 655 | coords = np.vstack((mx.row, mx.col)).transpose() 656 | values = mx.data 657 | shape = mx.shape 658 | return coords, values, shape 659 | 660 | if isinstance(sparse_mx, list): 661 | for i in range(len(sparse_mx)): 662 | sparse_mx[i] = to_tuple(sparse_mx[i]) 663 | else: 664 | sparse_mx = to_tuple(sparse_mx) 665 | 666 | return sparse_mx 667 | 668 | 669 | def standardize_data(f, train_mask): 670 | """Standardize feature matrix and convert to tuple representation""" 671 | # standardize data 672 | f = f.todense() 673 | mu = f[train_mask == True, :].mean(axis=0) 674 | sigma = f[train_mask == True, :].std(axis=0) 675 | f = f[:, np.squeeze(np.array(sigma > 0))] 676 | mu = f[train_mask == True, :].mean(axis=0) 677 | sigma = f[train_mask == True, :].std(axis=0) 678 | f = (f - mu) / sigma 679 | return f 680 | 681 | 682 | def preprocess_features(features): 683 | """Row-normalize feature matrix and convert to tuple representation""" 684 | rowsum = np.array(features.sum(1)) 685 | r_inv = np.power(rowsum, -1).flatten() 686 | r_inv[np.isinf(r_inv)] = 0. 687 | r_mat_inv = sp.diags(r_inv) 688 | features = r_mat_inv.dot(features) 689 | return features.todense(), sparse_to_tuple(features) 690 | 691 | 692 | def normalize_adj(adj): 693 | """Symmetrically normalize adjacency matrix.""" 694 | adj = sp.coo_matrix(adj) 695 | rowsum = np.array(adj.sum(1)) 696 | d_inv_sqrt = np.power(rowsum, -0.5).flatten() 697 | d_inv_sqrt[np.isinf(d_inv_sqrt)] = 0. 698 | d_mat_inv_sqrt = sp.diags(d_inv_sqrt) 699 | return adj.dot(d_mat_inv_sqrt).transpose().dot(d_mat_inv_sqrt).tocoo() 700 | 701 | 702 | def preprocess_adj(adj): 703 | """Preprocessing of adjacency matrix for simple GCN model and conversion to tuple representation.""" 704 | adj_normalized = normalize_adj(adj + sp.eye(adj.shape[0])) 705 | return sparse_to_tuple(adj_normalized) 706 | 707 | 708 | def preprocess_adj_bias(adj): 709 | num_nodes = adj.shape[0] 710 | adj = adj + sp.eye(num_nodes) # self-loop 711 | adj[adj > 0.0] = 1.0 712 | if not sp.isspmatrix_coo(adj): 713 | adj = adj.tocoo() 714 | adj = adj.astype(np.float32) 715 | indices = np.vstack( 716 | (adj.col, adj.row)).transpose() # This is where I made a mistake, I used (adj.row, adj.col) instead 717 | # return tf.SparseTensor(indices=indices, values=adj.data, dense_shape=adj.shape) 718 | return indices, adj.data, adj.shape 719 | -------------------------------------------------------------------------------- /utils/process_i.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pickle as pkl 3 | import networkx as nx 4 | import scipy.sparse as sp 5 | import copy 6 | from scipy.sparse.linalg.eigen.arpack import eigsh 7 | import sys 8 | from sklearn.preprocessing import label_binarize 9 | from sklearn.metrics import roc_curve, auc 10 | 11 | """ 12 | Generate training data for each dataset. 13 | """ 14 | 15 | def gen_train_data(dataset, split, time_step, nb_nodes, nhood, global_att, batch_size, view, reverse): 16 | def get_data(dimension, fr): 17 | if reverse == '1': 18 | used_data = 'target' 19 | else: 20 | used_data = 'source' 21 | input_data = np.load( 22 | 'Datasets/' + frames_ps + dataset + '_train_npy_data/' + used_data + '_' + dimension + '_' + dataset + '_' + str( 23 | fr) + '.npy') 24 | input_data = input_data.reshape([-1, time_step, nb_nodes]) 25 | spine_pos = input_data[:, :, 0] 26 | spine_pos = np.expand_dims(spine_pos, -1) 27 | input_data = input_data - spine_pos 28 | if dataset == 'IAS': 29 | t_input_data = np.load( 30 | 'Datasets/' + frames_ps + dataset + '_test_npy_data/t_' + used_data + '_' + dimension + '_IAS-' + split + '_' + str( 31 | fr) + '.npy') 32 | else: 33 | t_input_data = np.load( 34 | 'Datasets/' + frames_ps + dataset + '_test_npy_data/t_' + used_data + '_' + dimension + '_' + dataset + '_' + str( 35 | fr) + '.npy') 36 | t_input_data = t_input_data.reshape([-1, time_step, nb_nodes]) 37 | # Normalize 38 | t_spine_pos = t_input_data[:, :, 0] 39 | t_spine_pos = np.expand_dims(t_spine_pos, -1) 40 | t_input_data = t_input_data - t_spine_pos 41 | 42 | return input_data, t_input_data 43 | if view == '': 44 | frames_ps = dataset + '/' + str(time_step) + '/' 45 | else: 46 | frames_ps = dataset + '/' + str(time_step) + '/view_' + str(view) + '/' 47 | input_data_x, t_input_data_x = get_data('x', fr=time_step) 48 | input_data_y, t_input_data_y = get_data('y', fr=time_step) 49 | input_data_z, t_input_data_z = get_data('z', fr=time_step) 50 | 51 | X_train = np.concatenate([input_data_x, input_data_y, input_data_z], axis=-1) 52 | X_test = np.concatenate([t_input_data_x, t_input_data_y, t_input_data_z], axis=-1) 53 | 54 | ids = np.load( 55 | 'Datasets/' + frames_ps + dataset + '_train_npy_data/ids_' + dataset + '_' + str(time_step) + '.npy') 56 | ids = ids.item() 57 | if dataset == 'IAS': 58 | t_ids = np.load( 59 | 'Datasets/' + frames_ps + 'IAS_test_npy_data/ids_IAS-' + split + '_' + str(time_step) + '.npy') 60 | else: 61 | t_ids = np.load( 62 | 'Datasets/' + frames_ps + dataset + '_test_npy_data/ids_' + dataset + '_' + str(time_step) + '.npy') 63 | t_ids = t_ids.item() 64 | 65 | y_train = np.load( 66 | 'Datasets/' + frames_ps + dataset + '_train_npy_data/frame_id_' + dataset + '_' + str(time_step) + '.npy') 67 | if dataset == 'IAS': 68 | y_test = np.load( 69 | 'Datasets/' + frames_ps + 'IAS_test_npy_data/frame_id_IAS-' + split + '_' + str(time_step) + '.npy') 70 | else: 71 | y_test = np.load( 72 | 'Datasets/' + frames_ps + dataset + '_test_npy_data/frame_id_' + dataset + '_' + str(time_step) + '.npy') 73 | 74 | # if dataset != 'CASIA_B': 75 | X_train, y_train = class_samp_gen(X_train, y_train, ids, batch_size) 76 | # print(X_train.shape, y_train.shape) 77 | 78 | ids_keys = sorted(list(ids.keys())) 79 | classes = [i for i in ids_keys] 80 | y_train = label_binarize(y_train, classes=classes) 81 | t_ids_keys = sorted(list(t_ids.keys())) 82 | classes = [i for i in t_ids_keys] 83 | y_test = label_binarize(y_test, classes=classes) 84 | 85 | X_train_J = X_train.reshape([-1, time_step, 3, nb_nodes]) 86 | X_train_J = np.transpose(X_train_J, [0, 1, 3, 2]) 87 | X_train_P = reduce2part(X_train_J, nb_nodes) 88 | X_train_B = reduce2body(X_train_J, nb_nodes) 89 | # if dataset == 'KS20': 90 | # X_train_In = interpolation(X_train_J, nb_nodes) 91 | 92 | X_test_J = X_test.reshape([-1, time_step, 3, nb_nodes]) 93 | X_test_J = np.transpose(X_test_J, [0, 1, 3, 2]) 94 | X_test_P = reduce2part(X_test_J, nb_nodes) 95 | X_test_B = reduce2body(X_test_J, nb_nodes) 96 | # if dataset == 'KS20': 97 | # X_test_In = interpolation(X_test_J, nb_nodes) 98 | 99 | def generate_denser_adj(adj): 100 | adj_temp = copy.deepcopy(adj).tolist() 101 | node_num = len(adj_temp) 102 | # if global_att: 103 | # new_adj = np.ones([node_num * 2 - 1, node_num * 2 - 1]) 104 | # return new_adj 105 | new_adj = np.zeros([node_num * 2 - 1, node_num * 2 - 1]) 106 | cnt = node_num 107 | for i in range(node_num): 108 | for j in range(node_num): 109 | if adj_temp[i][j] == 1: 110 | new_adj[i, cnt] = new_adj[cnt, i] = new_adj[j, cnt] = new_adj[cnt, j] = 1 111 | adj_temp[i][j] = adj_temp[j][i] = 0 112 | # print(i, j, cnt) 113 | cnt += 1 114 | for i in range(node_num): 115 | for j in range(node_num): 116 | if adj_temp[i][j] == 1: 117 | assert new_adj[i, j] == new_adj[j, i] == 0 118 | # print(cnt) 119 | # np.save('interp_graph_adj.npy', new_adj) 120 | return new_adj 121 | import scipy.sparse 122 | if dataset == 'KS20': 123 | # Joint-Level adjacent matrix 124 | j_pair_1 = np.array([3, 2, 20, 8, 8, 9, 10, 9, 11, 10, 4, 20, 4, 5, 5, 6, 6, 7, 1, 20, 1, 0, 16, 0, 125 | 12, 0, 16, 17, 12, 13, 17, 18, 19, 18, 13, 14, 14, 15, 2, 20, 11, 23, 10, 24, 7, 21, 6, 22]) 126 | j_pair_2 = np.array([2, 3, 8, 20, 9, 8, 9, 10, 10, 11, 20, 4, 5, 4, 6, 5, 7, 6, 20, 1, 0, 1, 0, 16, 127 | 0, 12, 17, 16, 13, 12, 18, 17, 18, 19, 14, 13, 15, 14, 20, 2, 23, 11, 24, 10, 21, 7, 22, 6]) 128 | con_matrix = np.ones([48]) 129 | adj_joint = scipy.sparse.coo_matrix((con_matrix, (j_pair_1, j_pair_2)), shape=(nb_nodes, nb_nodes)).toarray() 130 | # if global_att: 131 | # adj_joint = np.ones([25, 25]) 132 | # hyper-Level (interpolation) adjacent matrix, NOT used, replaced by generate_denser_adj 133 | # i_pair_1 = np.array( 134 | # [3, 31, 31, 2, 2, 32, 32, 20, 20, 42, 42, 8, 8, 37, 37, 9, 9, 44, 44, 10, 10, 38, 38, 11, 11, 135 | # 39, 39, 23, 10, 40, 40, 24, 20, 41, 41, 4, 4, 33, 33, 5, 5, 43, 43, 6, 6, 34, 34, 7, 7, 35, 136 | # 35, 21, 6, 36, 36, 22, 137 | # 20, 45, 45, 1, 1, 30, 30, 0, 0, 46, 46, 12, 12, 26, 26, 13, 13, 48, 48, 14, 14, 27, 27, 15, 138 | # 0, 47, 47, 16, 16, 28, 28, 17, 17, 49, 49, 18, 18, 29, 29, 19]) 139 | # # miss 25 140 | # i_pair_1 = i_pair_1.tolist() 141 | # for i in range(len(i_pair_1)): 142 | # if i_pair_1[i] > 24: 143 | # i_pair_1[i] -= 1 144 | # i_pair_1 = np.array(i_pair_1) 145 | # i_pair_2 = np.array( 146 | # [31, 3, 2, 31, 32, 2, 20, 32, 42, 20, 8, 42, 37, 8, 9, 37, 44, 9, 10, 44, 38, 10, 11, 38, 39, 147 | # 11, 23, 39, 40, 10, 24, 40, 41, 20, 4, 41, 33, 4, 5, 33, 43, 5, 6, 43, 34, 6, 7, 34, 35, 7, 148 | # 21, 35, 36, 6, 22, 36, 149 | # 45, 20, 1, 45, 30, 1, 0, 30, 46, 0, 12, 46, 26, 12, 13, 26, 48, 13, 14, 48, 27, 14, 15, 27, 150 | # 47, 0, 16, 47, 28, 16, 17, 28, 49, 17, 18, 49, 29, 18, 19, 29]) 151 | # i_pair_2 = i_pair_2.tolist() 152 | # for i in range(len(i_pair_2)): 153 | # if i_pair_2[i] > 24: 154 | # i_pair_2[i] -= 1 155 | # i_pair_2 = np.array(i_pair_2) 156 | # # print(i_pair_1.shape, i_pair_2.shape) 157 | # con_matrix = np.ones([96]) 158 | # adj_interp = scipy.sparse.coo_matrix((con_matrix, (i_pair_1, i_pair_2)), 159 | # shape=(49, 49)).toarray() 160 | adj_interp = generate_denser_adj(adj_joint) 161 | elif dataset == 'CASIA_B': 162 | # Joint-Level adjacent matrix 163 | j_pair_1 = np.array([0, 1, 1, 2, 2, 3, 3, 4, 1, 5, 5, 6, 6, 7, 1, 8, 8, 9, 9, 10, 1, 11, 11, 12, 12, 13]) 164 | j_pair_2 = np.array([1, 0, 2, 1, 3, 2, 4, 3, 5, 1, 6, 5, 7, 6, 8, 1, 9, 8, 10, 9, 11, 1, 12, 11, 13, 12]) 165 | con_matrix = np.ones([26]) 166 | adj_joint = scipy.sparse.coo_matrix((con_matrix, (j_pair_1, j_pair_2)), shape=(nb_nodes, nb_nodes)).toarray() 167 | # if global_att: 168 | # adj_joint = np.ones([14, 14]) 169 | adj_interp = generate_denser_adj(adj_joint) 170 | else: 171 | # Joint-Level adjacent matrix 172 | j_pair_1 = np.array([3, 2, 2, 8, 8, 9, 10, 9, 11, 10, 4, 2, 4, 5, 5, 6, 6, 7, 1, 2, 1, 0, 16, 0, 173 | 12, 0, 16, 17, 12, 13, 17, 18, 19, 18, 13, 14, 14, 15]) 174 | j_pair_2 = np.array([2, 3, 8, 2, 9, 8, 9, 10, 10, 11, 2, 4, 5, 4, 6, 5, 7, 6, 2, 1, 0, 1, 0, 16, 175 | 0, 12, 17, 16, 13, 12, 18, 17, 18, 19, 14, 13, 15, 14]) 176 | con_matrix = np.ones([38]) 177 | adj_joint = scipy.sparse.coo_matrix((con_matrix, (j_pair_1, j_pair_2)), shape=(nb_nodes, nb_nodes)).toarray() 178 | # if global_att: 179 | # adj_joint = np.ones([20, 20]) 180 | adj_interp = generate_denser_adj(adj_joint) 181 | 182 | # Part-Level adjacent matrix 183 | p_pair_1 = np.array([5, 6, 5, 8, 6, 7, 8, 9, 5, 4, 4, 2, 4, 0, 2, 3, 1, 0]) 184 | p_pair_2 = np.array([6, 5, 8, 5, 7, 6, 9, 8, 4, 5, 2, 4, 0, 4, 3, 2, 0, 1]) 185 | con_matrix = np.ones([18]) 186 | adj_part = scipy.sparse.coo_matrix((con_matrix, (p_pair_1, p_pair_2)), shape=(10, 10)).toarray() 187 | 188 | # Body-Level adjacent matrix 189 | b_pair_1 = np.array([2, 3, 2, 4, 2, 1, 2, 0]) 190 | b_pair_2 = np.array([3, 2, 4, 2, 1, 2, 0, 2]) 191 | con_matrix = np.ones([8]) 192 | adj_body = scipy.sparse.coo_matrix((con_matrix, (b_pair_1, b_pair_2)), shape=(5, 5)).toarray() 193 | 194 | # if dataset != 'KS20': 195 | X_train_In = generate_denser_graph_data(X_train_J, adj_joint, nb_nodes) 196 | X_test_In = generate_denser_graph_data(X_test_J, adj_joint, nb_nodes) 197 | 198 | if global_att: 199 | if dataset == 'KS20': 200 | node_num = 25 201 | elif dataset == 'CASIA_B': 202 | node_num = 14 203 | else: 204 | node_num = 20 205 | adj_joint = np.ones([node_num, node_num]) 206 | adj_interp = np.ones([node_num * 2 - 1, node_num * 2 - 1]) 207 | adj_part = np.ones([10, 10]) 208 | adj_body = np.ones([5, 5]) 209 | 210 | if dataset == 'IAS': 211 | nb_classes = 11 212 | elif dataset == 'KGBD': 213 | nb_classes = 164 214 | elif dataset == 'BIWI': 215 | nb_classes = 28 216 | elif dataset == 'KS20': 217 | nb_classes = 20 218 | elif dataset == 'CASIA_B': 219 | nb_classes = 124 220 | 221 | adj_joint = adj_joint[np.newaxis] 222 | biases_joint = adj_to_bias(adj_joint, [nb_nodes], nhood=nhood) 223 | 224 | adj_part = adj_part[np.newaxis] 225 | biases_part = adj_to_bias(adj_part, [10], nhood=1) 226 | 227 | adj_body = adj_body[np.newaxis] 228 | biases_body = adj_to_bias(adj_body, [5], nhood=1) 229 | 230 | adj_interp = adj_interp[np.newaxis] 231 | biases_interp = adj_to_bias(adj_interp, [nb_nodes*2-1], nhood=1) 232 | 233 | return X_train_J, X_train_P, X_train_B, X_train_In, y_train, X_test_J, X_test_P, X_test_B, X_test_In, y_test, \ 234 | adj_joint, biases_joint, adj_part, biases_part, adj_body, biases_body, adj_interp, biases_interp, nb_classes 235 | 236 | """ 237 | Generate part-level skeleton graphs. 238 | """ 239 | 240 | def reduce2part(X, joint_num=20): 241 | if joint_num == 25: 242 | left_leg_up = [12, 13] 243 | left_leg_down = [14, 15] 244 | right_leg_up = [16, 17] 245 | right_leg_down = [18, 19] 246 | torso = [0, 1] 247 | head = [2, 3, 20] 248 | left_arm_up = [4, 5] 249 | left_arm_down = [6, 7, 21, 22] 250 | right_arm_up = [8, 9] 251 | right_arm_down = [10, 11, 23, 24] 252 | elif joint_num == 20: 253 | left_leg_up = [12, 13] 254 | left_leg_down = [14, 15] 255 | right_leg_up = [16, 17] 256 | right_leg_down = [18, 19] 257 | torso = [0, 1] 258 | head = [2, 3] 259 | left_arm_up = [4, 5] 260 | left_arm_down = [6, 7] 261 | right_arm_up = [8, 9] 262 | right_arm_down = [10, 11] 263 | elif joint_num == 14: 264 | left_leg_up = [11] 265 | left_leg_down = [12, 13] 266 | right_leg_up = [8] 267 | right_leg_down = [9, 10] 268 | torso = [1] 269 | head = [0] 270 | left_arm_up = [5] 271 | left_arm_down = [6, 7] 272 | right_arm_up = [2] 273 | right_arm_down = [3, 4] 274 | 275 | x_torso = np.mean(X[:, :, torso, :], axis=2) # [N * T, V=1] 276 | x_leftlegup = np.mean(X[:, :, left_leg_up, :], axis=2) 277 | x_leftlegdown = np.mean(X[:, :, left_leg_down, :], axis=2) 278 | x_rightlegup = np.mean(X[:, :, right_leg_up, :], axis=2) 279 | x_rightlegdown = np.mean(X[:, :, right_leg_down, :], axis=2) 280 | x_head = np.mean(X[:, :, head, :], axis=2) 281 | x_leftarmup = np.mean(X[:, :, left_arm_up, :], axis=2) 282 | x_leftarmdown = np.mean(X[:, :, left_arm_down, :], axis=2) 283 | x_rightarmup = np.mean(X[:, :, right_arm_up, :], axis=2) 284 | x_rightarmdown = np.mean(X[:, :, right_arm_down, :], axis=2) 285 | X_part = np.concatenate((x_leftlegup, x_leftlegdown, x_rightlegup, x_rightlegdown, x_torso, x_head, x_leftarmup, 286 | x_leftarmdown, x_rightarmup, x_rightarmdown), axis=-1) \ 287 | .reshape([X.shape[0], X.shape[1], 10, 3]) 288 | return X_part 289 | 290 | """ 291 | Generate body-level skeleton graphs. 292 | """ 293 | 294 | def reduce2body(X, joint_num=20): 295 | if joint_num == 25: 296 | left_leg = [12, 13, 14, 15] 297 | right_leg = [16, 17, 18, 19] 298 | torso = [0, 1, 2, 3, 20] 299 | left_arm = [4, 5, 6, 7, 21, 22] 300 | right_arm = [8, 9, 10, 11, 23, 24] 301 | elif joint_num == 20: 302 | left_leg = [12, 13, 14, 15] 303 | right_leg = [16, 17, 18, 19] 304 | torso = [0, 1, 2, 3] 305 | left_arm = [4, 5, 6, 7] 306 | right_arm = [8, 9, 10, 11] 307 | elif joint_num == 14: 308 | left_leg = [11, 12, 13] 309 | right_leg = [8, 9, 10] 310 | torso = [0, 1] 311 | left_arm = [5, 6, 7] 312 | right_arm = [2, 3, 4] 313 | 314 | x_torso = np.mean(X[:, :, torso, :], axis=2) # [N * T, V=1] 315 | x_leftleg = np.mean(X[:, :, left_leg, :], axis=2) 316 | x_rightleg = np.mean(X[:, :, right_leg, :], axis=2) 317 | x_leftarm = np.mean(X[:, :, left_arm, :], axis=2) 318 | x_rightarm = np.mean(X[:, :, right_arm, :], axis=2) 319 | X_body = np.concatenate((x_leftleg, x_rightleg, x_torso, x_leftarm, x_rightarm), axis=-1)\ 320 | .reshape([X.shape[0], X.shape[1], 5, 3]) 321 | return X_body 322 | 323 | """ 324 | Linear interpolation 325 | """ 326 | 327 | def interpolation(X, joint_num=20): 328 | if joint_num == 25: 329 | left_leg_up = [12, 13] 330 | left_leg_down = [14, 15] 331 | right_leg_up = [16, 17] 332 | right_leg_down = [18, 19] 333 | torso = [0, 1] 334 | head_1 = [2, 3] 335 | head_2 = [2, 20] 336 | left_arm_up = [4, 5] 337 | left_arm_down_1 = [6, 7] 338 | left_arm_down_2 = [7, 21] 339 | left_arm_down_3 = [6, 22] 340 | right_arm_up = [8, 9] 341 | right_arm_down_1 = [10, 11] 342 | right_arm_down_2 = [11, 23] 343 | right_arm_down_3 = [10, 24] 344 | shoulder_1 = [4, 20] 345 | shoulder_2 = [8, 20] 346 | elbow_1 = [5, 6] 347 | elbow_2 = [9, 10] 348 | spine_mm = [20, 1] 349 | hip_1 = [0, 12] 350 | hip_2 = [0, 16] 351 | knee_1 = [13, 14] 352 | knee_2 = [17, 18] 353 | x_torso = np.mean(X[:, :, torso, :], axis=2) # [N * T, V=1] 354 | x_leftlegup = np.mean(X[:, :, left_leg_up, :], axis=2) 355 | x_leftlegdown = np.mean(X[:, :, left_leg_down, :], axis=2) 356 | x_rightlegup = np.mean(X[:, :, right_leg_up, :], axis=2) 357 | x_rightlegdown = np.mean(X[:, :, right_leg_down, :], axis=2) 358 | x_head_1 = np.mean(X[:, :, head_1, :], axis=2) 359 | x_head_2 = np.mean(X[:, :, head_2, :], axis=2) 360 | x_leftarmup = np.mean(X[:, :, left_arm_up, :], axis=2) 361 | x_leftarmdown_1 = np.mean(X[:, :, left_arm_down_1, :], axis=2) 362 | x_leftarmdown_2 = np.mean(X[:, :, left_arm_down_2, :], axis=2) 363 | x_leftarmdown_3 = np.mean(X[:, :, left_arm_down_3, :], axis=2) 364 | x_rightarmup = np.mean(X[:, :, right_arm_up, :], axis=2) 365 | x_rightarmdown_1 = np.mean(X[:, :, right_arm_down_1, :], axis=2) 366 | x_rightarmdown_2 = np.mean(X[:, :, right_arm_down_2, :], axis=2) 367 | x_rightarmdown_3 = np.mean(X[:, :, right_arm_down_3, :], axis=2) 368 | shoulder_1 = np.mean(X[:, :, shoulder_1, :], axis=2) 369 | shoulder_2 = np.mean(X[:, :, shoulder_2, :], axis=2) 370 | elbow_1 = np.mean(X[:, :, elbow_1, :], axis=2) 371 | elbow_2 = np.mean(X[:, :, elbow_2, :], axis=2) 372 | spine_mm = np.mean(X[:, :, spine_mm, :], axis=2) 373 | hip_1 = np.mean(X[:, :, hip_1, :], axis=2) 374 | hip_2 = np.mean(X[:, :, hip_2, :], axis=2) 375 | knee_1 = np.mean(X[:, :, knee_1, :], axis=2) 376 | knee_2 = np.mean(X[:, :, knee_2, :], axis=2) 377 | X_part = np.concatenate((x_leftlegup, x_leftlegdown, x_rightlegup, 378 | x_rightlegdown, x_torso, x_head_1, x_head_2, x_leftarmup, 379 | x_leftarmdown_1, x_leftarmdown_2, x_leftarmdown_3, 380 | x_rightarmup, x_rightarmdown_1, x_rightarmdown_2, x_rightarmdown_3, 381 | shoulder_1, shoulder_2, elbow_1, elbow_2, spine_mm, 382 | hip_1, hip_2, knee_1, knee_2), axis=-1) \ 383 | .reshape([X.shape[0], X.shape[1], 24, 3]) 384 | # 25+24 385 | X_interp = np.concatenate((X, X_part), axis=-2) 386 | # Changed 387 | elif joint_num == 20: 388 | # left_leg_up = [12, 13] 389 | # left_leg_down = [14, 15] 390 | # right_leg_up = [16, 17] 391 | # right_leg_down = [18, 19] 392 | # torso = [0, 1] 393 | # head = [2, 3] 394 | # left_arm_up = [4, 5] 395 | # left_arm_down = [6, 7] 396 | # right_arm_up = [8, 9] 397 | # right_arm_down = [10, 11] 398 | # 399 | left_leg_up = [12, 13] 400 | left_leg_down = [14, 15] 401 | right_leg_up = [16, 17] 402 | right_leg_down = [18, 19] 403 | torso = [0, 1] 404 | head_1 = [2, 3] 405 | # head_2 = [2, 20] 406 | left_arm_up = [4, 5] 407 | left_arm_down_1 = [6, 7] 408 | # left_arm_down_2 = [7, 21] 409 | # left_arm_down_3 = [6, 22] 410 | right_arm_up = [8, 9] 411 | right_arm_down_1 = [10, 11] 412 | # right_arm_down_2 = [11, 23] 413 | # right_arm_down_3 = [10, 24] 414 | # shoulder_1 = [4, 20] 415 | # shoulder_2 = [8, 20] 416 | shoulder_1 = [4, 2] 417 | shoulder_2 = [8, 2] 418 | elbow_1 = [5, 6] 419 | elbow_2 = [9, 10] 420 | # spine_mm = [20, 1] 421 | spine_mm = [2, 1] 422 | hip_1 = [0, 12] 423 | hip_2 = [0, 16] 424 | knee_1 = [13, 14] 425 | knee_2 = [17, 18] 426 | 427 | x_torso = np.mean(X[:, :, torso, :], axis=2) # [N * T, V=1] 428 | x_leftlegup = np.mean(X[:, :, left_leg_up, :], axis=2) 429 | x_leftlegdown = np.mean(X[:, :, left_leg_down, :], axis=2) 430 | x_rightlegup = np.mean(X[:, :, right_leg_up, :], axis=2) 431 | x_rightlegdown = np.mean(X[:, :, right_leg_down, :], axis=2) 432 | x_head_1 = np.mean(X[:, :, head_1, :], axis=2) 433 | # x_head_2 = np.mean(X[:, :, head_2, :], axis=2) 434 | x_leftarmup = np.mean(X[:, :, left_arm_up, :], axis=2) 435 | x_leftarmdown_1 = np.mean(X[:, :, left_arm_down_1, :], axis=2) 436 | # x_leftarmdown_2 = np.mean(X[:, :, left_arm_down_2, :], axis=2) 437 | # x_leftarmdown_3 = np.mean(X[:, :, left_arm_down_3, :], axis=2) 438 | x_rightarmup = np.mean(X[:, :, right_arm_up, :], axis=2) 439 | x_rightarmdown_1 = np.mean(X[:, :, right_arm_down_1, :], axis=2) 440 | # x_rightarmdown_2 = np.mean(X[:, :, right_arm_down_2, :], axis=2) 441 | # x_rightarmdown_3 = np.mean(X[:, :, right_arm_down_3, :], axis=2) 442 | shoulder_1 = np.mean(X[:, :, shoulder_1, :], axis=2) 443 | shoulder_2 = np.mean(X[:, :, shoulder_2, :], axis=2) 444 | elbow_1 = np.mean(X[:, :, elbow_1, :], axis=2) 445 | elbow_2 = np.mean(X[:, :, elbow_2, :], axis=2) 446 | spine_mm = np.mean(X[:, :, spine_mm, :], axis=2) 447 | hip_1 = np.mean(X[:, :, hip_1, :], axis=2) 448 | hip_2 = np.mean(X[:, :, hip_2, :], axis=2) 449 | knee_1 = np.mean(X[:, :, knee_1, :], axis=2) 450 | knee_2 = np.mean(X[:, :, knee_2, :], axis=2) 451 | X_part = np.concatenate((x_leftlegup, x_leftlegdown, x_rightlegup, 452 | x_rightlegdown, x_torso, x_head_1, x_leftarmup, 453 | x_leftarmdown_1, 454 | x_rightarmup, x_rightarmdown_1, 455 | shoulder_1, shoulder_2, elbow_1, elbow_2, spine_mm, 456 | hip_1, hip_2, knee_1, knee_2), axis=-1) \ 457 | .reshape([X.shape[0], X.shape[1], 19, 3]) 458 | # 25+24 459 | X_interp = np.concatenate((X, X_part), axis=-2) 460 | return X_interp 461 | 462 | def generate_denser_graph_data(X, adj, joint_num=20): 463 | adj_temp = copy.deepcopy(adj) 464 | adj_temp = adj_temp.tolist() 465 | node_num = len(adj_temp) 466 | cnt = node_num 467 | for i in range(node_num): 468 | for j in range(node_num): 469 | if adj_temp[i][j] == 1: 470 | adj_temp[i][j] = adj_temp[j][i] = 0 471 | new_node = np.mean(X[:, :, [i, j], :], axis=2) 472 | # print(new_node.shape) 473 | if cnt == node_num: 474 | X_interp = new_node 475 | else: 476 | X_interp = np.concatenate((X_interp, new_node), axis=-1) 477 | # print(X_interp.shape) 478 | # print(i, j) 479 | # print(i, j, cnt) 480 | cnt += 1 481 | # print(X_interp.shape) 482 | X_interp = np.reshape(X_interp, [X.shape[0], X.shape[1], node_num-1, 3]) 483 | X_interp = np.concatenate((X, X_interp), axis=-2) 484 | return X_interp 485 | 486 | """ 487 | Calculate normalized area under curves. 488 | """ 489 | def cal_nAUC(scores, labels): 490 | scores = np.array(scores) 491 | labels = np.array(labels) 492 | # Compute micro-average ROC curve and ROC area 493 | fpr, tpr, thresholds = roc_curve(labels.ravel(), scores.ravel()) 494 | roc_auc = auc(fpr, tpr) 495 | return roc_auc 496 | 497 | """ 498 | Generate training data with evenly distributed classes. 499 | """ 500 | def class_samp_gen(X, y, ids_, batch_size): 501 | class_num = len(ids_.keys()) 502 | ids_ = sorted(ids_.items(), key=lambda item: item[0]) 503 | cnt = 0 504 | all_batch_X = [] 505 | all_batch_y = [] 506 | total = y.shape[0] 507 | batch_num = total // batch_size * 2 508 | class_in_bacth = class_num 509 | batch_per_class = batch_size // class_in_bacth 510 | class_cnt = class_in_bacth 511 | # print(total, batch_num, batch_per_class) 512 | for i in range(batch_num): 513 | batch_X = [] 514 | batch_y = [] 515 | for k, v in ids_[class_cnt-class_in_bacth:class_cnt]: 516 | # print(k, len(v)) 517 | # cnt += len(v) 518 | if len(v[batch_per_class*i:batch_per_class*(i+1)]) < batch_per_class: 519 | rand_ind = np.random.choice(len(v), batch_per_class) 520 | v_array = np.array(v) 521 | samp_per_class = v_array[rand_ind].tolist() 522 | batch_X.extend(samp_per_class) 523 | else: 524 | batch_X.extend(v[batch_per_class*i:batch_per_class*(i+1)]) 525 | batch_y.extend(batch_per_class * [k]) 526 | if class_cnt + class_in_bacth > class_num and class_cnt <= class_num: 527 | class_cnt = class_num 528 | else: 529 | class_cnt = class_cnt + class_in_bacth 530 | all_batch_X.extend(batch_X) 531 | all_batch_y.extend(batch_y) 532 | # print(len(all_batch_X), len(all_batch_y)) 533 | X_train = X[all_batch_X] 534 | y_train = np.array(all_batch_y) 535 | return X_train, y_train 536 | 537 | """ 538 | Prepare adjacency matrix by expanding up to a given neighbourhood. 539 | This will insert loops on every node. 540 | Finally, the matrix is converted to bias vectors. 541 | Expected shape: [graph, nodes, nodes] 542 | """ 543 | 544 | 545 | def adj_to_bias(adj, sizes, nhood=1): 546 | nb_graphs = adj.shape[0] 547 | mt = np.empty(adj.shape) 548 | for g in range(nb_graphs): 549 | mt[g] = np.eye(adj.shape[1]) 550 | for _ in range(nhood): 551 | mt[g] = np.matmul(mt[g], (adj[g] + np.eye(adj.shape[1]))) 552 | for i in range(sizes[g]): 553 | for j in range(sizes[g]): 554 | if mt[g][i][j] > 0.0: 555 | mt[g][i][j] = 1.0 556 | return -1e9 * (1.0 - mt) 557 | 558 | 559 | ############################################### 560 | # This section of code adapted from tkipf/gcn # 561 | ############################################### 562 | 563 | def parse_index_file(filename): 564 | """Parse index file.""" 565 | index = [] 566 | for line in open(filename): 567 | index.append(int(line.strip())) 568 | return index 569 | 570 | 571 | def sample_mask(idx, l): 572 | """Create mask.""" 573 | mask = np.zeros(l) 574 | mask[idx] = 1 575 | return np.array(mask, dtype=np.bool) 576 | 577 | 578 | def load_data(dataset_str): # {'pubmed', 'citeseer', 'cora'} 579 | """Load data.""" 580 | names = ['x', 'y', 'tx', 'ty', 'allx', 'ally', 'graph'] 581 | objects = [] 582 | for i in range(len(names)): 583 | with open("data/ind.{}.{}".format(dataset_str, names[i]), 'rb') as f: 584 | if sys.version_info > (3, 0): 585 | objects.append(pkl.load(f, encoding='latin1')) 586 | else: 587 | objects.append(pkl.load(f)) 588 | 589 | x, y, tx, ty, allx, ally, graph = tuple(objects) 590 | test_idx_reorder = parse_index_file("data/ind.{}.test.index".format(dataset_str)) 591 | test_idx_range = np.sort(test_idx_reorder) 592 | 593 | if dataset_str == 'citeseer': 594 | # Fix citeseer dataset (there are some isolated nodes in the graph) 595 | # Find isolated nodes, add them as zero-vecs into the right position 596 | test_idx_range_full = range(min(test_idx_reorder), max(test_idx_reorder) + 1) 597 | tx_extended = sp.lil_matrix((len(test_idx_range_full), x.shape[1])) 598 | tx_extended[test_idx_range - min(test_idx_range), :] = tx 599 | tx = tx_extended 600 | ty_extended = np.zeros((len(test_idx_range_full), y.shape[1])) 601 | ty_extended[test_idx_range - min(test_idx_range), :] = ty 602 | ty = ty_extended 603 | 604 | features = sp.vstack((allx, tx)).tolil() 605 | features[test_idx_reorder, :] = features[test_idx_range, :] 606 | adj = nx.adjacency_matrix(nx.from_dict_of_lists(graph)) 607 | 608 | labels = np.vstack((ally, ty)) 609 | labels[test_idx_reorder, :] = labels[test_idx_range, :] 610 | 611 | idx_test = test_idx_range.tolist() 612 | idx_train = range(len(y)) 613 | idx_val = range(len(y), len(y) + 500) 614 | 615 | train_mask = sample_mask(idx_train, labels.shape[0]) 616 | val_mask = sample_mask(idx_val, labels.shape[0]) 617 | test_mask = sample_mask(idx_test, labels.shape[0]) 618 | 619 | y_train = np.zeros(labels.shape) 620 | y_val = np.zeros(labels.shape) 621 | y_test = np.zeros(labels.shape) 622 | y_train[train_mask, :] = labels[train_mask, :] 623 | y_val[val_mask, :] = labels[val_mask, :] 624 | y_test[test_mask, :] = labels[test_mask, :] 625 | 626 | # print(adj.shape) 627 | # print(features.shape) 628 | 629 | return adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask 630 | 631 | 632 | def load_random_data(size): 633 | adj = sp.random(size, size, density=0.002) # density similar to cora 634 | features = sp.random(size, 1000, density=0.015) 635 | int_labels = np.random.randint(7, size=(size)) 636 | labels = np.zeros((size, 7)) # Nx7 637 | labels[np.arange(size), int_labels] = 1 638 | 639 | train_mask = np.zeros((size,)).astype(bool) 640 | train_mask[np.arange(size)[0:int(size / 2)]] = 1 641 | 642 | val_mask = np.zeros((size,)).astype(bool) 643 | val_mask[np.arange(size)[int(size / 2):]] = 1 644 | 645 | test_mask = np.zeros((size,)).astype(bool) 646 | test_mask[np.arange(size)[int(size / 2):]] = 1 647 | 648 | y_train = np.zeros(labels.shape) 649 | y_val = np.zeros(labels.shape) 650 | y_test = np.zeros(labels.shape) 651 | y_train[train_mask, :] = labels[train_mask, :] 652 | y_val[val_mask, :] = labels[val_mask, :] 653 | y_test[test_mask, :] = labels[test_mask, :] 654 | 655 | # sparse NxN, sparse NxF, norm NxC, ..., norm Nx1, ... 656 | return adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask 657 | 658 | 659 | def sparse_to_tuple(sparse_mx): 660 | """Convert sparse matrix to tuple representation.""" 661 | 662 | def to_tuple(mx): 663 | if not sp.isspmatrix_coo(mx): 664 | mx = mx.tocoo() 665 | coords = np.vstack((mx.row, mx.col)).transpose() 666 | values = mx.data 667 | shape = mx.shape 668 | return coords, values, shape 669 | 670 | if isinstance(sparse_mx, list): 671 | for i in range(len(sparse_mx)): 672 | sparse_mx[i] = to_tuple(sparse_mx[i]) 673 | else: 674 | sparse_mx = to_tuple(sparse_mx) 675 | 676 | return sparse_mx 677 | 678 | 679 | def standardize_data(f, train_mask): 680 | """Standardize feature matrix and convert to tuple representation""" 681 | # standardize data 682 | f = f.todense() 683 | mu = f[train_mask == True, :].mean(axis=0) 684 | sigma = f[train_mask == True, :].std(axis=0) 685 | f = f[:, np.squeeze(np.array(sigma > 0))] 686 | mu = f[train_mask == True, :].mean(axis=0) 687 | sigma = f[train_mask == True, :].std(axis=0) 688 | f = (f - mu) / sigma 689 | return f 690 | 691 | 692 | def preprocess_features(features): 693 | """Row-normalize feature matrix and convert to tuple representation""" 694 | rowsum = np.array(features.sum(1)) 695 | r_inv = np.power(rowsum, -1).flatten() 696 | r_inv[np.isinf(r_inv)] = 0. 697 | r_mat_inv = sp.diags(r_inv) 698 | features = r_mat_inv.dot(features) 699 | return features.todense(), sparse_to_tuple(features) 700 | 701 | 702 | def normalize_adj(adj): 703 | """Symmetrically normalize adjacency matrix.""" 704 | adj = sp.coo_matrix(adj) 705 | rowsum = np.array(adj.sum(1)) 706 | d_inv_sqrt = np.power(rowsum, -0.5).flatten() 707 | d_inv_sqrt[np.isinf(d_inv_sqrt)] = 0. 708 | d_mat_inv_sqrt = sp.diags(d_inv_sqrt) 709 | return adj.dot(d_mat_inv_sqrt).transpose().dot(d_mat_inv_sqrt).tocoo() 710 | 711 | 712 | def preprocess_adj(adj): 713 | """Preprocessing of adjacency matrix for simple GCN model and conversion to tuple representation.""" 714 | adj_normalized = normalize_adj(adj + sp.eye(adj.shape[0])) 715 | return sparse_to_tuple(adj_normalized) 716 | 717 | 718 | def preprocess_adj_bias(adj): 719 | num_nodes = adj.shape[0] 720 | adj = adj + sp.eye(num_nodes) # self-loop 721 | adj[adj > 0.0] = 1.0 722 | if not sp.isspmatrix_coo(adj): 723 | adj = adj.tocoo() 724 | adj = adj.astype(np.float32) 725 | indices = np.vstack( 726 | (adj.col, adj.row)).transpose() # This is where I made a mistake, I used (adj.row, adj.col) instead 727 | # return tf.SparseTensor(indices=indices, values=adj.data, dense_shape=adj.shape) 728 | return indices, adj.data, adj.shape 729 | -------------------------------------------------------------------------------- /SM-SGE.py: -------------------------------------------------------------------------------- 1 | import time 2 | import numpy as np 3 | import tensorflow as tf 4 | import os, sys 5 | from models import MGRN_S 6 | from utils import process_i as process 7 | from tensorflow.python.layers.core import Dense 8 | from sklearn.preprocessing import label_binarize 9 | 10 | 11 | dataset = 'IAS' 12 | split = 'B' 13 | pretext = 'recon' 14 | RN_dir = 'RN/' # save MLP models for person Re-ID 15 | pre_dir = 'Pre-Trained/' # save self-supervised SM-SGE models 16 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 17 | config = tf.ConfigProto() 18 | config.gpu_options.allow_growth = True 19 | pre_epochs = 200 # epochs for self-supervised training 20 | sample_num = 1 # number of sampling rounds (r) 21 | nb_nodes = 20 # number of nodes in joint-scale graph 22 | nhood = 1 # structural relation learning (nhood=1 for neighbor nodes) 23 | fusion_lambda = 1 # collaboration fusion coefficient 24 | ft_size = 3 # originial node feature dimension (D) 25 | time_step = 6 # sequence length (f) 26 | 27 | LSTM_embed = 256 # number of hidden units per layer in LSTM (D_{h}) 28 | num_layers = 2 # number of LSTM layers 29 | 30 | # training params 31 | batch_size = 256 32 | nb_epochs = 100000 33 | patience = 150 # patience for early stopping 34 | lr = 0.005 # learning rate 35 | hid_units = [8] # numbers of hidden units per each attention head in each layer 36 | Ps = [8, 1] # additional entry for the output layer 37 | residual = False 38 | nonlinearity = tf.nn.elu 39 | model = MGRN_S 40 | 41 | tf.app.flags.DEFINE_string('dataset', 'BIWI', "Dataset: BIWI, IAS, KS20 or KGBD") 42 | tf.app.flags.DEFINE_string('length', '6', "4, 6, 8 or 10") 43 | tf.app.flags.DEFINE_string('split', '', "for IAS-Lab testing splits") 44 | tf.app.flags.DEFINE_string('gpu', '0', "GPU number") 45 | tf.app.flags.DEFINE_string('task', 'pre', "prediction") # we regard reconstruction as a prediction with 0 pred_margin 46 | tf.app.flags.DEFINE_string('frozen', '1', "Frozen LSTM states") # frozen encoded graph states for person Re-ID 47 | tf.app.flags.DEFINE_string('pre_epochs', '200', "epochs for pre-training") 48 | tf.app.flags.DEFINE_string('pre_train', '1', "pre-train or not") 49 | tf.app.flags.DEFINE_string('s_range', 'all', "1, f/2, f-1, all") 50 | tf.app.flags.DEFINE_string('pred_margin', '0', "0, 1, 2") 51 | tf.app.flags.DEFINE_string('view', '', "test different views on CASIA B") 52 | tf.app.flags.DEFINE_string('reverse', '0', "use reverse sequences") 53 | tf.app.flags.DEFINE_string('bi', '0', "bi-directional prediction") 54 | tf.app.flags.DEFINE_string('save_flag', '1', "save model or not") 55 | tf.app.flags.DEFINE_string('consecutive_pre', '0', "consecutive_pre") 56 | tf.app.flags.DEFINE_string('single_level', '0', "single_level") 57 | tf.app.flags.DEFINE_string('global_att', '0', "global_att") 58 | tf.app.flags.DEFINE_string('last_pre', '0', "last_pre") 59 | tf.app.flags.DEFINE_string('random_sample', '0', "random_sample") 60 | tf.app.flags.DEFINE_string('ord_sample', '0', "ord_sample") 61 | tf.app.flags.DEFINE_string('concate', '0', "concate") 62 | tf.app.flags.DEFINE_string('struct_only', '0', "struct_only") 63 | tf.app.flags.DEFINE_string('abla', '0', "abla") 64 | tf.app.flags.DEFINE_string('P', '8', "P") 65 | tf.app.flags.DEFINE_string('n_hood', '1', "n_hood") 66 | tf.app.flags.DEFINE_string('probe_type', '', "probe.gallery") 67 | tf.app.flags.DEFINE_string('no_MSR', '0', "using MSR or not") 68 | tf.app.flags.DEFINE_string('patience', '100', "epochs for early stopping") 69 | tf.app.flags.DEFINE_string('sample_num', '1', "sampling times") 70 | tf.app.flags.DEFINE_string('fusion_lambda', '1', "collaboration fusion coefficient") 71 | tf.app.flags.DEFINE_string('loss', 'l1', "use l1, l2 or MSE loss") 72 | 73 | FLAGS = tf.app.flags.FLAGS 74 | 75 | # check parameters 76 | if FLAGS.dataset not in ['IAS', 'KGBD', 'KS20']: 77 | raise Exception('Dataset must be IAS, KGBD, or KS20.') 78 | if not FLAGS.gpu.isdigit() or int(FLAGS.gpu) < 0: 79 | raise Exception('GPU number must be a positive integer.') 80 | if FLAGS.length not in ['4', '6', '8', '10']: 81 | raise Exception('Length number must be 4, 6, 8 or 10.') 82 | if FLAGS.split not in ['', 'A', 'B']: 83 | raise Exception('Datset split must be "A" (for IAS-A), "B" (for IAS-B), "" (for other datasets).') 84 | if float(FLAGS.fusion_lambda) < 0 or float(FLAGS.fusion_lambda) > 1: 85 | raise Exception('Collaboration Fusion coefficient must be not less than 0 or not larger than 1.') 86 | if FLAGS.pre_train not in ['1', '0']: 87 | raise Exception('Pre-train Flag must be 0 or 1.') 88 | if FLAGS.save_flag not in ['1', '0']: 89 | raise Exception('Save_flag must be 0 or 1.') 90 | 91 | os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu 92 | dataset = FLAGS.dataset 93 | # optimal paramters 94 | if dataset == 'KS20': 95 | batch_size = 64 96 | LSTM_embed = 256 97 | lr = 0.0005 98 | elif dataset == 'IAS' and split == 'A': 99 | batch_size = 128 100 | LSTM_embed = 256 101 | lr = 0.0025 102 | elif dataset == 'IAS' and split == 'B': 103 | batch_size = 64 104 | LSTM_embed = 256 105 | lr = 0.0025 106 | elif dataset == 'CASIA_B': 107 | batch_size = 128 108 | LSTM_embed = 256 109 | lr = 0.0005 110 | # patience = 50 111 | if int(FLAGS.pre_epochs) != 200: 112 | pre_epochs = int(FLAGS.pre_epochs) 113 | elif dataset == 'KGBD': 114 | batch_size = 256 115 | LSTM_embed = 128 116 | lr = 0.0025 117 | pre_epochs = 160 118 | if dataset == 'IAS' or dataset == 'BIWI': 119 | pre_epochs = 120 120 | if dataset == 'CASIA_B': 121 | pre_epochs = 160 122 | if FLAGS.probe_type != '': 123 | pre_epochs = 200 124 | time_step = int(FLAGS.length) 125 | k_values = list(range(1, time_step - int(FLAGS.pred_margin) + 1)) 126 | k_filter = -1 127 | if FLAGS.s_range == '1': 128 | k_filter = 1 129 | elif FLAGS.s_range == 'f/2': 130 | k_filter = int(time_step//2) 131 | elif FLAGS.s_range == 'f-1': 132 | k_filter = time_step-1 133 | elif FLAGS.s_range == 'f': 134 | k_filter = time_step 135 | 136 | fusion_lambda = float(FLAGS.fusion_lambda) 137 | split = FLAGS.split 138 | pretext = FLAGS.task 139 | levels = len(k_values) 140 | frozen= FLAGS.frozen 141 | nhood = int(FLAGS.n_hood) 142 | pred_margin = FLAGS.pred_margin 143 | s_range = FLAGS.s_range 144 | save_flag = FLAGS.save_flag 145 | patience = int(FLAGS.patience) 146 | sample_num = int(FLAGS.sample_num) 147 | 148 | consecutive_pre = False 149 | single_level = False 150 | global_att = False 151 | MG_only = False 152 | last_pre = False 153 | ord_sample = False 154 | concate = False 155 | struct_only = False 156 | abla = False 157 | P = '8' 158 | 159 | only_ske_embed = False 160 | three_layer = False 161 | embed_half = False 162 | one_layer = False 163 | CAGEs_embed = False 164 | no_interp = False 165 | no_multi_pred = False 166 | 167 | change = '_formal' 168 | 169 | if FLAGS.loss != 'l1': 170 | change += '_loss_' + FLAGS.loss 171 | 172 | if sample_num != 1: 173 | change = '_sample_num_' + str(sample_num) 174 | if FLAGS.probe_type != '': 175 | change = '_CME' 176 | if FLAGS.fusion_lambda != '1': 177 | change = '_lambda_' + FLAGS.fusion_lambda 178 | 179 | if FLAGS.consecutive_pre == '1': 180 | consecutive_pre = True 181 | if FLAGS.single_level == '1': 182 | single_level = True 183 | if FLAGS.global_att == '1': 184 | global_att = True 185 | if FLAGS.last_pre == '1': 186 | last_pre = True 187 | if FLAGS.ord_sample == '1': 188 | ord_sample = True 189 | if FLAGS.concate == '1': 190 | concate = True 191 | if FLAGS.struct_only == '1': 192 | struct_only = True 193 | if FLAGS.P != '8': 194 | P = FLAGS.P 195 | Ps = [int(P), 1] 196 | 197 | try: 198 | os.mkdir(RN_dir) 199 | except: 200 | pass 201 | try: 202 | os.mkdir(pre_dir) 203 | except: 204 | pass 205 | 206 | if consecutive_pre: 207 | pre_dir += '_consec_pre' 208 | RN_dir += '_consec_pre' 209 | if single_level: 210 | fusion_lambda = 0 211 | pre_dir += '_single' 212 | RN_dir += '_single' 213 | if global_att: 214 | pre_dir += '_global_att' 215 | RN_dir += '_global_att' 216 | if MG_only: 217 | pre_dir += '_MG_only' 218 | RN_dir += '_MG_only' 219 | if last_pre: 220 | pre_dir += '_last_pre' 221 | RN_dir += '_last_pre' 222 | if ord_sample: 223 | pre_dir += '_ord_sample' 224 | RN_dir += '_ord_sample' 225 | if concate: 226 | pre_dir += '_concate' 227 | RN_dir += '_concate' 228 | if frozen == '1': 229 | pre_dir += '_frozen' 230 | RN_dir += '_frozen' 231 | if struct_only: 232 | pre_dir += '_struct_only' 233 | RN_dir += '_struct_only' 234 | if P != '8': 235 | pre_dir += '_P_' + P 236 | RN_dir += '_P_' + P 237 | 238 | if pretext == 'none': 239 | pre_epochs = 0 240 | I_nodes = 39 241 | if dataset == 'KS20': 242 | nb_nodes = 25 243 | I_nodes = 49 244 | if dataset == 'CASIA_B': 245 | nb_nodes = 14 246 | I_nodes = 27 247 | 248 | if FLAGS.view != '': 249 | view_dir = '_view_' + FLAGS.view 250 | else: 251 | view_dir = '' 252 | 253 | print('Dataset: ' + dataset) 254 | 255 | print('----- Opt. hyperparams -----') 256 | print('pre_train_epochs: ' + str(pre_epochs)) 257 | print('nhood: ' + str(nhood)) 258 | print('skeleton_nodes: ' + str(nb_nodes)) 259 | print('seqence_length: ' + str(time_step)) 260 | print('pretext: ' + str(pretext)) 261 | print('fusion_lambda: ' + str(fusion_lambda)) 262 | print('batch_size: ' + str(batch_size)) 263 | print('lr: ' + str(lr)) 264 | print('view: ' + FLAGS.view) 265 | print('P: ' + FLAGS.P) 266 | print('fusion_lambda: ' + FLAGS.fusion_lambda) 267 | print('loss_type: ' + FLAGS.loss) 268 | print('patience: ' + FLAGS.patience) 269 | print('save_flag: ' + FLAGS.save_flag) 270 | 271 | print('----- Archi. hyperparams -----') 272 | print('structural relation matrix number: ' + str(Ps[0])) 273 | print('LSTM_embed_num: ' + str(LSTM_embed)) 274 | print('LSTM_layer_num: ' + str(num_layers)) 275 | 276 | """ 277 | Obtain training and testing data in hyper-joint-scale, joint-scale, part-scale, and body-scale. 278 | Generate corresponding adjacent matrix and bias. 279 | """ 280 | if FLAGS.probe_type == '': 281 | X_train_J, X_train_P, X_train_B, X_train_I, y_train, X_test_J, X_test_P, X_test_B, X_test_I, y_test, \ 282 | adj_J, biases_J, adj_P, biases_P, adj_B, biases_B, adj_I, biases_I, nb_classes = \ 283 | process.gen_train_data(dataset=dataset, split=split, time_step=time_step, 284 | nb_nodes=nb_nodes, nhood=nhood, global_att=global_att, batch_size=batch_size, view=FLAGS.view, reverse=FLAGS.reverse) 285 | else: 286 | from utils import process_cme as process 287 | X_train_J, X_train_P, X_train_B, X_train_I, y_train, X_test_J, X_test_P, X_test_B, X_test_I, y_test, \ 288 | adj_J, biases_J, adj_P, biases_P, adj_B, biases_B, adj_I, biases_I, nb_classes = \ 289 | process.gen_train_data(dataset=dataset, split=split, time_step=time_step, 290 | nb_nodes=nb_nodes, nhood=nhood, global_att=global_att, batch_size=batch_size, 291 | reverse=FLAGS.reverse, PG_type=FLAGS.probe_type.split('.')[0]) 292 | print('## [Probe].[Gallery]', FLAGS.probe_type) 293 | 294 | 295 | if FLAGS.pre_train == '1': 296 | with tf.Graph().as_default(): 297 | with tf.name_scope('Input'): 298 | lbl_in = tf.placeholder(dtype=tf.int32, shape=(batch_size, nb_classes)) 299 | J_in = tf.placeholder(dtype=tf.float32, shape=(batch_size*time_step, nb_nodes, ft_size)) 300 | P_in = tf.placeholder(dtype=tf.float32, shape=(batch_size * time_step, 10, ft_size)) 301 | B_in = tf.placeholder(dtype=tf.float32, shape=(batch_size * time_step, 5, ft_size)) 302 | # Interpolation 303 | I_in = tf.placeholder(dtype=tf.float32, shape=(batch_size * time_step, I_nodes, ft_size)) 304 | J_bias_in = tf.placeholder(dtype=tf.float32, shape=(1, nb_nodes, nb_nodes)) 305 | P_bias_in = tf.placeholder(dtype=tf.float32, shape=(1, 10, 10)) 306 | B_bias_in = tf.placeholder(dtype=tf.float32, shape=(1, 5, 5)) 307 | I_bias_in = tf.placeholder(dtype=tf.float32, shape=(1, I_nodes, I_nodes)) 308 | attn_drop = tf.placeholder(dtype=tf.float32, shape=()) 309 | ffd_drop = tf.placeholder(dtype=tf.float32, shape=()) 310 | is_train = tf.placeholder(dtype=tf.bool, shape=()) 311 | 312 | with tf.name_scope("Multi_Scale"), tf.variable_scope("Multi_Scale", reuse=tf.AUTO_REUSE): 313 | def SRL(J_in, J_bias_in, nb_nodes): 314 | W_h = tf.Variable(tf.random_normal([3, hid_units[-1]])) 315 | b_h = tf.Variable(tf.zeros(shape=[hid_units[-1], ])) 316 | J_h = tf.reshape(J_in, [-1, ft_size]) 317 | 318 | J_h = tf.matmul(J_h, W_h) + b_h 319 | J_h = tf.reshape(J_h, [batch_size*time_step, nb_nodes, hid_units[-1]]) 320 | if not concate: 321 | J_seq_ftr = model.inference(J_h, 0, nb_nodes, is_train, 322 | attn_drop, ffd_drop, 323 | bias_mat=J_bias_in, 324 | hid_units=hid_units, n_heads=Ps, 325 | residual=residual, activation=nonlinearity, r_pool=True) 326 | else: 327 | J_seq_ftr = model.inference(J_h, 0, nb_nodes, is_train, 328 | attn_drop, ffd_drop, 329 | bias_mat=J_bias_in, 330 | hid_units=hid_units, n_heads=Ps, 331 | residual=residual, activation=nonlinearity, r_pool=False) 332 | return J_seq_ftr 333 | 334 | 335 | def CRL(s1, s2, s1_num, s2_num, hid_in): 336 | r_unorm = tf.matmul(s2, tf.transpose(s1, [0, 2, 1])) 337 | att_w = tf.nn.softmax(r_unorm) 338 | att_w = tf.expand_dims(att_w, axis=-1) 339 | s1 = tf.reshape(s1, [s1.shape[0], 1, s1.shape[1], hid_in]) 340 | c_ftr = tf.reduce_sum(att_w * s1, axis=2) 341 | c_ftr = tf.reshape(c_ftr, [-1, hid_in]) 342 | att_w = tf.reshape(att_w, [-1, s1_num * s2_num]) 343 | return r_unorm, c_ftr 344 | 345 | 346 | def MGRN(J_in, P_in, B_in, I_in, J_bias_in, P_bias_in, B_bias_in, I_bias_in, hid_in, hid_out): 347 | h_J_seq_ftr = SRL(J_in=J_in, J_bias_in=J_bias_in, nb_nodes=nb_nodes) 348 | h_P_seq_ftr = SRL(J_in=P_in, J_bias_in=P_bias_in, nb_nodes=10) 349 | h_B_seq_ftr = SRL(J_in=B_in, J_bias_in=B_bias_in, nb_nodes=5) 350 | h_I_seq_ftr = SRL(J_in=I_in, J_bias_in=I_bias_in, nb_nodes=I_nodes) 351 | 352 | h_J_seq_ftr = tf.reshape(h_J_seq_ftr, [-1, nb_nodes, hid_in]) 353 | h_P_seq_ftr = tf.reshape(h_P_seq_ftr, [-1, 10, hid_in]) 354 | h_B_seq_ftr = tf.reshape(h_B_seq_ftr, [-1, 5, hid_in]) 355 | h_I_seq_ftr = tf.reshape(h_I_seq_ftr, [-1, I_nodes, hid_in]) 356 | 357 | W_cs_12 = tf.Variable(tf.random_normal([hid_in, hid_out])) 358 | W_cs_23 = tf.Variable(tf.random_normal([hid_in, hid_out])) 359 | W_cs_13 = tf.Variable(tf.random_normal([hid_in, hid_out])) 360 | W_cs_I = tf.Variable(tf.random_normal([hid_in, hid_out])) 361 | W_cs_01 = tf.Variable(tf.random_normal([hid_in, hid_out])) 362 | W_cs_02 = tf.Variable(tf.random_normal([hid_in, hid_out])) 363 | W_cs_03 = tf.Variable(tf.random_normal([hid_in, hid_out])) 364 | 365 | 366 | W_self_0 = tf.Variable(tf.random_normal([hid_in, hid_out])) 367 | W_self_1 = tf.Variable(tf.random_normal([hid_in, hid_out])) 368 | W_self_2 = tf.Variable(tf.random_normal([hid_in, hid_out])) 369 | W_self_3 = tf.Variable(tf.random_normal([hid_in, hid_out])) 370 | self_a_0, self_r_0 = CRL(h_I_seq_ftr, h_I_seq_ftr, I_nodes, I_nodes, hid_in) 371 | self_a_1, self_r_1 = CRL(h_J_seq_ftr, h_J_seq_ftr, nb_nodes, nb_nodes, hid_in) 372 | self_a_2, self_r_2 = CRL(h_P_seq_ftr, h_P_seq_ftr, 10, 10, hid_in) 373 | self_a_3, self_r_3 = CRL(h_B_seq_ftr, h_B_seq_ftr, 5, 5, hid_in) 374 | 375 | h_J_seq_ftr = tf.reshape(h_J_seq_ftr, [-1, hid_in]) 376 | h_P_seq_ftr = tf.reshape(h_P_seq_ftr, [-1, hid_in]) 377 | h_B_seq_ftr = tf.reshape(h_B_seq_ftr, [-1, hid_in]) 378 | h_I_seq_ftr = tf.reshape(h_I_seq_ftr, [-1, hid_in]) 379 | 380 | 381 | h_J_seq_ftr = tf.reshape(h_J_seq_ftr, [-1, nb_nodes, hid_in]) 382 | h_P_seq_ftr = tf.reshape(h_P_seq_ftr, [-1, 10, hid_in]) 383 | h_B_seq_ftr = tf.reshape(h_B_seq_ftr, [-1, 5, hid_in]) 384 | h_I_seq_ftr = tf.reshape(h_I_seq_ftr, [-1, I_nodes, hid_in]) 385 | 386 | 387 | a_12, r_12 = CRL(h_P_seq_ftr, h_J_seq_ftr, 10, nb_nodes, hid_in) 388 | a_13, r_13 = CRL(h_B_seq_ftr, h_J_seq_ftr, 5, nb_nodes, hid_in) 389 | a_01, r_01 = CRL(h_J_seq_ftr, h_I_seq_ftr, nb_nodes, I_nodes, hid_in) 390 | a_02, r_02 = CRL(h_P_seq_ftr, h_I_seq_ftr, 10, I_nodes, hid_in) 391 | a_03, r_03 = CRL(h_B_seq_ftr, h_I_seq_ftr, 5, I_nodes, hid_in) 392 | a_23, r_23 = CRL(h_B_seq_ftr, h_P_seq_ftr, 5, 10, hid_in) 393 | 394 | 395 | 396 | h_J_seq_ftr = tf.reshape(h_J_seq_ftr, [-1, hid_in]) 397 | h_P_seq_ftr = tf.reshape(h_P_seq_ftr, [-1, hid_in]) 398 | h_B_seq_ftr = tf.reshape(h_B_seq_ftr, [-1, hid_in]) 399 | h_I_seq_ftr = tf.reshape(h_I_seq_ftr, [-1, hid_in]) 400 | 401 | if not struct_only: 402 | h_J_seq_ftr = h_J_seq_ftr + float(FLAGS.fusion_lambda) * (tf.matmul(self_r_1, W_self_1) + tf.matmul(r_12, W_cs_12) + tf.matmul(r_13, W_cs_13)) 403 | h_I_seq_ftr = h_I_seq_ftr + float(FLAGS.fusion_lambda) * (tf.matmul(self_r_0, W_self_0) + tf.matmul(r_01, W_cs_01) + tf.matmul(r_02, W_cs_02) + tf.matmul(r_03, W_cs_03)) 404 | h_P_seq_ftr = h_P_seq_ftr + float(FLAGS.fusion_lambda) * (tf.matmul(self_r_2, W_self_2) + tf.matmul(r_23, W_cs_23)) 405 | h_B_seq_ftr = h_B_seq_ftr + float(FLAGS.fusion_lambda) * (tf.matmul(self_r_3, W_self_3)) 406 | 407 | 408 | 409 | h_J_seq_ftr = tf.reshape(h_J_seq_ftr, [-1, nb_nodes, hid_out]) 410 | h_P_seq_ftr = tf.reshape(h_P_seq_ftr, [-1, 10, hid_out]) 411 | h_B_seq_ftr = tf.reshape(h_B_seq_ftr, [-1, 5, hid_out]) 412 | h_I_seq_ftr = tf.reshape(h_I_seq_ftr, [-1, I_nodes, hid_out]) 413 | 414 | return h_B_seq_ftr, h_P_seq_ftr, h_J_seq_ftr, h_I_seq_ftr 415 | 416 | if not concate: 417 | h_B_seq_ftr, h_P_seq_ftr, h_J_seq_ftr, h_I_seq_ftr = MGRN(J_in, P_in, B_in, 418 | I_in, J_bias_in, 419 | P_bias_in, 420 | B_bias_in, 421 | I_bias_in, 422 | hid_units[-1], 423 | hid_units[-1]) 424 | else: 425 | h_B_seq_ftr, h_P_seq_ftr, h_J_seq_ftr = MSGAM(J_in, P_in, B_in, J_bias_in, P_bias_in, B_bias_in, 426 | hid_units[-1] * Ps[0], hid_units[-1] * Ps[0]) 427 | h_J_seq_ftr = tf.reshape(h_J_seq_ftr, [-1, hid_units[-1]]) 428 | h_P_seq_ftr = tf.reshape(h_P_seq_ftr, [-1, hid_units[-1]]) 429 | h_B_seq_ftr = tf.reshape(h_B_seq_ftr, [-1, hid_units[-1]]) 430 | h_I_seq_ftr = tf.reshape(h_I_seq_ftr, [-1, hid_units[-1]]) 431 | 432 | ftr_in = J_in 433 | P_ftr_in = P_in 434 | B_ftr_in = B_in 435 | I_ftr_in = I_in 436 | 437 | J_seq_ftr = tf.reshape(h_J_seq_ftr, [batch_size, time_step, -1]) 438 | P_seq_ftr = tf.reshape(h_P_seq_ftr, [batch_size, time_step, -1]) 439 | B_seq_ftr = tf.reshape(h_B_seq_ftr, [batch_size, time_step, -1]) 440 | I_seq_ftr = tf.reshape(h_I_seq_ftr, [batch_size, time_step, -1]) 441 | 442 | J_T_m_ftr = [] 443 | J_Pred_tar = [] 444 | J_Test_ftr = [] 445 | J_skes_in = tf.reshape(ftr_in, [batch_size, time_step, -1]) 446 | # part-scale 447 | P_T_m_ftr = [] 448 | P_Pred_tar = [] 449 | P_Test_ftr = [] 450 | P_skes_in = tf.reshape(P_ftr_in, [batch_size, time_step, -1]) 451 | # body-scale 452 | B_T_m_ftr = [] 453 | B_Pred_tar = [] 454 | B_Test_ftr = [] 455 | B_skes_in = tf.reshape(B_ftr_in, [batch_size, time_step, -1]) 456 | # interpolation 457 | I_T_m_ftr = [] 458 | I_Pred_tar = [] 459 | I_Test_ftr = [] 460 | I_skes_in = tf.reshape(I_ftr_in, [batch_size, time_step, -1]) 461 | print_flag = False 462 | for i in range(sample_num): 463 | for k in range(1, len(k_values)+2): 464 | seq_ind = np.arange(batch_size).reshape(-1, 1) 465 | seq_ind = np.tile(seq_ind, [1, k]).reshape(-1, 1) 466 | if k <= time_step - int(pred_margin): 467 | # in order 468 | if ord_sample: 469 | T_m = np.arange(k).reshape(-1, 1) 470 | else: 471 | T_m = np.random.choice(time_step - int(pred_margin), size=[k], replace=False).reshape(-1, 1) 472 | T_m = np.sort(T_m, axis=0) 473 | if pretext == 'pre': 474 | Pred_t = T_m + int(pred_margin) 475 | T_m = T_m.astype(dtype=np.int32) 476 | Pred_t = Pred_t.astype(dtype=np.int32) 477 | T_m = np.tile(T_m, [batch_size, 1]).reshape(-1, 1) 478 | # T_m_tar.append(T_m) 479 | Pred_t = np.tile(Pred_t, [batch_size, 1]).reshape(-1, 1) 480 | 481 | T_m = np.hstack([seq_ind, T_m]) 482 | Pred_t = np.hstack([seq_ind, Pred_t]) 483 | J_sampled_seq_ftr = tf.gather_nd(J_seq_ftr, T_m) 484 | # P, B 485 | P_sampled_seq_ftr = tf.gather_nd(P_seq_ftr, T_m) 486 | B_sampled_seq_ftr = tf.gather_nd(B_seq_ftr, T_m) 487 | I_sampled_seq_ftr = tf.gather_nd(I_seq_ftr, T_m) 488 | J_Pred_t_seq = tf.gather_nd(J_skes_in, Pred_t) 489 | J_sampled_seq_ftr = tf.reshape(J_sampled_seq_ftr, [batch_size, k, -1]) 490 | J_Pred_t_seq = tf.reshape(J_Pred_t_seq, [batch_size, k, -1]) 491 | # P,B 492 | P_Pred_t_seq = tf.gather_nd(P_skes_in, Pred_t) 493 | P_sampled_seq_ftr = tf.reshape(P_sampled_seq_ftr, [batch_size, k, -1]) 494 | P_Pred_t_seq = tf.reshape(P_Pred_t_seq, [batch_size, k, -1]) 495 | B_Pred_t_seq = tf.gather_nd(B_skes_in, Pred_t) 496 | B_sampled_seq_ftr = tf.reshape(B_sampled_seq_ftr, [batch_size, k, -1]) 497 | B_Pred_t_seq = tf.reshape(B_Pred_t_seq, [batch_size, k, -1]) 498 | I_Pred_t_seq = tf.gather_nd(I_skes_in, Pred_t) 499 | I_sampled_seq_ftr = tf.reshape(I_sampled_seq_ftr, [batch_size, k, -1]) 500 | I_Pred_t_seq = tf.reshape(I_Pred_t_seq, [batch_size, k, -1]) 501 | 502 | J_T_m_ftr.append(J_sampled_seq_ftr) 503 | J_Pred_tar.append(J_Pred_t_seq) 504 | # 505 | P_T_m_ftr.append(P_sampled_seq_ftr) 506 | P_Pred_tar.append(P_Pred_t_seq) 507 | # 508 | B_T_m_ftr.append(B_sampled_seq_ftr) 509 | B_Pred_tar.append(B_Pred_t_seq) 510 | I_T_m_ftr.append(I_sampled_seq_ftr) 511 | I_Pred_tar.append(I_Pred_t_seq) 512 | 513 | if i == 0: 514 | T_m_test = np.arange(k).reshape(-1, 1) 515 | # print(T_m_test) 516 | T_m_test = T_m_test.astype(dtype=np.int32) 517 | T_m_test = np.tile(T_m_test, [batch_size, 1]).reshape(-1, 1) 518 | T_m_test = np.hstack([seq_ind, T_m_test]) 519 | J_test_seq_ftr = tf.gather_nd(J_seq_ftr, T_m_test) 520 | J_test_seq_ftr = tf.reshape(J_test_seq_ftr, [batch_size, k, -1]) 521 | J_Test_ftr.append(J_test_seq_ftr) 522 | # 523 | P_test_seq_ftr = tf.gather_nd(P_seq_ftr, T_m_test) 524 | P_test_seq_ftr = tf.reshape(P_test_seq_ftr, [batch_size, k, -1]) 525 | P_Test_ftr.append(P_test_seq_ftr) 526 | B_test_seq_ftr = tf.gather_nd(B_seq_ftr, T_m_test) 527 | B_test_seq_ftr = tf.reshape(B_test_seq_ftr, [batch_size, k, -1]) 528 | B_Test_ftr.append(B_test_seq_ftr) 529 | I_test_seq_ftr = tf.gather_nd(I_seq_ftr, T_m_test) 530 | I_test_seq_ftr = tf.reshape(I_test_seq_ftr, [batch_size, k, -1]) 531 | I_Test_ftr.append(I_test_seq_ftr) 532 | 533 | 534 | 535 | if FLAGS.bi == '1': 536 | for i in range(sample_num): 537 | for k in range(1, len(k_values) + 2): 538 | seq_ind = np.arange(batch_size).reshape(-1, 1) 539 | seq_ind = np.tile(seq_ind, [1, k]).reshape(-1, 1) 540 | if k <= time_step - int(pred_margin): 541 | # in order 542 | if ord_sample: 543 | T_m = np.arange(k).reshape(-1, 1) 544 | else: 545 | T_m = np.random.choice(time_step - int(pred_margin), size=[k], replace=False).reshape(-1, 1) 546 | # print (T_m) 547 | # print(T_m.shape) 548 | # if not random_sample: 549 | T_m = np.sort(T_m, axis=0) 550 | # Reverse 551 | T_m = np.sort(-T_m) 552 | T_m += time_step 553 | if pretext == 'pre': 554 | Pred_t = T_m - int(pred_margin) 555 | # no used 556 | # elif pretext == 'recon': 557 | # Pred_t = T_m 558 | # elif pretext == 'rev': 559 | # Pred_t = np.sort(-T_m) 560 | # Pred_t = -Pred_t 561 | print(T_m) 562 | print(Pred_t) 563 | T_m = T_m.astype(dtype=np.int32) 564 | Pred_t = Pred_t.astype(dtype=np.int32) 565 | T_m = np.tile(T_m, [batch_size]).reshape(-1, 1) 566 | # T_m_tar.append(T_m) 567 | Pred_t = np.tile(Pred_t, [batch_size]).reshape(-1, 1) 568 | 569 | T_m = np.hstack([seq_ind, T_m]) 570 | Pred_t = np.hstack([seq_ind, Pred_t]) 571 | J_sampled_seq_ftr = tf.gather_nd(J_seq_ftr, T_m) 572 | # P, B 573 | P_sampled_seq_ftr = tf.gather_nd(P_seq_ftr, T_m) 574 | B_sampled_seq_ftr = tf.gather_nd(B_seq_ftr, T_m) 575 | I_sampled_seq_ftr = tf.gather_nd(I_seq_ftr, T_m) 576 | # 577 | J_Pred_t_seq = tf.gather_nd(J_skes_in, Pred_t) 578 | J_sampled_seq_ftr = tf.reshape(J_sampled_seq_ftr, [batch_size, k, -1]) 579 | J_Pred_t_seq = tf.reshape(J_Pred_t_seq, [batch_size, k, -1]) 580 | # P,B 581 | P_Pred_t_seq = tf.gather_nd(P_skes_in, Pred_t) 582 | P_sampled_seq_ftr = tf.reshape(P_sampled_seq_ftr, [batch_size, k, -1]) 583 | P_Pred_t_seq = tf.reshape(P_Pred_t_seq, [batch_size, k, -1]) 584 | B_Pred_t_seq = tf.gather_nd(B_skes_in, Pred_t) 585 | B_sampled_seq_ftr = tf.reshape(B_sampled_seq_ftr, [batch_size, k, -1]) 586 | B_Pred_t_seq = tf.reshape(B_Pred_t_seq, [batch_size, k, -1]) 587 | I_Pred_t_seq = tf.gather_nd(I_skes_in, Pred_t) 588 | I_sampled_seq_ftr = tf.reshape(I_sampled_seq_ftr, [batch_size, k, -1]) 589 | I_Pred_t_seq = tf.reshape(I_Pred_t_seq, [batch_size, k, -1]) 590 | # 591 | # sorted random frames 592 | J_T_m_ftr.append(J_sampled_seq_ftr) 593 | J_Pred_tar.append(J_Pred_t_seq) 594 | # 595 | P_T_m_ftr.append(P_sampled_seq_ftr) 596 | P_Pred_tar.append(P_Pred_t_seq) 597 | # 598 | B_T_m_ftr.append(B_sampled_seq_ftr) 599 | B_Pred_tar.append(B_Pred_t_seq) 600 | I_T_m_ftr.append(I_sampled_seq_ftr) 601 | I_Pred_tar.append(I_Pred_t_seq) 602 | 603 | if FLAGS.bi == '1': 604 | sample_num = sample_num * 2 605 | 606 | with tf.name_scope("MSR"), tf.variable_scope("MSR", reuse=tf.AUTO_REUSE): 607 | J_cell = tf.contrib.rnn.MultiRNNCell([tf.contrib.rnn.LSTMCell(LSTM_embed) for _ in range(num_layers)]) 608 | P_cell = tf.contrib.rnn.MultiRNNCell([tf.contrib.rnn.LSTMCell(LSTM_embed) for _ in range(num_layers)]) 609 | B_cell = tf.contrib.rnn.MultiRNNCell([tf.contrib.rnn.LSTMCell(LSTM_embed) for _ in range(num_layers)]) 610 | I_cell = tf.contrib.rnn.MultiRNNCell([tf.contrib.rnn.LSTMCell(LSTM_embed) for _ in range(num_layers)]) 611 | 612 | J_all_pred_loss = [] 613 | # P, B 614 | P_all_pred_loss = [] 615 | B_all_pred_loss = [] 616 | I_all_pred_loss = [] 617 | # 618 | a0_all_pred_loss = [] 619 | a1_all_pred_loss = [] 620 | a2_all_pred_loss = [] 621 | # all_mask_loss = [] 622 | J_en_outs = [] 623 | J_en_outs_whole = [] 624 | J_en_outs_test = [] 625 | # 626 | P_en_outs = [] 627 | P_en_outs_whole = [] 628 | P_en_outs_test = [] 629 | B_en_outs = [] 630 | B_en_outs_whole = [] 631 | B_en_outs_test = [] 632 | I_en_outs = [] 633 | I_en_outs_whole = [] 634 | I_en_outs_test = [] 635 | # 636 | ske_en_outs = [] 637 | # all_mask_acc = [] 638 | with tf.name_scope("J_pred"), tf.variable_scope("J_pred", reuse=tf.AUTO_REUSE): 639 | J_W1_pred = tf.Variable(tf.random_normal([LSTM_embed, LSTM_embed])) 640 | J_b1_pred = tf.Variable(tf.zeros(shape=[LSTM_embed, ])) 641 | J_W2_pred = tf.Variable(tf.random_normal([LSTM_embed, nb_nodes * 3])) 642 | J_b2_pred = tf.Variable(tf.zeros(shape=[nb_nodes * 3, ])) 643 | P_W1_pred = tf.Variable(tf.random_normal([LSTM_embed, LSTM_embed])) 644 | P_b1_pred = tf.Variable(tf.zeros(shape=[LSTM_embed, ])) 645 | P_W2_pred = tf.Variable(tf.random_normal([LSTM_embed, 10 * 3])) 646 | P_b2_pred = tf.Variable(tf.zeros(shape=[10 * 3, ])) 647 | B_W1_pred = tf.Variable(tf.random_normal([LSTM_embed, LSTM_embed])) 648 | B_b1_pred = tf.Variable(tf.zeros(shape=[LSTM_embed, ])) 649 | B_W2_pred = tf.Variable(tf.random_normal([LSTM_embed, 5 * 3])) 650 | B_b2_pred = tf.Variable(tf.zeros(shape=[5 * 3, ])) 651 | 652 | 653 | for i in range((levels - int(pred_margin)) * sample_num): 654 | J_sampled_seq_ftr = J_T_m_ftr[i] 655 | J_encoder_output, J_encoder_state = tf.nn.dynamic_rnn(J_cell, J_sampled_seq_ftr, dtype=tf.float32) 656 | 657 | J_encoder_output = tf.reshape(J_encoder_output, [-1, LSTM_embed]) 658 | # Skeleton-level Prediction 659 | J_pred_embedding_J = tf.nn.relu(tf.matmul(J_encoder_output, J_W1_pred) + J_b1_pred) 660 | # 661 | J_pred_skeleton = tf.matmul(J_pred_embedding_J, J_W2_pred) + J_b2_pred 662 | 663 | J_pred_skeleton = tf.reshape(J_pred_skeleton, [batch_size, k_values[i % levels], nb_nodes * 3]) 664 | 665 | 666 | # P, B 667 | P_pred_embedding_J = tf.nn.relu(tf.matmul(J_encoder_output, P_W1_pred) + P_b1_pred) 668 | P_pred_skeleton = tf.matmul(P_pred_embedding_J, P_W2_pred) + P_b2_pred 669 | P_pred_skeleton = tf.reshape(P_pred_skeleton, [batch_size, k_values[i % levels], 10 * 3]) 670 | B_pred_embedding_J = tf.nn.relu(tf.matmul(J_encoder_output, B_W1_pred) + B_b1_pred) 671 | B_pred_skeleton = tf.matmul(B_pred_embedding_J, B_W2_pred) + B_b2_pred 672 | B_pred_skeleton = tf.reshape(B_pred_skeleton, [batch_size, k_values[i % levels], 5 * 3]) 673 | if FLAGS.loss == 'l1': 674 | J_pred_loss = tf.reduce_mean(tf.losses.absolute_difference(J_pred_skeleton, J_Pred_tar[i])) 675 | # P, B 676 | # tf.nn.l1_loss 677 | P_pred_loss = tf.reduce_mean(tf.losses.absolute_difference(P_pred_skeleton, P_Pred_tar[i])) 678 | B_pred_loss = tf.reduce_mean(tf.losses.absolute_difference(B_pred_skeleton, B_Pred_tar[i])) 679 | elif FLAGS.loss == 'MSE': 680 | J_pred_loss = tf.reduce_mean(tf.losses.mean_squared_error(J_pred_skeleton, J_Pred_tar[i])) 681 | # P, B 682 | # tf.nn.l1_loss 683 | P_pred_loss = tf.reduce_mean(tf.losses.mean_squared_error(P_pred_skeleton, P_Pred_tar[i])) 684 | B_pred_loss = tf.reduce_mean(tf.losses.mean_squared_error(B_pred_skeleton, B_Pred_tar[i])) 685 | elif FLAGS.loss == 'l2': 686 | J_pred_loss = tf.reduce_mean(tf.nn.l2_loss(J_pred_skeleton - J_Pred_tar[i])) 687 | # P, B 688 | # tf.nn.l1_loss 689 | P_pred_loss = tf.reduce_mean(tf.nn.l2_loss(P_pred_skeleton - P_Pred_tar[i])) 690 | B_pred_loss = tf.reduce_mean(tf.nn.l2_loss(B_pred_skeleton - B_Pred_tar[i])) 691 | 692 | if k_filter == -1 or (k_filter !=-1 and i == 0): 693 | if no_multi_pred: 694 | J_all_pred_loss.append(J_pred_loss) 695 | else: 696 | J_all_pred_loss.append(J_pred_loss+P_pred_loss+B_pred_loss) 697 | 698 | 699 | J_encoder_output = tf.reshape(J_encoder_output, [batch_size, k_values[i % levels], -1]) 700 | # Average 701 | J_en_outs.append(J_encoder_output[:, -1, :]) 702 | # en_outs.append(encoder_output) 703 | # if i == sample_num * levels - 1: 704 | # J_en_outs_whole = J_encoder_output 705 | for i in range(levels+1): 706 | J_test_seq_ftr = J_Test_ftr[i] 707 | J_encoder_output, J_encoder_state = tf.nn.dynamic_rnn(J_cell, J_test_seq_ftr, dtype=tf.float32) 708 | J_encoder_output = tf.reshape(J_encoder_output, [-1, LSTM_embed]) 709 | # Skeleton-level Prediction 710 | J_encoder_output = tf.reshape(J_encoder_output, [batch_size, i+1, -1]) 711 | # en_outs_test.append(encoder_output) 712 | J_en_outs_test.append(J_encoder_output[:, -1, :]) 713 | 714 | with tf.name_scope("P_pred"), tf.variable_scope("P_pred", reuse=tf.AUTO_REUSE): 715 | P_W1_pred = tf.Variable(tf.random_normal([LSTM_embed, LSTM_embed])) 716 | P_b1_pred = tf.Variable(tf.zeros(shape=[LSTM_embed, ])) 717 | P_W2_pred = tf.Variable(tf.random_normal([LSTM_embed, 10 * 3])) 718 | P_b2_pred = tf.Variable(tf.zeros(shape=[10 * 3, ])) 719 | B_W1_pred = tf.Variable(tf.random_normal([LSTM_embed, LSTM_embed])) 720 | B_b1_pred = tf.Variable(tf.zeros(shape=[LSTM_embed, ])) 721 | B_W2_pred = tf.Variable(tf.random_normal([LSTM_embed, 5 * 3])) 722 | B_b2_pred = tf.Variable(tf.zeros(shape=[5 * 3, ])) 723 | for i in range((levels - int(pred_margin)) * sample_num): 724 | # if k_filter !=-1 and i != 0: 725 | # continue 726 | P_sampled_seq_ftr = P_T_m_ftr[i] 727 | P_encoder_output, P_encoder_state = tf.nn.dynamic_rnn(P_cell, P_sampled_seq_ftr, dtype=tf.float32) 728 | 729 | P_encoder_output = tf.reshape(P_encoder_output, [-1, LSTM_embed]) 730 | # Skeleton-level Prediction 731 | P_pred_embedding_P = tf.nn.relu(tf.matmul(P_encoder_output, P_W1_pred) + P_b1_pred) 732 | 733 | P_pred_skeleton = tf.matmul(P_pred_embedding_P, P_W2_pred) + P_b2_pred 734 | 735 | P_pred_skeleton = tf.reshape(P_pred_skeleton, [batch_size, k_values[i % levels], 10 * 3]) 736 | 737 | B_pred_embedding_P = tf.nn.relu(tf.matmul(P_encoder_output, B_W1_pred) + B_b1_pred) 738 | B_pred_skeleton = tf.matmul(B_pred_embedding_P, B_W2_pred) + B_b2_pred 739 | B_pred_skeleton = tf.reshape(B_pred_skeleton, [batch_size, k_values[i % levels], 5 * 3]) 740 | 741 | if k_filter == -1 or (k_filter !=-1 and i == 0): 742 | if FLAGS.loss == 'l1': 743 | P_pred_loss = tf.reduce_mean(tf.losses.absolute_difference(P_pred_skeleton, P_Pred_tar[i])) 744 | B_pred_loss = tf.reduce_mean(tf.losses.absolute_difference(B_pred_skeleton, B_Pred_tar[i])) 745 | elif FLAGS.loss == 'MSE': 746 | P_pred_loss = tf.reduce_mean(tf.losses.mean_squared_error(P_pred_skeleton, P_Pred_tar[i])) 747 | B_pred_loss = tf.reduce_mean(tf.losses.mean_squared_error(B_pred_skeleton, B_Pred_tar[i])) 748 | elif FLAGS.loss == 'l2': 749 | P_pred_loss = tf.reduce_mean(tf.nn.l2_loss(P_pred_skeleton - P_Pred_tar[i])) 750 | B_pred_loss = tf.reduce_mean(tf.nn.l2_loss(B_pred_skeleton - B_Pred_tar[i])) 751 | if no_multi_pred: 752 | P_all_pred_loss.append(P_pred_loss) 753 | else: 754 | P_all_pred_loss.append(P_pred_loss+B_pred_loss) 755 | P_encoder_output = tf.reshape(P_encoder_output, [batch_size, k_values[i % levels], -1]) 756 | # Average 757 | P_en_outs.append(P_encoder_output[:, -1, :]) 758 | 759 | for i in range(levels + 1): 760 | P_test_seq_ftr = P_Test_ftr[i] 761 | P_encoder_output, P_encoder_state = tf.nn.dynamic_rnn(P_cell, P_test_seq_ftr, dtype=tf.float32) 762 | P_encoder_output = tf.reshape(P_encoder_output, [-1, LSTM_embed]) 763 | P_encoder_output = tf.reshape(P_encoder_output, [batch_size, i+1, -1]) 764 | P_en_outs_test.append(P_encoder_output[:, -1, :]) 765 | with tf.name_scope("B_pred"), tf.variable_scope("B_pred", reuse=tf.AUTO_REUSE): 766 | B_W1_pred = tf.Variable(tf.random_normal([LSTM_embed, LSTM_embed])) 767 | B_b1_pred = tf.Variable(tf.zeros(shape=[LSTM_embed, ])) 768 | B_W2_pred = tf.Variable(tf.random_normal([LSTM_embed, 5 * 3])) 769 | B_b2_pred = tf.Variable(tf.zeros(shape=[5 * 3, ])) 770 | 771 | for i in range((levels - int(pred_margin)) * sample_num): 772 | B_sampled_seq_ftr = B_T_m_ftr[i] 773 | B_encoder_output, B_encoder_state = tf.nn.dynamic_rnn(B_cell, B_sampled_seq_ftr, dtype=tf.float32) 774 | 775 | B_encoder_output = tf.reshape(B_encoder_output, [-1, LSTM_embed]) 776 | # Skeleton-level Prediction 777 | B_pred_embedding_B = tf.nn.relu(tf.matmul(B_encoder_output, B_W1_pred) + B_b1_pred) 778 | 779 | 780 | B_pred_skeleton = tf.matmul(B_pred_embedding_B, B_W2_pred) + B_b2_pred 781 | B_pred_skeleton = tf.reshape(B_pred_skeleton, [batch_size, k_values[i % levels], 5 * 3]) 782 | # tf.nn.l1_loss 783 | if k_filter == -1 or (k_filter !=-1 and i == 0): 784 | if FLAGS.loss == 'l1': 785 | B_pred_loss = tf.reduce_mean(tf.losses.absolute_difference(B_pred_skeleton, B_Pred_tar[i])) 786 | elif FLAGS.loss == 'MSE': 787 | B_pred_loss = tf.reduce_mean(tf.losses.mean_squared_error(B_pred_skeleton, B_Pred_tar[i])) 788 | elif FLAGS.loss == 'l2': 789 | B_pred_loss = tf.reduce_mean(tf.nn.l2_loss(B_pred_skeleton - B_Pred_tar[i])) 790 | B_all_pred_loss.append(B_pred_loss) 791 | B_encoder_output = tf.reshape(B_encoder_output, [batch_size, k_values[i % levels], -1]) 792 | # Average 793 | B_en_outs.append(B_encoder_output[:, -1, :]) 794 | 795 | for i in range(levels + 1): 796 | B_test_seq_ftr = B_Test_ftr[i] 797 | B_encoder_output, B_encoder_state = tf.nn.dynamic_rnn(B_cell, B_test_seq_ftr, dtype=tf.float32) 798 | B_encoder_output = tf.reshape(B_encoder_output, [-1, LSTM_embed]) 799 | B_encoder_output = tf.reshape(B_encoder_output, [batch_size, i+1, -1]) 800 | B_en_outs_test.append(B_encoder_output[:, -1, :]) 801 | with tf.name_scope("I_pred"), tf.variable_scope("I_pred", reuse=tf.AUTO_REUSE): 802 | J_W1_pred = tf.Variable(tf.random_normal([LSTM_embed, LSTM_embed])) 803 | J_b1_pred = tf.Variable(tf.zeros(shape=[LSTM_embed, ])) 804 | J_W2_pred = tf.Variable(tf.random_normal([LSTM_embed, nb_nodes * 3])) 805 | J_b2_pred = tf.Variable(tf.zeros(shape=[nb_nodes * 3, ])) 806 | P_W1_pred = tf.Variable(tf.random_normal([LSTM_embed, LSTM_embed])) 807 | P_b1_pred = tf.Variable(tf.zeros(shape=[LSTM_embed, ])) 808 | P_W2_pred = tf.Variable(tf.random_normal([LSTM_embed, 10 * 3])) 809 | P_b2_pred = tf.Variable(tf.zeros(shape=[10 * 3, ])) 810 | B_W1_pred = tf.Variable(tf.random_normal([LSTM_embed, LSTM_embed])) 811 | B_b1_pred = tf.Variable(tf.zeros(shape=[LSTM_embed, ])) 812 | B_W2_pred = tf.Variable(tf.random_normal([LSTM_embed, 5 * 3])) 813 | B_b2_pred = tf.Variable(tf.zeros(shape=[5 * 3, ])) 814 | I_W1_pred = tf.Variable(tf.random_normal([LSTM_embed, LSTM_embed])) 815 | I_b1_pred = tf.Variable(tf.zeros(shape=[LSTM_embed, ])) 816 | I_W2_pred = tf.Variable(tf.random_normal([LSTM_embed, I_nodes * 3])) 817 | I_b2_pred = tf.Variable(tf.zeros(shape=[I_nodes * 3, ])) 818 | for i in range((levels - int(pred_margin)) * sample_num): 819 | # if k_filter == time_step and i != 0: 820 | # continue 821 | I_sampled_seq_ftr = I_T_m_ftr[i] 822 | 823 | I_encoder_output, I_encoder_state = tf.nn.dynamic_rnn(I_cell, I_sampled_seq_ftr, dtype=tf.float32) 824 | I_encoder_output = tf.reshape(I_encoder_output, [-1, LSTM_embed]) 825 | # Skeleton-level Prediction 826 | I_pred_embedding_I = tf.nn.relu(tf.matmul(I_encoder_output, I_W1_pred) + I_b1_pred) 827 | 828 | I_pred_skeleton = tf.matmul(I_pred_embedding_I, I_W2_pred) + I_b2_pred 829 | I_pred_skeleton = tf.reshape(I_pred_skeleton, [batch_size, k_values[i % levels], I_nodes * 3]) 830 | # tf.nn.l1_loss 831 | J_pred_embedding_I = tf.nn.relu(tf.matmul(I_encoder_output, J_W1_pred) + J_b1_pred) 832 | J_pred_skeleton = tf.matmul(J_pred_embedding_I, J_W2_pred) + J_b2_pred 833 | J_pred_skeleton = tf.reshape(J_pred_skeleton, [batch_size, k_values[i % levels], nb_nodes * 3]) 834 | 835 | B_pred_embedding_I = tf.nn.relu(tf.matmul(I_encoder_output, B_W1_pred) + B_b1_pred) 836 | B_pred_skeleton = tf.matmul(B_pred_embedding_I, B_W2_pred) + B_b2_pred 837 | B_pred_skeleton = tf.reshape(B_pred_skeleton, [batch_size, k_values[i % levels], 5 * 3]) 838 | P_pred_embedding_I = tf.nn.relu(tf.matmul(I_encoder_output, P_W1_pred) + P_b1_pred) 839 | P_pred_skeleton = tf.matmul(P_pred_embedding_I, P_W2_pred) + P_b2_pred 840 | P_pred_skeleton = tf.reshape(P_pred_skeleton, [batch_size, k_values[i % levels], 10 * 3]) 841 | # 842 | if k_filter == -1 or (k_filter !=-1 and i == 0): 843 | if FLAGS.loss == 'l1': 844 | I_pred_loss = tf.reduce_mean(tf.losses.absolute_difference(I_pred_skeleton, I_Pred_tar[i])) 845 | J_pred_loss = tf.reduce_mean(tf.losses.absolute_difference(J_pred_skeleton, J_Pred_tar[i])) 846 | P_pred_loss = tf.reduce_mean(tf.losses.absolute_difference(P_pred_skeleton, P_Pred_tar[i])) 847 | B_pred_loss = tf.reduce_mean(tf.losses.absolute_difference(B_pred_skeleton, B_Pred_tar[i])) 848 | elif FLAGS.loss == 'MSE': 849 | I_pred_loss = tf.reduce_mean(tf.losses.mean_squared_error(I_pred_skeleton, I_Pred_tar[i])) 850 | J_pred_loss = tf.reduce_mean(tf.losses.mean_squared_error(J_pred_skeleton, J_Pred_tar[i])) 851 | P_pred_loss = tf.reduce_mean(tf.losses.mean_squared_error(P_pred_skeleton, P_Pred_tar[i])) 852 | B_pred_loss = tf.reduce_mean(tf.losses.mean_squared_error(B_pred_skeleton, B_Pred_tar[i])) 853 | elif FLAGS.loss == 'l2': 854 | I_pred_loss = tf.reduce_mean(tf.nn.l2_loss(I_pred_skeleton - I_Pred_tar[i])) 855 | J_pred_loss = tf.reduce_mean(tf.nn.l2_loss(J_pred_skeleton - J_Pred_tar[i])) 856 | P_pred_loss = tf.reduce_mean(tf.nn.l2_loss(P_pred_skeleton - P_Pred_tar[i])) 857 | B_pred_loss = tf.reduce_mean(tf.nn.l2_loss(B_pred_skeleton - B_Pred_tar[i])) 858 | if no_multi_pred: 859 | I_all_pred_loss.append(I_pred_loss) 860 | else: 861 | I_all_pred_loss.append(P_pred_loss + J_pred_loss + I_pred_loss + B_pred_loss) 862 | I_encoder_output = tf.reshape(I_encoder_output, [batch_size, k_values[i % levels], -1]) 863 | # Average 864 | I_en_outs.append(I_encoder_output[:, -1, :]) 865 | for i in range(levels + 1): 866 | I_test_seq_ftr = I_Test_ftr[i] 867 | I_encoder_output, I_encoder_state = tf.nn.dynamic_rnn(I_cell, I_test_seq_ftr, dtype=tf.float32) 868 | I_encoder_output = tf.reshape(I_encoder_output, [-1, LSTM_embed]) 869 | I_encoder_output = tf.reshape(I_encoder_output, [batch_size, i+1, -1]) 870 | I_en_outs_test.append(I_encoder_output[:, -1, :]) 871 | # en_outs.append(tf.reshape(encoder_output, [batch_size, k_values[i], -1])) 872 | J_pred_opt = tf.train.AdamOptimizer(learning_rate=lr) 873 | J_pred_train_op = J_pred_opt.minimize(tf.reduce_mean(J_all_pred_loss)) 874 | P_pred_opt = tf.train.AdamOptimizer(learning_rate=lr) 875 | P_pred_train_op = P_pred_opt.minimize(tf.reduce_mean(P_all_pred_loss)) 876 | B_pred_opt = tf.train.AdamOptimizer(learning_rate=lr) 877 | B_pred_train_op = B_pred_opt.minimize(tf.reduce_mean(B_all_pred_loss)) 878 | I_pred_opt = tf.train.AdamOptimizer(learning_rate=lr) 879 | I_pred_train_op = I_pred_opt.minimize(tf.reduce_mean(I_all_pred_loss)) 880 | 881 | 882 | with tf.name_scope("Recognition"), tf.variable_scope("Recognition", reuse=tf.AUTO_REUSE): 883 | # 884 | if no_interp: 885 | en_to_pred = tf.concat([J_en_outs_test[0], P_en_outs_test[0], 886 | B_en_outs_test[0]], axis=-1) 887 | for i in range(1, levels + 1): 888 | temp = tf.concat([J_en_outs_test[i], P_en_outs_test[i], 889 | B_en_outs_test[i]], axis=-1) 890 | en_to_pred = tf.concat([en_to_pred, temp], axis=0) 891 | elif not only_ske_embed: 892 | en_to_pred = tf.concat([I_en_outs_test[0], J_en_outs_test[0], P_en_outs_test[0], 893 | B_en_outs_test[0]], axis=-1) 894 | for i in range(1, levels+1): 895 | temp = tf.concat([I_en_outs_test[i], J_en_outs_test[i], P_en_outs_test[i], 896 | B_en_outs_test[i]], axis=-1) 897 | en_to_pred = tf.concat([en_to_pred, temp], axis=0) 898 | else: 899 | en_to_pred = tf.concat([J_en_outs_test[0]], axis=-1) 900 | for i in range(1, levels + 1): 901 | temp = tf.concat([J_en_outs_test[i]], axis=-1) 902 | en_to_pred = tf.concat([en_to_pred, temp], axis=0) 903 | 904 | # Frozen 905 | if frozen == '1': 906 | en_to_pred = tf.stop_gradient(en_to_pred) 907 | 908 | if no_interp: 909 | # original 910 | W_1 = tf.Variable(tf.random_normal([LSTM_embed * 3, LSTM_embed * 3])) 911 | b_1 = tf.Variable(tf.zeros(shape=[LSTM_embed * 3, ])) 912 | W_2 = tf.Variable(tf.random_normal([LSTM_embed * 3, nb_classes])) 913 | b_2 = tf.Variable(tf.zeros(shape=[nb_classes, ])) 914 | elif not only_ske_embed: 915 | # original 916 | W_1 = tf.Variable(tf.random_normal([LSTM_embed * 4, LSTM_embed * 4])) 917 | b_1 = tf.Variable(tf.zeros(shape=[LSTM_embed * 4, ])) 918 | W_2 = tf.Variable(tf.random_normal([LSTM_embed * 4, nb_classes])) 919 | b_2 = tf.Variable(tf.zeros(shape=[nb_classes, ])) 920 | else: 921 | W_1 = tf.Variable(tf.random_normal([LSTM_embed, LSTM_embed])) 922 | b_1 = tf.Variable(tf.zeros(shape=[LSTM_embed, ])) 923 | W_2 = tf.Variable(tf.random_normal([LSTM_embed, nb_classes])) 924 | b_2 = tf.Variable(tf.zeros(shape=[nb_classes, ])) 925 | 926 | # original 927 | logits = tf.matmul(tf.nn.relu(tf.matmul(en_to_pred, W_1) + b_1), W_2) + b_2 928 | logits_pred = tf.matmul(tf.nn.relu(tf.matmul(en_to_pred, W_1) + b_1), W_2) + b_2 929 | 930 | 931 | log_resh = tf.reshape(logits, [-1, nb_classes]) 932 | lab_resh = tf.reshape(lbl_in, [-1, nb_classes]) 933 | 934 | 935 | if not last_pre: 936 | aver_pred = logits[:batch_size] 937 | aver_final_pred = logits_pred[:batch_size] 938 | for i in range(1, levels+1): 939 | aver_pred += logits[batch_size*i:batch_size*(i+1)] 940 | aver_final_pred += logits_pred[batch_size * i:batch_size * (i + 1)] 941 | else: 942 | aver_pred = logits[-batch_size:] 943 | aver_final_pred = logits_pred[-batch_size:] 944 | 945 | correct_pred = tf.equal(tf.argmax(aver_pred, -1), tf.argmax(lab_resh, -1)) 946 | accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) 947 | 948 | correct_final_pred = tf.equal(tf.argmax(aver_final_pred, -1), tf.argmax(lab_resh, -1)) 949 | accuracy_final = tf.reduce_mean(tf.cast(correct_final_pred, tf.float32)) 950 | 951 | loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=aver_pred, labels=lab_resh)) 952 | 953 | # frozen 954 | opt = tf.train.AdamOptimizer(learning_rate=lr) 955 | train_op = opt.minimize(loss) 956 | 957 | saver = tf.train.Saver() 958 | 959 | init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) 960 | 961 | vlss_mn = np.inf 962 | vacc_mx = 0.0 963 | vnAUC_mx = 0.0 964 | curr_step = 0 965 | 966 | X_train = X_train_J 967 | X_test = X_test_J 968 | with tf.Session(config=config) as sess: 969 | sess.run(init_op) 970 | 971 | train_loss_avg = 0 972 | train_acc_avg = 0 973 | val_loss_avg = 0 974 | val_acc_avg = 0 975 | 976 | for epoch in range(pre_epochs): 977 | tr_step = 0 978 | tr_size = X_train.shape[0] 979 | while tr_step * batch_size < tr_size: 980 | if (tr_step + 1) * batch_size > tr_size: 981 | break 982 | X_input_J = X_train_J[tr_step * batch_size:(tr_step + 1) * batch_size] 983 | X_input_J = X_input_J.reshape([-1, nb_nodes, 3]) 984 | X_input_P = X_train_P[tr_step * batch_size:(tr_step + 1) * batch_size] 985 | X_input_P = X_input_P.reshape([-1, 10, 3]) 986 | X_input_B = X_train_B[tr_step * batch_size:(tr_step + 1) * batch_size] 987 | X_input_B = X_input_B.reshape([-1, 5, 3]) 988 | # interpolation 989 | X_input_I = X_train_I[tr_step * batch_size:(tr_step + 1) * batch_size] 990 | X_input_I = X_input_I.reshape([-1, I_nodes, 3]) 991 | loss_rec, loss_attr, acc_attr, loss_pred = 0, 0, 0, 0 992 | if no_interp: 993 | _, _, _, loss_pred, P_loss_pred, B_loss_pred, I_loss_pred, \ 994 | = sess.run([J_pred_train_op, P_pred_train_op, 995 | B_pred_train_op, 996 | J_pred_loss, P_pred_loss, B_pred_loss, I_pred_loss, 997 | ], 998 | feed_dict={ 999 | J_in: X_input_J, 1000 | P_in: X_input_P, 1001 | B_in: X_input_B, 1002 | I_in: X_input_I, 1003 | J_bias_in: biases_J, 1004 | P_bias_in: biases_P, 1005 | B_bias_in: biases_B, 1006 | I_bias_in: biases_I, 1007 | lbl_in: y_train[tr_step * batch_size:(tr_step + 1) * batch_size], 1008 | is_train: True, 1009 | attn_drop: 0.0, ffd_drop: 0.0}) 1010 | else: 1011 | _, _, _, _, loss_pred, P_loss_pred, B_loss_pred, I_loss_pred, \ 1012 | = sess.run([J_pred_train_op, P_pred_train_op, 1013 | B_pred_train_op, I_pred_train_op, 1014 | J_pred_loss, P_pred_loss, B_pred_loss, I_pred_loss, 1015 | ], 1016 | feed_dict={ 1017 | J_in: X_input_J, 1018 | P_in: X_input_P, 1019 | B_in: X_input_B, 1020 | I_in: X_input_I, 1021 | J_bias_in: biases_J, 1022 | P_bias_in: biases_P, 1023 | B_bias_in: biases_B, 1024 | I_bias_in: biases_I, 1025 | lbl_in: y_train[tr_step * batch_size:(tr_step + 1) * batch_size], 1026 | is_train: True, 1027 | attn_drop: 0.0, ffd_drop: 0.0}) 1028 | tr_step += 1 1029 | print('[%s / %s] Train Loss (I. Pre.): %.5f | (J. Pre.): %.5f | (P. Pre.): %.5f | (B. Pre.): %.5f' % 1030 | (str(epoch), str(pre_epochs), I_loss_pred, loss_pred, P_loss_pred, B_loss_pred)) 1031 | if (epoch + 1) % 40 == 0: 1032 | checkpt_file = pre_dir + '_' + str(fusion_lambda) + '-' + pretext + '-' + str(nhood) + '-' + str( 1033 | time_step) + \ 1034 | '-' + dataset + '/' + split + \ 1035 | '_' + str(epoch + 1) + '_' + str(s_range) + '_' + str(pred_margin) + '_' + FLAGS.reverse \ 1036 | + view_dir + change + '.ckpt' 1037 | print(checkpt_file) 1038 | saver.save(sess, checkpt_file) 1039 | checkpt_file = pre_dir + '_' + str(fusion_lambda) + '-' + pretext + '-' + str(nhood) + '-' + str( 1040 | time_step) + \ 1041 | '-' + dataset + '/' + split + \ 1042 | '_' + str(pre_epochs) + '_' + str(s_range) + '_' + str(pred_margin) + '_' + FLAGS.reverse +\ 1043 | view_dir + change + '.ckpt' 1044 | saver.save(sess, checkpt_file) 1045 | 1046 | if split == 'B' and dataset == 'IAS': 1047 | checkpt_file = pre_dir + '_' + str(fusion_lambda) + '-' + pretext + '-' + str(nhood) + '-' + str(time_step) + \ 1048 | '-' + dataset + '/A' + \ 1049 | '_' + str(pre_epochs) + '_' + str(s_range) + '_' + str(pred_margin) + '_' + FLAGS.reverse + view_dir + change + '.ckpt' 1050 | else: 1051 | checkpt_file = pre_dir + '_' + str(fusion_lambda) + '-' + pretext + '-' + str(nhood) + '-' + str(time_step) + \ 1052 | '-' + dataset + '/' + split + \ 1053 | '_' + str(pre_epochs) + '_' + str(s_range) + '_' + str(pred_margin) + '_' + FLAGS.reverse + view_dir + change + '.ckpt' 1054 | print(checkpt_file) 1055 | # if FLAGS.pre_train == '1': 1056 | # saver.save(sess, checkpt_file) 1057 | 1058 | # lr = 0.0005 1059 | if dataset == 'CASIA_B': 1060 | lr = 0.0005 1061 | else: 1062 | lr = 0.0025 1063 | loaded_graph = tf.Graph() 1064 | # init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) 1065 | with tf.Session(graph=loaded_graph, config=config) as sess: 1066 | loader = tf.train.import_meta_graph(checkpt_file + '.meta') 1067 | # loader.restore(sess, checkpt_file) 1068 | J_in = loaded_graph.get_tensor_by_name("Input/Placeholder_1:0") 1069 | P_in = loaded_graph.get_tensor_by_name("Input/Placeholder_2:0") 1070 | B_in = loaded_graph.get_tensor_by_name("Input/Placeholder_3:0") 1071 | I_in = loaded_graph.get_tensor_by_name("Input/Placeholder_4:0") 1072 | J_bias_in = loaded_graph.get_tensor_by_name("Input/Placeholder_5:0") 1073 | P_bias_in = loaded_graph.get_tensor_by_name("Input/Placeholder_6:0") 1074 | B_bias_in = loaded_graph.get_tensor_by_name("Input/Placeholder_7:0") 1075 | I_bias_in = loaded_graph.get_tensor_by_name("Input/Placeholder_8:0") 1076 | lbl_in = loaded_graph.get_tensor_by_name("Input/Placeholder:0") 1077 | is_train = loaded_graph.get_tensor_by_name("Input/Placeholder_11:0") 1078 | attn_drop = loaded_graph.get_tensor_by_name("Input/Placeholder_9:0") 1079 | ffd_drop = loaded_graph.get_tensor_by_name("Input/Placeholder_10:0") 1080 | en_to_pred = loaded_graph.get_tensor_by_name("Recognition/Recognition/StopGradient:0") 1081 | 1082 | # print(I_en_outs_test, J_en_outs_test, P_en_outs_test, B_en_outs_test) 1083 | # exit() 1084 | with tf.name_scope("Recognition"), tf.variable_scope("Recognition", reuse=tf.AUTO_REUSE): 1085 | if frozen == '1': 1086 | # en_to_loss = tf.stop_gradient(en_to_loss) 1087 | en_to_pred = tf.stop_gradient(en_to_pred) 1088 | if no_interp: 1089 | W_1 = tf.Variable(tf.random_normal([LSTM_embed * 3, LSTM_embed * 3])) 1090 | b_1 = tf.Variable(tf.zeros(shape=[LSTM_embed * 3, ])) 1091 | elif not only_ske_embed: 1092 | # original 1093 | W_1 = tf.Variable(tf.random_normal([LSTM_embed * 4, LSTM_embed * 4])) 1094 | b_1 = tf.Variable(tf.zeros(shape=[LSTM_embed * 4, ])) 1095 | else: 1096 | W_1 = tf.Variable(tf.random_normal([LSTM_embed, LSTM_embed])) 1097 | b_1 = tf.Variable(tf.zeros(shape=[LSTM_embed, ])) 1098 | 1099 | if no_interp: 1100 | W_2 = tf.Variable(tf.random_normal([LSTM_embed * 3, nb_classes])) 1101 | b_2 = tf.Variable(tf.zeros(shape=[nb_classes, ])) 1102 | elif not only_ske_embed: 1103 | W_2 = tf.Variable(tf.random_normal([LSTM_embed * 4, nb_classes])) 1104 | b_2 = tf.Variable(tf.zeros(shape=[nb_classes, ])) 1105 | else: 1106 | W_2 = tf.Variable(tf.random_normal([LSTM_embed, nb_classes])) 1107 | b_2 = tf.Variable(tf.zeros(shape=[nb_classes, ])) 1108 | 1109 | logits = tf.matmul(tf.nn.relu(tf.matmul(en_to_pred, W_1) + b_1), W_2) + b_2 1110 | logits_pred = tf.matmul(tf.nn.relu(tf.matmul(en_to_pred, W_1) + b_1), W_2) + b_2 1111 | 1112 | log_resh = tf.reshape(logits, [-1, nb_classes]) 1113 | lab_resh = tf.reshape(lbl_in, [-1, nb_classes]) 1114 | 1115 | if not last_pre: 1116 | aver_pred = logits[:batch_size] 1117 | aver_final_pred = logits_pred[:batch_size] 1118 | 1119 | # Multi-Scale 1120 | for i in range(1, levels + 1): 1121 | aver_pred += logits[batch_size * i:batch_size * (i + 1)] 1122 | aver_final_pred += logits_pred[batch_size * i:batch_size * (i + 1)] 1123 | else: 1124 | aver_pred = logits[-batch_size:] 1125 | aver_final_pred = logits_pred[-batch_size:] 1126 | 1127 | correct_pred = tf.equal(tf.argmax(aver_pred, -1), tf.argmax(lab_resh, -1)) 1128 | accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) 1129 | 1130 | correct_final_pred = tf.equal(tf.argmax(aver_final_pred, -1), tf.argmax(lab_resh, -1)) 1131 | accuracy_final = tf.reduce_mean(tf.cast(correct_final_pred, tf.float32)) 1132 | 1133 | loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=aver_pred, labels=lab_resh)) 1134 | # print(aver_pred, accuracy_final, loss) 1135 | # exit() 1136 | opt = tf.train.AdamOptimizer(learning_rate=lr) 1137 | train_op = opt.minimize(loss) 1138 | init_op = tf.global_variables_initializer() 1139 | sess.run(init_op) 1140 | loader.restore(sess, checkpt_file) 1141 | saver = tf.train.Saver() 1142 | vlss_mn = np.inf 1143 | vacc_mx = 0.0 1144 | vnAUC_mx = 0.0 1145 | curr_step = 0 1146 | X_train = X_train_J 1147 | X_test = X_test_J 1148 | train_loss_avg = 0 1149 | train_acc_avg = 0 1150 | val_loss_avg = 0 1151 | val_acc_avg = 0 1152 | for epoch in range(nb_epochs): 1153 | tr_step = 0 1154 | tr_size = X_train.shape[0] 1155 | train_acc_avg_final = 0 1156 | logits_all = [] 1157 | labels_all = [] 1158 | while tr_step * batch_size < tr_size: 1159 | if (tr_step + 1) * batch_size > tr_size: 1160 | break 1161 | X_input_J = X_train_J[tr_step * batch_size:(tr_step + 1) * batch_size] 1162 | X_input_J = X_input_J.reshape([-1, nb_nodes, 3]) 1163 | X_input_P = X_train_P[tr_step * batch_size:(tr_step + 1) * batch_size] 1164 | X_input_P = X_input_P.reshape([-1, 10, 3]) 1165 | X_input_B = X_train_B[tr_step * batch_size:(tr_step + 1) * batch_size] 1166 | X_input_B = X_input_B.reshape([-1, 5, 3]) 1167 | X_input_I = X_train_I[tr_step * batch_size:(tr_step + 1) * batch_size] 1168 | X_input_I = X_input_I.reshape([-1, I_nodes, 3]) 1169 | _, loss_value_tr, acc_tr, acc_tr_final, logits, labels = sess.run([train_op, loss, accuracy, accuracy_final, aver_final_pred, lab_resh], 1170 | feed_dict={ 1171 | J_in: X_input_J, 1172 | P_in: X_input_P, 1173 | B_in: X_input_B, 1174 | I_in: X_input_I, 1175 | J_bias_in: biases_J, 1176 | P_bias_in: biases_P, 1177 | B_bias_in: biases_B, 1178 | I_bias_in: biases_I, 1179 | lbl_in: y_train[tr_step * batch_size:(tr_step + 1) * batch_size], 1180 | is_train: True, 1181 | attn_drop: 0.0, ffd_drop: 0.0}) 1182 | logits_all.extend(logits.tolist()) 1183 | labels_all.extend(labels.tolist()) 1184 | train_loss_avg += loss_value_tr 1185 | train_acc_avg += acc_tr 1186 | train_acc_avg_final += acc_tr_final 1187 | tr_step += 1 1188 | train_nAUC = process.cal_nAUC(scores=np.array(logits_all), labels=np.array(labels_all)) 1189 | 1190 | vl_step = 0 1191 | vl_size = X_test.shape[0] 1192 | val_acc_avg_final = 0 1193 | logits_all = [] 1194 | labels_all = [] 1195 | loaded_graph = tf.get_default_graph() 1196 | 1197 | while vl_step * batch_size < vl_size: 1198 | if (vl_step + 1) * batch_size > vl_size: 1199 | break 1200 | X_input_J = X_test_J[vl_step * batch_size:(vl_step + 1) * batch_size] 1201 | X_input_J = X_input_J.reshape([-1, nb_nodes, 3]) 1202 | X_input_P = X_test_P[vl_step * batch_size:(vl_step + 1) * batch_size] 1203 | X_input_P = X_input_P.reshape([-1, 10, 3]) 1204 | X_input_B = X_test_B[vl_step * batch_size:(vl_step + 1) * batch_size] 1205 | X_input_B = X_input_B.reshape([-1, 5, 3]) 1206 | X_input_I = X_test_I[vl_step * batch_size:(vl_step + 1) * batch_size] 1207 | X_input_I = X_input_I.reshape([-1, I_nodes, 3]) 1208 | loss_value_vl, acc_vl, acc_vl_final, logits, labels = sess.run([loss, accuracy, accuracy_final, aver_final_pred, lab_resh], 1209 | feed_dict={ 1210 | J_in: X_input_J, 1211 | P_in: X_input_P, 1212 | B_in: X_input_B, 1213 | I_in: X_input_I, 1214 | J_bias_in: biases_J, 1215 | P_bias_in: biases_P, 1216 | B_bias_in: biases_B, 1217 | I_bias_in: biases_I, 1218 | lbl_in: y_test[vl_step * batch_size:(vl_step + 1) * batch_size], 1219 | # msk_in: val_mask[vl_step * batch_size:(vl_step + 1) * batch_size], 1220 | is_train: False, 1221 | attn_drop: 0.0, ffd_drop: 0.0}) 1222 | logits_all.extend(logits.tolist()) 1223 | labels_all.extend(labels.tolist()) 1224 | val_loss_avg += loss_value_vl 1225 | val_acc_avg += acc_vl 1226 | val_acc_avg_final += acc_vl_final 1227 | vl_step += 1 1228 | val_nAUC = process.cal_nAUC(scores=np.array(logits_all), labels=np.array(labels_all)) 1229 | print('Training: loss = %.5f, acc = %.5f, nAUC = %.5f | Val: loss = %.5f, acc = %.5f, nAUC = %.5f' % 1230 | (train_loss_avg / tr_step, train_acc_avg_final / tr_step, train_nAUC, 1231 | val_loss_avg / vl_step, val_acc_avg_final / vl_step, val_nAUC)) 1232 | # print(att_w) 1233 | 1234 | if val_acc_avg / vl_step >= vacc_mx or val_loss_avg / vl_step <= vlss_mn: 1235 | # if val_acc_avg / vl_step >= vacc_mx and val_loss_avg / vl_step <= vlss_mn: 1236 | vnAUC_mx = val_nAUC 1237 | if dataset == 'IAS': 1238 | if val_acc_avg / vl_step > vacc_mx and epoch >= 10: 1239 | vacc_early_model = val_acc_avg / vl_step 1240 | vlss_early_model = val_loss_avg / vl_step 1241 | checkpt_file = RN_dir + '_' + str(fusion_lambda) + '-' + pretext + '-' + str(nhood) + '-' + str( 1242 | time_step) + \ 1243 | '-' + dataset + '/' + split \ 1244 | + '_' + str(pre_epochs) + '_' + str(s_range) + '_' + str( 1245 | pred_margin) + view_dir + '_' + str(FLAGS.bi) \ 1246 | + '_' + str(round(vacc_early_model * 100, 1)) + '_' + str( 1247 | round(vnAUC_mx * 100, 1)) + change + '.ckpt' 1248 | print(checkpt_file) 1249 | if save_flag == '1': 1250 | saver.save(sess, checkpt_file) 1251 | else: 1252 | if val_acc_avg / vl_step > vacc_mx: 1253 | vacc_early_model = val_acc_avg / vl_step 1254 | vlss_early_model = val_loss_avg / vl_step 1255 | # checkpt_file = RN_dir + '_' + str(fusion_lambda) + '-' + pretext + '-' + str(nhood) + '-' + str(time_step) + \ 1256 | # '-' + dataset + '/' + split \ 1257 | # + '_' + str(pre_epochs) + '_' + str(s_range) + '_' + str(pred_margin) + view_dir + '_' + str(FLAGS.bi) \ 1258 | # + '_' + str(round(vacc_early_model*100, 1)) + '_' + str(round(vnAUC_mx*100, 1)) + change + '.ckpt' 1259 | checkpt_file = RN_dir + '_' + str(fusion_lambda) + '-' + pretext + '-' + str(nhood) + '-' + str(time_step) + \ 1260 | '-' + dataset + '/' + split \ 1261 | + '_' + str(pre_epochs) + '_' + str(s_range) + '_' + str(pred_margin) + view_dir + '_' + str(FLAGS.bi) \ 1262 | + '_best' + change + '.ckpt' 1263 | print(checkpt_file) 1264 | if save_flag == '1': 1265 | saver.save(sess, checkpt_file) 1266 | vacc_mx = np.max((val_acc_avg / vl_step, vacc_mx)) 1267 | vlss_mn = np.min((val_loss_avg / vl_step, vlss_mn)) 1268 | curr_step = 0 1269 | else: 1270 | curr_step += 1 1271 | if curr_step == patience: 1272 | print('Early stop! Min loss: ', vlss_mn, ', Max accuracy: ', vacc_mx, ', nAUC: ', vnAUC_mx) 1273 | print('Early stop model validation loss: ', vlss_early_model, ', accuracy: ', vacc_early_model, ', nAUC: ', val_nAUC) 1274 | break 1275 | train_loss_avg = 0 1276 | train_acc_avg = 0 1277 | val_loss_avg = 0 1278 | val_acc_avg = 0 1279 | saver.restore(sess, checkpt_file) 1280 | sess.close() 1281 | 1282 | print('Dataset: ' + dataset) 1283 | 1284 | print('----- Opt. hyperparams -----') 1285 | print('pre_train_epochs: ' + str(pre_epochs)) 1286 | print('nhood: ' + str(nhood)) 1287 | print('skeleton_nodes: ' + str(nb_nodes)) 1288 | print('seqence_length: ' + str(time_step)) 1289 | print('pretext: ' + str(pretext)) 1290 | print('fusion_lambda: ' + str(fusion_lambda)) 1291 | print('batch_size: ' + str(batch_size)) 1292 | print('lr: ' + str(lr)) 1293 | print('view: ' + FLAGS.view) 1294 | print('P: ' + FLAGS.P) 1295 | print('fusion_lambda: ' + FLAGS.fusion_lambda) 1296 | print('loss_type: ' + FLAGS.loss) 1297 | print('patience: ' + FLAGS.patience) 1298 | print('save_flag: ' + FLAGS.save_flag) 1299 | 1300 | print('----- Archi. hyperparams -----') 1301 | print('structural relation matrix number: ' + str(Ps[0])) 1302 | print('LSTM_embed_num: ' + str(LSTM_embed)) 1303 | print('LSTM_layer_num: ' + str(num_layers)) 1304 | 1305 | --------------------------------------------------------------------------------