├── README.md ├── SyncNetModel.py ├── Theano_Version ├── README ├── cnn_classifier.py ├── cnn_layers.py ├── main_autism.py ├── optimizers.py └── utils.py ├── make_synthetic_data.m ├── plot_d_b.m ├── syncnet.py └── utils.py /README.md: -------------------------------------------------------------------------------- 1 | # Targeting EEG/LFP Synchrony with Neural Nets 2 | This is the code for 'Targeting EEG/LFP Synchrony with Neural Nets' (SyncNet). Input data format is Number x channel x time. The sample code gives the result of the synthetic dataset. 3 | 4 | For a quick start, run make_synthetic_data.m to get toy.mat as toy EEG data. Then run SyncNet.py to test the SyncNet performance on the toy EEG data. 5 | 6 | 7 | ```bash 8 | @inproceedings{li2017targeting, 9 | title={Targeting EEG/LFP Synchrony with Neural Nets}, 10 | author={Li, Yitong and Murias, Michael and Major, Samantha and Dawson, Geraldin and Dzirasa, Kafui and Carin, Lawrence and Carlson, David E.}, 11 | booktitle={Advances in Neural Information Processing Systems}, 12 | pages={4621--4631}, 13 | year={2017} 14 | } 15 | ``` 16 | -------------------------------------------------------------------------------- /SyncNetModel.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import utils 3 | import numpy as np 4 | 5 | class SyncNetModel(object): 6 | def __init__(self, options): 7 | self.l = tf.placeholder(tf.float32, []) 8 | self.lr = tf.placeholder(tf.float32, []) 9 | self.sample_type = tf.float32 10 | self.num_labels = options['num_labels'] 11 | self.sample_shape = options['sample_shape'] 12 | self.batch_size = options['batch_size'] 13 | self.dropout_rate = options['dropout_rate'] 14 | self.X = tf.placeholder(tf.as_dtype(self.sample_type), [None] + list(self.sample_shape), name="input_X") 15 | self.y = tf.placeholder(tf.float32, [None, self.num_labels], name="input_labels") 16 | self.train = tf.placeholder(tf.bool, [], name = 'train') 17 | self._build_model(options) 18 | self._setup_train_ops() 19 | 20 | 21 | def SyncNetFilters(self, options): 22 | b=tf.Variable(tf.random_uniform([1,1,options['C'],options['K']], minval=-0.05, maxval=0.05, dtype=tf.float32),name='b') 23 | omega=tf.Variable(tf.random_uniform ([1,1,1,options['K']], minval = 0., maxval = 1.),name='omega') 24 | zero_pad = tf.zeros( (1, 1, 1, options['K']), dtype = tf.float32, name ='zero_pad') 25 | phi_ini=tf.Variable(tf.random_normal([1,1,options['C']-1, options['K']],mean=0.0, stddev=0.05, dtype=tf.float32), name='phi') 26 | phi = tf.concat([zero_pad, phi_ini], axis = 2) 27 | beta=tf.Variable(tf.random_uniform([1,1,1,options['K']], minval = 0., maxval = 0.05), dtype = tf.float32,name='beta') 28 | #t=np.reshape(np.linspace(-options['Nt']/2.,options['Nt']/2.,options['Nt']),[1,options['Nt'],1,1]) 29 | t=np.reshape(range(-options['Nt']/2,options['Nt']/2),[1,options['Nt'],1,1]) 30 | tc=tf.constant(np.single(t),name='t') 31 | W_osc=tf.multiply(b,tf.cos(tc*omega+phi)) 32 | W_decay=tf.exp(-tf.pow(tc,2)*beta) 33 | W=tf.multiply(W_osc,W_decay) 34 | self.beta_op = tf.assign(beta, tf.clip_by_value(beta, 0, np.infty)) 35 | return W 36 | 37 | def feature_extractor(self, X, options): 38 | self.dropout_x = utils.channel_dropout(X, self.dropout_rate) 39 | X = tf.expand_dims(self.dropout_x, axis = 1, name = 'reshaped_input') 40 | with tf.variable_scope('syncnet_conv',reuse = True): 41 | W = self.SyncNetFilters(options) 42 | bias = tf.Variable(tf.constant(0.0, dtype = tf.float32, shape = [options['K']]), name = 'bias') 43 | h_conv1 = tf.nn.relu(tf.nn.conv2d(X, W, strides=[1, 1, 1, 1], padding='SAME') + bias) 44 | h_pool1 = tf.nn.max_pool(h_conv1, ksize=[1, 1, options['pool_size'], 1], strides=[1, 1, options['pool_size'], 1], padding='SAME') 45 | self.h_pool1 = h_pool1 46 | features = tf.reshape(h_pool1, [-1, options['cl_Wy']]) 47 | return features 48 | 49 | def label_predictor(self, features): 50 | with tf.variable_scope('label_predictor_logits'): 51 | logits = utils.fully_connected_layer(features, self.num_labels) 52 | return logits 53 | 54 | def _build_model(self, options): 55 | self.features = self.feature_extractor(self.X, options) 56 | logits = self.label_predictor(self.features) 57 | self.y_pred = tf.nn.softmax(logits) 58 | self.y_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits, labels = self.y)) 59 | self.y_acc = utils.predictor_accuracy(self.y_pred,self.y) 60 | 61 | 62 | def _setup_train_ops(self): 63 | self.train_ops = tf.train.AdamOptimizer(self.lr).minimize(self.y_loss) 64 | 65 | -------------------------------------------------------------------------------- /Theano_Version/README: -------------------------------------------------------------------------------- 1 | Thie is an older version using theano. The code is not tested. 2 | -------------------------------------------------------------------------------- /Theano_Version/cnn_classifier.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | import theano 4 | import theano.tensor as tensor 5 | 6 | from collections import OrderedDict 7 | 8 | from utils import uniform_weight, dropout_channel, zero_bias, numpy_floatX 9 | from theano.tensor.nnet import conv 10 | from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams 11 | from cnn_layers import conv_pool, conv_pool2, deconv_depool, conv_pool_ae, ReLU 12 | from utils import _p 13 | 14 | # Set the random number generators' seeds for consistency 15 | SEED = 3435 16 | np.random.seed(SEED) 17 | rng = np.random.RandomState(SEED) 18 | 19 | """ init. parameters. """ 20 | 21 | def init_params(options,prefix): 22 | temp = prefix 23 | params = OrderedDict() 24 | selected = np.random.permutation(options['C0']) 25 | params['r'] = options['Lx'][selected[0:options['C']],:] 26 | params['gp_beta'] = np.float32(1.) 27 | params['gp_alpha'] = np.float32(2.) 28 | if prefix == 'e' or temp == 'all': 29 | prefix = 'e' 30 | params[_p(prefix,'b')] = uniform_weight(options[_p(prefix,'K')],options['C']) 31 | params[_p(prefix,'phi')] = uniform_weight(options[_p(prefix,'K')],options['C'] - 1) 32 | params[_p(prefix,'beta')] = uniform_weight(options[_p(prefix,'K')],1,0,0.05) 33 | params[_p(prefix,'omega')] = uniform_weight(options[_p(prefix,'K')],1, 0,1.) 34 | params[_p(prefix,'bias')] = np.zeros((options[_p(prefix,'K')]),dtype = theano.config.floatX) 35 | 36 | if prefix == 'cl' or temp == 'all': 37 | prefix = 'cl' 38 | params[_p(prefix,'Wy')] = uniform_weight(options[_p(prefix,'Wy')],options[_p(prefix,'ny')]) 39 | params[_p(prefix,'by')] = np.zeros((options[_p(prefix,'ny')]),dtype = theano.config.floatX) 40 | return params 41 | 42 | def init_tparams(params): 43 | tparams = OrderedDict() 44 | for kk, pp in params.iteritems(): 45 | tparams[kk] = theano.shared(params[kk], name=kk) 46 | #tparams[kk].tag.test_value = params[kk] 47 | return tparams 48 | 49 | def _slice(_x,n,dim): 50 | if _x.ndim == 3: 51 | return _x[:,:,n * dim:(n+1)*dim] 52 | return _x[:, n * dim:(n+1)*dim] 53 | 54 | def Gaussian_Process(tparams,options): 55 | #_rzz = tparams['r'].repeat(options['C'],axis = 0).reshape((options['C'],options['C'],3)) 56 | #_distancezz = (_rzz - _rzz.dimshuffle(1,0,2)).norm(1, axis = 2) 57 | #_Kzz = tparams['gp_beta'] * tensor.exp( - tparams['gp_alpha'] * _distancezz) 58 | _rxx = np.repeat(options['Lx'],options['C0'], axis = 0).reshape((options['C0'],options['C0'],options['P'])) 59 | _distancexx = np.linalg.norm(_rxx - _rxx.transpose(1,0,2), 1, axis = 2) 60 | _rzx = tparams['r'].repeat(options['C0'],axis = 0 ).reshape((options['C'],options['C0'],options['P'])) 61 | _Kxx = tparams['gp_beta'] * tensor.exp( - tparams['gp_alpha'] * _distancexx) 62 | _x_rep = np.repeat(options['Lx'], options['C'], axis = 0).reshape((options['C0'], options['C'] , options['P'])) 63 | _distancezx = (_rzx - _x_rep.transpose((1,0,2))).norm(1,axis = 2) 64 | _Kzx = tparams['gp_beta'] * tensor.exp( - tparams['gp_alpha'] * _distancezx) 65 | _eta = tensor.nlinalg.matrix_inverse(_Kxx + options['sigma'] * np.eye(options['C0'],dtype = 'float32')) 66 | _W = _Kzx.dot(_eta) 67 | return _W 68 | 69 | def inv_Gaussian_Process(tparams,options): 70 | _rzz = tparams['r'].repeat(options['C'],axis = 0).reshape((options['C'],options['C'],options['P'])) 71 | _distancezz = (_rzz - _rzz.dimshuffle(1,0,2)).norm(1, axis = 2) 72 | _Kzz = tparams['gp_beta'] * tensor.exp( - tparams['gp_alpha'] * _distancezz) 73 | # _rxx = np.repeat(options['Lx'],options['C0'], axis = 0).reshape((options['C0'],options['C0'],options['P'])) 74 | # _distancexx = np.linalg.norm(_rxx - _rxx.transpose(1,0,2), 1, axis = 2) 75 | # _Kxx = tparams['gp_beta'] * tensor.exp( - tparams['gp_alpha'] * _distancexx) 76 | _rzx = tparams['r'].repeat(options['C0'],axis = 0 ).reshape((options['C'],options['C0'],options['P'])) 77 | _x_rep = np.repeat(options['Lx'], options['C'], axis = 0).reshape((options['C0'], options['C'] , options['P'])) 78 | _distancezx = (_rzx - _x_rep.transpose((1,0,2))).norm(1,axis = 2) 79 | _Kzx = tparams['gp_beta'] * tensor.exp( - tparams['gp_alpha'] * _distancezx) 80 | _Kxz = _Kzx.dimshuffle(1,0) 81 | _eta = tensor.nlinalg.matrix_inverse(_Kzz + options['sigma'] * np.eye(options['C'],dtype = 'float32')) 82 | _W = _Kxz.dot(_eta) 83 | return _W 84 | """ Building model... """ 85 | 86 | def build_model(tparams,options): 87 | 88 | # Used for dropout. 89 | #use_noise = theano.shared(numpy_floatX(0.)) 90 | # input sentence: n_samples * n_steps 91 | _x = tensor.tensor3('x', dtype=theano.config.floatX) 92 | _y = tensor.vector('y',dtype='int32') 93 | trng = RandomStreams(10) 94 | p = theano.shared(np.float32(options['dropout_rate'])) 95 | _x_dropout = dropout_channel(_x,trng,p) 96 | 97 | _Wzx = Gaussian_Process(tparams,options) 98 | _Wzx = _Wzx.dimshuffle(1,0) 99 | _input = tensor.dot(_x_dropout.dimshuffle(0,2,1),_Wzx) 100 | _input = _input.dimshuffle(0,2,'x',1) 101 | 102 | def encoder(layer0_input): 103 | 104 | """ filter_shape: (number of filters, num input feature maps, filter height, 105 | filter width) 106 | image_shape: (batch_size, num input feature maps, image height, image width) 107 | """ 108 | output = conv_pool(layer0_input, tparams, options, 'e') 109 | output = output.reshape((output.shape[0],options['cl_Wy'])) 110 | 111 | 112 | return output 113 | 114 | _output = encoder(_input) 115 | f_conv = theano.function([_x], _output) 116 | # this is the label prediction you made 117 | pred = tensor.nnet.softmax(tensor.dot(_output, tparams['cl_Wy']) + tparams['cl_by']) 118 | f_pred_prob = theano.function([_x], pred, name='f_pred_prob') 119 | f_pred = theano.function([_x], pred.argmax(axis=1), name='f_pred') 120 | 121 | index = tensor.arange(_x.shape[0]) 122 | cost = -tensor.log(pred[index, _y] + 1e-6).mean() #+ 10 * theano.tensor.abs_(tparams['b']).mean() #+10**2 * theano.tensor.abs_(tparams['phi']).mean() 123 | 124 | return _x,_y,f_conv, f_pred_prob,f_pred,cost 125 | 126 | -------------------------------------------------------------------------------- /Theano_Version/cnn_layers.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | import theano 4 | import theano.tensor as tensor 5 | import theano.tensor.shared_randomstreams 6 | from theano.tensor.nnet import conv 7 | from utils import _p 8 | 9 | rng = np.random.RandomState(3435) 10 | 11 | def ReLU(x): 12 | y = theano.tensor.maximum(0.0, x) 13 | return(y) 14 | 15 | def get_filter(tparams,options,prefix): 16 | if prefix is 'e': 17 | Channel_num = options['C'] 18 | Filter_num = options['e_K'] 19 | Filter_len = options['e_Nt'] 20 | elif prefix is 'd': 21 | Channel_num = options['e_K'] 22 | Filter_num = options['C'] 23 | Filter_len = options['e_Nt'] 24 | elif prefix is 'e2': 25 | Channel_num = options['e_K'] 26 | Filter_num = options['e2_K'] 27 | Filter_len = options['e2_Nt'] 28 | 29 | zero_pad = tensor.basic.zeros([Filter_num,1]) 30 | #_beta = (tparams['beta'] + theano.tensor.abs_(tparams['beta']))/2 31 | _beta = tensor.extra_ops.repeat(tparams[_p(prefix,'beta')],Channel_num,axis = 1).dimshuffle(0,1,'x','x') 32 | _beta = _beta.repeat(Filter_len,axis = 3) 33 | _omega = tensor.extra_ops.repeat(tparams[_p(prefix,'omega')],Channel_num,axis = 1).dimshuffle(0,1,'x','x') 34 | _omega = _omega.repeat(Filter_len,axis = 3) 35 | _b = tparams[_p(prefix,'b')].dimshuffle(0,1,'x','x') 36 | _b = _b.repeat(Filter_len,axis = 3) 37 | _phi = tensor.concatenate([zero_pad,tparams[_p(prefix,'phi')]],axis = 1).dimshuffle(0,1,'x','x') 38 | _phi = _phi.repeat(Filter_len,axis = 3) 39 | t = np.array(range(-Filter_len/2,Filter_len/2)).reshape(1,1,1,-1) 40 | t = t.repeat(Filter_num,axis = 0) 41 | t = t.repeat(Channel_num,axis = 1) 42 | _W = _b * tensor.cos(_omega * t + _phi) * tensor.exp(-_beta * t * t) 43 | return _W 44 | 45 | 46 | def depool_repeat(_X,rate): 47 | _X_repeat = theano.tensor.extra_ops.repeat(_X.dimshuffle(0,1,'x',2),rate,axis = 3) 48 | _output = theano.tensor.extra_ops.squeeze(_X_repeat) 49 | return _output 50 | """ Encoder using Convolutional Neural Network. """ 51 | 52 | 53 | 54 | 55 | def conv_pool_ae(layer0_input, tparams, options, prefix): 56 | """ filter_shape: (number of filters, num input feature maps, filter height, 57 | filter width) 58 | image_shape: (batch_size, num input feature maps, image height, image width) 59 | """ 60 | _W = get_filter(tparams,options,prefix).astype(theano.config.floatX) 61 | s = int(np.floor(options[_p(prefix,'Nt')]/2.)) 62 | conv_out = conv.conv2d(input=layer0_input, filters=_W, 63 | filter_shape= options[_p(prefix,'filter_shape')], border_mode = 'full')[:,:,:,s-1:-s] 64 | conv_out_relu = ReLU(conv_out + tparams[_p(prefix,'bias')].dimshuffle('x', 0, 'x', 'x')) 65 | output_re = theano.tensor.signal.pool.pool_2d(input=conv_out_relu.dimshuffle(0,2,1,3), 66 | ds=(1,options[_p(prefix,'pool_size')]), ignore_border=True, mode = 'max') 67 | output_re =output_re.reshape((output_re.shape[0],options[_p(prefix,'K')],output_re.shape[3])) 68 | output_cl = theano.tensor.signal.pool.pool_2d(input=conv_out_relu.dimshuffle(0,2,1,3), 69 | ds=(1,options['cl_pool_size']), ignore_border=True, mode = 'max') 70 | output_cl = output_cl.reshape((output_cl.shape[0], options[_p(prefix,'K')],output_cl.shape[3])) 71 | 72 | return output_cl, output_re 73 | 74 | 75 | def conv_pool(layer0_input, tparams, options, prefix): 76 | """ filter_shape: (number of filters, num input feature maps, filter height, 77 | filter width) 78 | image_shape: (batch_size, num input feature maps, image height, image width) 79 | """ 80 | _W = get_filter(tparams,options,prefix).astype(theano.config.floatX) 81 | s = int(np.floor(options[_p(prefix,'Nt')]/2.)) 82 | conv_out = conv.conv2d(input=layer0_input, filters=_W, 83 | filter_shape= options[_p(prefix,'filter_shape')], border_mode = 'full')[:,:,:,s-1:-s] 84 | conv_out_relu = ReLU(conv_out + tparams[_p(prefix,'bias')].dimshuffle('x', 0, 'x', 'x')) 85 | output = theano.tensor.signal.pool.pool_2d(input=conv_out_relu.dimshuffle(0,2,1,3), 86 | ds=(1,options['cl_pool_size']), ignore_border=True, mode = 'max') 87 | output = output.reshape((output.shape[0],options[_p(prefix,'K')],output.shape[3])) 88 | return output 89 | 90 | def conv_pool2(layer0_input, tparams, options,prefix): 91 | 92 | s = max( int(np.floor(options[_p(prefix,'Nt')]/2.)),1) 93 | h = options[_p(prefix,'filter_shape')][2] - 1 94 | 95 | conv_out = theano.tensor.nnet.conv.conv2d(input=layer0_input, filters=tparams[_p(prefix,'W')], 96 | filter_shape= options[_p(prefix,'filter_shape')], border_mode = 'full')[:,:,h,s-1:-s] 97 | conv_out_relu = ReLU(conv_out + tparams[_p(prefix,'bias')].dimshuffle('x', 0, 'x')) 98 | output = theano.tensor.signal.pool.pool_2d(input=conv_out_relu.dimshuffle(0,1,'x',2), 99 | ds=(1,options[_p(prefix,'pool_size')]), ignore_border=True, mode = 'max') 100 | output =output.reshape((output.shape[0],options[_p(prefix,'K')],output.shape[3])) 101 | 102 | return output 103 | 104 | def deconv_depool(layer0_input,tparams, options,prefix): 105 | if prefix == 'd': 106 | depool_out = depool_repeat(layer0_input,options['e_pool_size']) 107 | elif prefix == 'd2': 108 | depool_out = depool_repeat(layer0_input,options['e2_pool_size']) 109 | s = int(np.floor(options[_p(prefix,'Nt')]/2.)) 110 | _W = get_filter(tparams,options,prefix).astype(theano.config.floatX) 111 | deconv_out = conv.conv2d(input=depool_out.dimshuffle(0,1,'x',2), filters=_W, 112 | filter_shape= options[_p(prefix,'filter_shape')], border_mode = 'full')[:,:,:,s-1:-s] 113 | doutput = (deconv_out + tparams[_p(prefix,'bias')].dimshuffle('x',0,'x','x')).reshape((deconv_out.shape[0],options['C'],options['T'])) 114 | return doutput 115 | 116 | def deconv_depool2(layer0_input,tparams, options,prefix): 117 | if prefix == 'd': 118 | depool_out = depool_repeat(layer0_input,options['e_pool_size']) 119 | elif prefix == 'd2': 120 | depool_out = depool_repeat(layer0_input,options['e2_pool_size']) 121 | s = int(np.floor(options[_p(prefix,'Nt')]/2.)) 122 | h = int((2 * options[_p(prefix,'K')] - 2)/2.) 123 | deconv_out = conv.conv2d(input=depool_out.dimshuffle(0,'x',1,2), filters=tparams[_p(prefix,'W')], 124 | filter_shape= options[_p(prefix,'filter_shape')], border_mode = 'full')[:,:,h,s-1:-s] 125 | doutput = (deconv_out + tparams[_p(prefix,'bias')].dimshuffle('x',0,'x'))#.reshape((deconv_out.shape[0],options['C'],options['T'])) 126 | return doutput -------------------------------------------------------------------------------- /Theano_Version/main_autism.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import time 4 | import scipy.io 5 | import os 6 | import numpy as np 7 | import theano 8 | import theano.tensor as tensor 9 | import theano.tensor.signal 10 | import theano.tensor.signal.pool 11 | 12 | from cnn_classifier import init_params, init_tparams 13 | from cnn_classifier import build_model 14 | import sklearn 15 | from sklearn import metrics 16 | import optimizers 17 | 18 | from utils import get_minibatches_idx, Read_Autism_cross 19 | from utils import unzip, zipp 20 | 21 | SEED = 1101 22 | def pred_error(f_pred, data, label): 23 | preds = f_pred(data) 24 | errs = (preds == label).sum().astype(theano.config.floatX) 25 | errs = 1. - errs/data.shape[0] 26 | errs = errs.astype(theano.config.floatX) 27 | return errs 28 | 29 | def pred_error_all(f_pred, f_pred_prob, data, label): 30 | step = 2000. 31 | N,C,T = data.shape 32 | temp = f_pred_prob(data[0:1,:,:]) 33 | pred = np.zeros((data.shape[0],)) 34 | prob = np.zeros((data.shape[0],temp.shape[1])) 35 | for i in range(np.int32(np.ceil(data.shape[0]/step))): 36 | srt = np.int32(i * step) 37 | edn = np.int32(min((i+1) * step, data.shape[0])) 38 | pred[srt:edn] = f_pred(data[srt:edn,:,:]) 39 | prob[srt:edn,:] = f_pred_prob(data[srt:edn,:,:]) 40 | errs = (pred == label).sum().astype(theano.config.floatX) 41 | errs = 1. - errs/data.shape[0] 42 | errs = errs.astype(theano.config.floatX) 43 | return prob, pred, errs 44 | 45 | """ Training the model. """ 46 | 47 | 48 | """ used to calculate the prediction error. """ 49 | 50 | if __name__ == '__main__': 51 | # data is of size N x C x T 52 | # https://docs.python.org/2/howto/logging-cookbook.html 53 | Nt = 40 54 | K = 10 55 | batch_size = 200 56 | max_epochs = 40 57 | patience = 10 58 | lrate = 0.002 59 | valid_batch = 100 60 | dispFreq = 2 61 | validFreq = 10 62 | pool_size_cl = 40 63 | C = 60 64 | drop_rate = 0.2 65 | saveFreq = 100 66 | result_path = './Result_gpcnn_p' + str(drop_rate) + '_K_' + str(K) + '_Nt' + str(Nt) + '_C' + str(C) 67 | file_path = '/media/lyt/SSD/Autism/All_zscores_200/' 68 | if not os.path.exists(result_path): 69 | os.mkdir(result_path) 70 | for test_id in np.array([21]): 71 | if test_id == 4 or test_id == 7 or test_id == 22: 72 | continue 73 | train,val,test,labtrain,_,labval,labtest = Read_Autism_cross(file_path,[test_id + 1]) 74 | _,C0,T = train.shape 75 | options = {} 76 | options['uidx'] = 0 77 | options['P'] = 2 78 | options['sigma'] = np.float32(0.001) 79 | options['C0'] = C0 80 | mat = scipy.io.loadmat('./Autism_position.mat') 81 | options['Lx'] = mat['chanlocs'].astype('float32') 82 | options['T'] = T 83 | options['C'] = C 84 | options['e_K'] = K 85 | options['e_Nt'] = Nt 86 | options['e_filter_shape'] = (K,C,1,Nt) 87 | options['cl_pool_size'] = pool_size_cl 88 | options['Wy'] =int(np.floor(float(T)/float(options['cl_pool_size']))) 89 | options['dropout_rate'] = drop_rate 90 | options['cl_Wy'] = options['Wy'] * K 91 | options['cl_ny'] = np.max(labtrain) + 1 92 | options['pre_Wy'] = options['Wy'] * K 93 | options['pre_ny'] = options['cl_ny'] 94 | options['n_y'] = np.max(labtrain) + 1 95 | estop = False 96 | history_errs = [] 97 | history_aucs = [] 98 | best_p = None 99 | bad_counter = 0 100 | uidx = 0 # number of update done 101 | inits = init_params(options,'all') 102 | params = inits 103 | tparams = init_tparams(params) 104 | before = np.zeros((2,)) 105 | _x,_y,f_conv,f_pred_prob, f_pred, _cost= build_model(tparams,options) 106 | _lr = tensor.scalar(name = 'lr') 107 | f_cost = theano.function([_x,_y],_cost) 108 | f_grad_shared, f_update = optimizers.Adam(tparams,_cost,[_x,_y],_lr) 109 | 110 | print('Start Pre-Training...') 111 | start_time = time.time() 112 | try: 113 | for eidx in xrange(max_epochs): 114 | batch_index = get_minibatches_idx(train.shape[0], batch_size,shuffle = True) 115 | for _, train_index in batch_index: 116 | uidx = uidx + 1 117 | options['uidx'] = options['uidx'] + 1 118 | x = train[train_index,:,:] 119 | y = labtrain[train_index] 120 | cost = f_grad_shared(x,y) 121 | f_update(lrate,0.) 122 | if np.mod(eidx + 1,dispFreq) == 0: 123 | print('Epoch ' + str(eidx) + ' Update ' + str(uidx) + ' Cost ' + str(cost)) 124 | if np.mod(eidx + 1, saveFreq) == 0: 125 | if best_p is not None: 126 | params = best_p 127 | else: 128 | params = unzip(tparams) 129 | np.savez('./model.npz',history_errs = history_errs,**params) 130 | if np.mod(eidx + 1,validFreq) == 0: 131 | train_prod,train_pred, train_err = pred_error_all(f_pred,f_pred_prob, train, labtrain) 132 | #train_auc = metrics.roc_auc_score(labtrain,train_pred[:,1]) 133 | val_prod, val_pred, val_err = pred_error_all(f_pred,f_pred_prob, val, labval) 134 | #val_auc = metrics.roc_auc_score(labval,val_pred[:,1]) 135 | test_prod, test_pred, test_err = pred_error_all(f_pred,f_pred_prob, test, labtest) 136 | #test_auc = metrics.roc_auc_score(labtest,test_pred[:,1]) 137 | history_errs.append([train_err,val_err,test_err]) 138 | #history_aucs.append([train_auc,val_auc,test_auc]) 139 | print('Train ' + str(train_err) + ' Val ' + str(val_err) + ' Test ' + str(test_err)) 140 | #print('Train ' + str(train_auc) + ' Val ' + str(val_auc) + ' Test ' + str(test_auc)) 141 | if uidx == 0 or val_err<=np.array(history_errs)[:,1].min(): 142 | best_p = unzip(tparams) 143 | bad_counter = 0 144 | before[0] = test_err 145 | #before[1] = test_auc 146 | confusion = sklearn.metrics.confusion_matrix(labtest,test_pred) 147 | if len(history_errs) > patience and val_err >= np.array(history_errs)[:-patience,0].min(): 148 | bad_count = bad_counter + 1 149 | if bad_counter > patience: 150 | estop = True 151 | break 152 | if estop: 153 | break 154 | 155 | 156 | 157 | except KeyboardInterrupt: 158 | print('Training interrupted') 159 | end_time = time.time() 160 | if best_p is not None: 161 | zipp(best_p,tparams) 162 | else: 163 | best_p = unzip(tparams) 164 | 165 | 166 | 167 | max_epochs = 100 168 | patience = 10 169 | lrate = 0.002 170 | valid_batch = 100 171 | print('Start Training...') 172 | start_time = time.time() 173 | try: 174 | for eidx in xrange(max_epochs): 175 | batch_index = get_minibatches_idx(train.shape[0], batch_size,shuffle = True) 176 | for _, train_index in batch_index: 177 | uidx = uidx + 1 178 | options['uidx'] = options['uidx'] + 1 179 | x = train[train_index,:,:] 180 | y = labtrain[train_index] 181 | cost = f_grad_shared(x,y) 182 | f_update(lrate,1.) 183 | if np.mod(eidx + 1,dispFreq) == 0: 184 | print('Epoch ' + str(eidx) + ' Update ' + str(uidx) + ' Cost ' + str(cost)) 185 | if np.mod(eidx + 1, saveFreq) == 0: 186 | if best_p is not None: 187 | params = best_p 188 | else: 189 | params = unzip(tparams) 190 | np.savez('./model.npz',history_errs = history_errs,**params) 191 | if np.mod(eidx + 1,validFreq) == 0: 192 | train_prod, train_pred, train_err = pred_error_all(f_pred,f_pred_prob, train, labtrain) 193 | #train_auc = metrics.roc_auc_score(labtrain,train_pred[:,1]) 194 | val_prod, val_pred, val_err = pred_error_all(f_pred,f_pred_prob, val, labval) 195 | #val_auc = metrics.roc_auc_score(labval,val_pred[:,1]) 196 | test_prod, test_pred, test_err = pred_error_all(f_pred,f_pred_prob, test, labtest) 197 | #test_auc = metrics.roc_auc_score(labtest,test_pred[:,1]) 198 | history_errs.append([train_err,val_err,test_err]) 199 | #history_aucs.append([train_auc,val_auc,test_auc]) 200 | print('Train ' + str(train_err) + ' Val ' + str(val_err) + ' Test ' + str(test_err)) 201 | #print('Train ' + str(train_auc) + ' Val ' + str(val_auc) + ' Test ' + str(test_auc)) 202 | if uidx == 0 or val_err<=np.array(history_errs)[:,1].min(): 203 | best_p = unzip(tparams) 204 | bad_counter = 0 205 | before[0] = test_err 206 | #before[1] = test_auc 207 | confusion = sklearn.metrics.confusion_matrix(labtest,test_pred) 208 | if len(history_errs) > patience and val_err >= np.array(history_errs)[:-patience,0].min(): 209 | bad_count = bad_counter + 1 210 | if bad_counter > patience: 211 | estop = True 212 | break 213 | if estop: 214 | break 215 | 216 | except KeyboardInterrupt: 217 | print('Training interrupted') 218 | end_time = time.time() 219 | if best_p is not None: 220 | zipp(best_p,tparams) 221 | else: 222 | best_p = unzip(tparams) 223 | scipy.io.savemat(result_path + '/Result_' + str(test_id + 1) + '_' + str(before[0]) + '.mat',{'inits':inits, 'best_p':best_p, 'history_errs':history_errs,'confusion':confusion}) 224 | print('Best Test Error is: ' + str(before[0]) + ', AUC is' + str(before[1])) 225 | -------------------------------------------------------------------------------- /Theano_Version/optimizers.py: -------------------------------------------------------------------------------- 1 | import theano 2 | import theano.tensor as tensor 3 | from utils import numpy_floatX 4 | import numpy 5 | 6 | def SGD(tparams, cost, inps, lr,clip_norm=5): 7 | """ default: lr=0.01 """ 8 | 9 | grads = tensor.grad(cost, tparams.values()) 10 | norm = tensor.sqrt(sum([tensor.sum(g**2) for g in grads])) 11 | if tensor.ge(norm, clip_norm): 12 | grads = [g*clip_norm/norm for g in grads] 13 | 14 | gshared = [theano.shared(p.get_value() * 0., name='%s_grad'%k) 15 | for k, p in tparams.iteritems()] 16 | gsup = [(gs, g) for gs, g in zip(gshared, grads)] 17 | f_grad_shared = theano.function(inps, cost, updates=gsup) 18 | 19 | updates = [] 20 | 21 | for p, g in zip(tparams.values(), gshared): 22 | updated_p = p - lr * g 23 | updates.append((p, updated_p)) 24 | 25 | f_update = theano.function([lr], [], updates=updates) 26 | 27 | return f_grad_shared, f_update 28 | 29 | def Momentum(tparams, cost, inps, lr, momentum=0.9,clip_norm=5): 30 | """ default: lr=0.01 """ 31 | 32 | grads = tensor.grad(cost, tparams.values()) 33 | norm = tensor.sqrt(sum([tensor.sum(g**2) for g in grads])) 34 | if tensor.ge(norm, clip_norm): 35 | grads = [g*clip_norm/norm for g in grads] 36 | 37 | gshared = [theano.shared(p.get_value() * 0., name='%s_grad'%k) 38 | for k, p in tparams.iteritems()] 39 | gsup = [(gs, g) for gs, g in zip(gshared, grads)] 40 | f_grad_shared = theano.function(inps, cost, updates=gsup) 41 | 42 | updates = [] 43 | 44 | for p, g in zip(tparams.values(), gshared): 45 | m = theano.shared(p.get_value() * 0.) 46 | m_new = momentum * m - lr * g 47 | updates.append((m, m_new)) 48 | 49 | updated_p = p + m_new 50 | updates.append((p, updated_p)) 51 | 52 | f_update = theano.function([lr], [], updates=updates) 53 | 54 | return f_grad_shared, f_update 55 | 56 | def NAG(tparams, cost, inps, lr, momentum=0.9,clip_norm=5): 57 | """ default: lr=0.01 """ 58 | 59 | grads = tensor.grad(cost, tparams.values()) 60 | norm = tensor.sqrt(sum([tensor.sum(g**2) for g in grads])) 61 | if tensor.ge(norm, clip_norm): 62 | grads = [g*clip_norm/norm for g in grads] 63 | 64 | gshared = [theano.shared(p.get_value() * 0., name='%s_grad'%k) 65 | for k, p in tparams.iteritems()] 66 | gsup = [(gs, g) for gs, g in zip(gshared, grads)] 67 | f_grad_shared = theano.function(inps, cost, updates=gsup) 68 | 69 | updates = [] 70 | 71 | for p, g in zip(tparams.values(), gshared): 72 | m = theano.shared(p.get_value() * 0.) 73 | m_new = momentum * m - lr * g 74 | updates.append((m, m_new)) 75 | 76 | updated_p = p + momentum * m_new - lr * g 77 | updates.append((p, updated_p)) 78 | 79 | f_update = theano.function([lr], [], updates=updates) 80 | 81 | return f_grad_shared, f_update 82 | 83 | def Adagrad(tparams, cost, inps, lr, epsilon=1e-6,clip_norm=5): 84 | """ default: lr=0.01 """ 85 | 86 | grads = tensor.grad(cost, tparams.values()) 87 | norm = tensor.sqrt(sum([tensor.sum(g**2) for g in grads])) 88 | if tensor.ge(norm, clip_norm): 89 | grads = [g*clip_norm/norm for g in grads] 90 | 91 | gshared = [theano.shared(p.get_value() * 0., name='%s_grad'%k) 92 | for k, p in tparams.iteritems()] 93 | gsup = [(gs, g) for gs, g in zip(gshared, grads)] 94 | f_grad_shared = theano.function(inps, cost, updates=gsup) 95 | 96 | updates = [] 97 | 98 | for p, g in zip(tparams.values(), gshared): 99 | acc = theano.shared(p.get_value() * 0.) 100 | acc_t = acc + g ** 2 101 | updates.append((acc, acc_t)) 102 | p_t = p - (lr / tensor.sqrt(acc_t + epsilon)) * g 103 | updates.append((p, p_t)) 104 | 105 | f_update = theano.function([lr], [], updates=updates) 106 | 107 | return f_grad_shared, f_update 108 | 109 | def Adadelta(tparams, cost, inps, lr, rho=0.95, epsilon=1e-6,clip_norm=5): 110 | """ default: lr=0.5 """ 111 | 112 | grads = tensor.grad(cost, tparams.values()) 113 | norm = tensor.sqrt(sum([tensor.sum(g**2) for g in grads])) 114 | if tensor.ge(norm, clip_norm): 115 | grads = [g*clip_norm/norm for g in grads] 116 | 117 | gshared = [theano.shared(p.get_value() * 0., name='%s_grad'%k) 118 | for k, p in tparams.iteritems()] 119 | gsup = [(gs, g) for gs, g in zip(gshared, grads)] 120 | f_grad_shared = theano.function(inps, cost, updates=gsup) 121 | 122 | updates = [] 123 | 124 | for p, g in zip(tparams.values(), gshared): 125 | acc = theano.shared(p.get_value() * 0.) 126 | acc_delta = theano.shared(p.get_value() * 0.) 127 | acc_new = rho * acc + (1 - rho) * g ** 2 128 | updates.append((acc,acc_new)) 129 | 130 | update = g * tensor.sqrt(acc_delta + epsilon) / tensor.sqrt(acc_new + epsilon) 131 | updated_p = p - lr * update 132 | updates.append((p, updated_p)) 133 | 134 | acc_delta_new = rho * acc_delta + (1 - rho) * update ** 2 135 | updates.append((acc_delta,acc_delta_new)) 136 | 137 | f_update = theano.function([lr], [], updates=updates) 138 | 139 | return f_grad_shared, f_update 140 | 141 | 142 | def RMSprop_v1(tparams, cost, inps, lr, rho=0.9, epsilon=1e-6,clip_norm=5): 143 | """ default: lr=0.001 144 | This is the implementation of the RMSprop algorithm used in 145 | http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf. 146 | """ 147 | 148 | grads = tensor.grad(cost, tparams.values()) 149 | norm = tensor.sqrt(sum([tensor.sum(g**2) for g in grads])) 150 | if tensor.ge(norm, clip_norm): 151 | grads = [g*clip_norm/norm for g in grads] 152 | 153 | gshared = [theano.shared(p.get_value() * 0., name='%s_grad'%k) 154 | for k, p in tparams.iteritems()] 155 | gsup = [(gs, g) for gs, g in zip(gshared, grads)] 156 | f_grad_shared = theano.function(inps, cost, updates=gsup) 157 | 158 | updates = [] 159 | 160 | for p, g in zip(tparams.values(), gshared): 161 | acc = theano.shared(p.get_value() * 0.) 162 | acc_new = rho * acc + (1 - rho) * g ** 2 163 | updates.append((acc, acc_new)) 164 | 165 | updated_p = p - lr * (g / tensor.sqrt(acc_new + epsilon)) 166 | updates.append((p, updated_p)) 167 | 168 | f_update = theano.function([lr], [], updates=updates) 169 | 170 | return f_grad_shared, f_update 171 | 172 | def RMSprop_v2(tparams, cost, inps, lr, rho=0.95, momentum=0.9, epsilon=1e-4, clip_norm=5): 173 | """ default: lr=0.0001 174 | This is the implementation of the RMSprop algorithm used in 175 | http://arxiv.org/pdf/1308.0850v5.pdf 176 | """ 177 | 178 | grads = tensor.grad(cost, tparams.values()) 179 | norm = tensor.sqrt(sum([tensor.sum(g**2) for g in grads])) 180 | if tensor.ge(norm, clip_norm): 181 | grads = [g*clip_norm/norm for g in grads] 182 | 183 | gshared = [theano.shared(p.get_value() * 0., name='%s_grad'%k) 184 | for k, p in tparams.iteritems()] 185 | gsup = [(gs, g) for gs, g in zip(gshared, grads)] 186 | f_grad_shared = theano.function(inps, cost, updates=gsup) 187 | 188 | updates = [] 189 | 190 | for p, g in zip(tparams.values(), gshared): 191 | acc = theano.shared(p.get_value() * 0.) 192 | acc2 = theano.shared(p.get_value() * 0.) 193 | acc_new = rho * acc + (1.-rho) * g 194 | acc2_new = rho * acc + (1.-rho) * (g ** 2) 195 | updates.append((acc, acc_new)) 196 | updates.append((acc2, acc2_new)) 197 | 198 | updir = theano.shared(p.get_value() * 0.) 199 | updir_new = momentum * updir - lr * g / tensor.sqrt(acc2_new -acc_new ** 2 + epsilon) 200 | updates.append((updir, updir_new)) 201 | 202 | updated_p = p + updir_new 203 | updates.append((p, updated_p)) 204 | 205 | f_update = theano.function([lr], [], updates=updates) 206 | 207 | return f_grad_shared, f_update 208 | 209 | def Adam(tparams, cost, inps, lr, b1=0.1, b2=0.001, e=1e-8, clip_norm=5): 210 | """ default: lr=0.0002 211 | This is the implementation of the Adam algorithm 212 | Reference: http://arxiv.org/pdf/1412.6980v8.pdf 213 | """ 214 | 215 | grads = tensor.grad(cost, tparams.values()) 216 | norm = tensor.sqrt(sum([tensor.sum(g**2) for g in grads])) 217 | if tensor.ge(norm, clip_norm): 218 | grads = [g*clip_norm/(norm + e) for g in grads] 219 | zero = numpy.float32(0) 220 | gshared = [theano.shared(p.get_value() * zero, name='%s_grad'%k) 221 | for k, p in tparams.iteritems()] 222 | 223 | gsup = [(gs, g) for gs, g in zip(gshared, grads)] 224 | f_grad_shared = theano.function(inps, cost, updates=gsup) 225 | updates = [] 226 | 227 | i = theano.shared(numpy_floatX(0.)) 228 | i_t = i + 1. 229 | fix1 = 1. - b1**(i_t) 230 | fix2 = 1. - b2**(i_t) 231 | lr_t = lr * (tensor.sqrt(fix2) / fix1) 232 | _s = tensor.scalar('s',dtype = 'float32') 233 | for p, g in zip(tparams.values(), gshared): 234 | m = theano.shared(p.get_value() * zero) 235 | v = theano.shared(p.get_value() * zero) 236 | m_t = (b1 * g) + ((1. - b1) * m) 237 | v_t = (b2 * tensor.sqr(g)) + ((1. - b2) * v) 238 | g_t = m_t / (tensor.sqrt(v_t) + e) 239 | p_t = p - (lr_t * g_t) 240 | if tensor.eq(_s,0.) and (p.name is 'gp_beta' or p.name is 'gp_alpha' or p.name is 'r'): 241 | p_t = p - (_s * lr_t * g_t) 242 | #elif tensor.eq(_s,1.) and (p.name is not 'gp_beta' and p.name is not 'gp_alpha' and p.name is not 'r'): 243 | # p_t = p - ((1-_s) * lr_t * g_t) 244 | if p.name == 'e_beta' or p.name == 'd_beta': 245 | p_t = p_t * (p_t>0) 246 | elif p.name is 'gp_beta' or p.name is 'gp_alpha': 247 | m_t = m_t.astype('float32') 248 | v_t = v_t.astype('float32') 249 | p_t = p_t.astype('float32') 250 | updates.append((m, m_t)) 251 | updates.append((v, v_t)) 252 | updates.append((p, p_t)) 253 | updates.append((i, i_t)) 254 | 255 | f_update = theano.function([lr,_s], [], updates=updates) 256 | 257 | return f_grad_shared, f_update 258 | -------------------------------------------------------------------------------- /Theano_Version/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import theano 3 | import os 4 | import scipy.io 5 | from theano import config 6 | from collections import OrderedDict 7 | 8 | def Read_Autism_cross(dir_path,key): 9 | train = [] 10 | val = [] 11 | test = [] 12 | labtrain = [] 13 | labtrain2 = [] 14 | labval = [] 15 | labtest = [] 16 | pool = [] 17 | lab_dic = {} 18 | lab_dic2 = {} 19 | for i in range(1,26): 20 | if i in key: 21 | continue 22 | pool.append(str(i).zfill(3)) 23 | lab_dic2[str(i).zfill(3)] = i - 1 24 | pool = np.array(pool) 25 | shuffle = np.random.permutation(len(pool)) 26 | N1 = int(np.ceil(0.8 * len(pool))) 27 | train_pool = pool[shuffle[:N1]] 28 | val_pool = pool[shuffle[N1:]] 29 | files = [f for f in os.listdir(dir_path) if f.endswith('.mat')] 30 | for k in range(len(key)): 31 | key[k] = str(key[k]).zfill(3) 32 | lab_dic['T1'] = 0 33 | lab_dic['T2'] = 1 34 | lab_dic['T3'] = 2 35 | for idx in range(len(files)): 36 | f_id = files[idx][18:21] 37 | f_lab = files[idx][22:24] 38 | mat = scipy.io.loadmat(dir_path + files[idx]) 39 | data = mat['trial'].astype('float32') 40 | if f_id in train_pool: 41 | train.append(data.astype('float32')) 42 | labtrain.append(lab_dic[f_lab] * np.ones((data.shape[0],))) 43 | labtrain2.append(lab_dic2[f_id] * np.ones((data.shape[0],))) 44 | elif f_id in val_pool: 45 | val.append(data.astype('float32')) 46 | labval.append(lab_dic[f_lab] * np.ones((data.shape[0],))) 47 | elif f_id in key: 48 | test.append(data.astype('float32')) 49 | labtest.append(lab_dic[f_lab] * np.ones((data.shape[0],))) 50 | else: 51 | print 'Error in Reading' + f_id 52 | train = np.concatenate(train,axis = 0) 53 | val = np.concatenate(val,axis = 0) 54 | test = np.concatenate(test,axis = 0) 55 | labtrain = np.concatenate(labtrain,axis = 0).astype('int32') 56 | labtrain2 = np.concatenate(labtrain2,axis = 0).astype('int32') 57 | labval = np.concatenate(labval,axis = 0).astype('int32') 58 | labtest = np.concatenate(labtest,axis = 0).astype('int32') 59 | return train,val,test,labtrain,labtrain2,labval,labtest 60 | 61 | 62 | 63 | def numpy_floatX(data): 64 | return np.asarray(data, dtype=config.floatX) 65 | 66 | def zipp(params, tparams): 67 | """ 68 | When we reload the model. Needed for the GPU stuff. 69 | """ 70 | for kk, vv in params.iteritems(): 71 | tparams[kk].set_value(vv) 72 | 73 | def unzip(zipped): 74 | """ 75 | When we pickle the model. Needed for the GPU stuff. 76 | """ 77 | new_params = OrderedDict() 78 | for kk, vv in zipped.iteritems(): 79 | new_params[kk] = vv.get_value() 80 | return new_params 81 | 82 | 83 | def get_minibatches_idx(n, minibatch_size, shuffle=False): 84 | idx_list = np.arange(n, dtype="int32") 85 | 86 | if shuffle: 87 | np.random.shuffle(idx_list) 88 | 89 | minibatches = [] 90 | minibatch_start = 0 91 | for i in range(n // minibatch_size): 92 | minibatches.append(idx_list[minibatch_start: 93 | minibatch_start + minibatch_size]) 94 | minibatch_start += minibatch_size 95 | 96 | if (minibatch_start != n): 97 | # Make a minibatch out of what is left 98 | minibatches.append(idx_list[minibatch_start:]) 99 | 100 | return zip(range(len(minibatches)), minibatches) 101 | 102 | def _p(pp, name): 103 | return '%s_%s' % (pp, name) 104 | 105 | def dropout(X, trng, p=0.): 106 | if p != 0: 107 | retain_prob = 1 - p 108 | X = X / retain_prob * trng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX) 109 | return X 110 | 111 | def dropout_channel(X, trng, p=0.): 112 | if p != 0: 113 | retain_prob = 1 - p 114 | C = X.shape[1] 115 | N = X.shape[0] 116 | _n = trng.binomial((N,C), p = retain_prob, dtype = theano.config.floatX).dimshuffle(0,1,'x') 117 | X = X / retain_prob * _n 118 | return X 119 | """ used for initialization of the parameters. """ 120 | 121 | def ortho_weight(ndim): 122 | W = np.random.randn(ndim, ndim) 123 | u, s, v = np.linalg.svd(W) 124 | return u.astype(config.floatX) 125 | 126 | def uniform_weight(nin,nout=None, lowscale=-0.05, highscale = 0.05): 127 | if nout == None: 128 | nout = nin 129 | W = np.random.uniform(low=lowscale, high=highscale, size=(nin, nout)) 130 | return W.astype(config.floatX) 131 | 132 | def normal_weight(nin,nout=None, scale=0.05): 133 | if nout == None: 134 | nout = nin 135 | W = np.random.randn(nin, nout) * scale 136 | return W.astype(config.floatX) 137 | 138 | def zero_bias(ndim): 139 | b = np.zeros((ndim,)) 140 | return b.astype(config.floatX) 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | -------------------------------------------------------------------------------- /make_synthetic_data.m: -------------------------------------------------------------------------------- 1 | set(0,'defaultTextInterpreter','latex') 2 | %% 3 | sr=50; %sampling rate 4 | t=linspace(0,1,sr); 5 | dt=bsxfun(@minus,t',t); 6 | w=10*2*pi; 7 | beta=40; 8 | co = @(dt) exp(-beta.*dt.^2).*exp(j*dt.*w); 9 | K_block=co(dt); 10 | %% 11 | C=8; 12 | tmp=randn(C,C)+j*randn(C,C); 13 | %S0= tmp'*tmp; 14 | S0 = eye(C); 15 | shat=(linspace(.5,2,C).*exp([0:C-1]/C*2*pi*j)); 16 | S1=S0+shat'*shat; 17 | %% generate data 18 | K0=real(kron(S0,K_block)); 19 | K1=real(kron(S1,K_block)); 20 | sig=.1; 21 | K0n=K0+eye(size(K0))*sig; 22 | K1n=K1+eye(size(K1))*sig; 23 | g0=chol(K0n)'; 24 | g1=chol(K1n)'; 25 | N=12000; 26 | n0=6000; 27 | x=zeros(numel(t),C,N); 28 | for n=1:N 29 | if mod(n,1000) == 0 30 | fprintf('n=%d,\n',n); 31 | end 32 | if n<=n0 33 | x(:,:,n)=reshape(g0*randn(numel(t)*C,1),numel(t),C); 34 | else 35 | x(:,:,n)=reshape(g1*randn(numel(t)*C,1),numel(t),C); 36 | end 37 | end 38 | %% 39 | figure(1); 40 | plot(t,x(:,:,1)) 41 | xlabel('Time, \it Seconds') 42 | ylabel('Amplitude') 43 | title('Example Signal') 44 | figure(2); 45 | plot(shat); 46 | xlabel('Real') 47 | ylabel('Imaginary') 48 | title('Best Separating Vector') 49 | %% other figure 50 | figure(3) 51 | mw=co(t-.5); 52 | xt=x(:,:,1); 53 | xi=conv2(xt,mw(:),'same'); 54 | pr=xi*shat'; 55 | plot(t,real(pr),'k-',... 56 | t,abs(pr),'b--','LineWidth',3) 57 | set(gca,'FontSize',24) 58 | xlabel('Time, \it Seconds') 59 | ylabel('$h^{(k)}(t)$') 60 | % orient landscape 61 | % print -dpdf circular_invariance 62 | 63 | 64 | 65 | 66 | %% 67 | data = permute(x,[3,2,1]); 68 | label = [zeros(n0,1);ones(N - n0,1)]; 69 | shuffle = randperm(N); 70 | data = data(shuffle,:,:); 71 | label = label(shuffle); 72 | N1 = fix(0.7 * N); 73 | N2 = fix(0.8 * N); 74 | train = data(1:N1,:,:); 75 | labtrain = label(1:N1, :,:); 76 | val = data(N1 + 1:N2,:,:); 77 | labval = label(N1+1:N2); 78 | test = data(N2+1:end,:,:); 79 | labtest = label(N2+1:end); 80 | save('./toy.mat','train','val','test','labtrain','labval','labtest'); 81 | -------------------------------------------------------------------------------- /plot_d_b.m: -------------------------------------------------------------------------------- 1 | params = load('./result.mat'); 2 | params.b = squeeze(params.b); 3 | params.phi = squeeze(params.phi); 4 | figure('Color',[1,1,1]); 5 | %plot(shat,'.-'); hold on; 6 | [C,K] = size(params.b); 7 | b = params.b; 8 | phi = [zeros(K,1);params.phi]; 9 | plot(-b.*exp(j*phi),'.-'); hold on; -------------------------------------------------------------------------------- /syncnet.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | from SyncNetModel import SyncNetModel 4 | import scipy.io 5 | import os 6 | import utils 7 | 8 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 9 | gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.4) 10 | 11 | 12 | batch_size = 10 13 | num_steps = 2000 14 | valid_steps = 1000 15 | mat = scipy.io.loadmat('toy.mat') 16 | train = mat['train'].astype('float32') 17 | val = mat['val'].astype('float32') 18 | test = mat['test'].astype('float32') 19 | labtrain = utils.to_one_hot(mat['labtrain']) 20 | labval = utils.to_one_hot(mat['labval']) 21 | labtest = utils.to_one_hot(mat['labtest']) 22 | train = train.transpose(0,2,1) 23 | val = val.transpose(0,2,1) 24 | test = test.transpose(0,2,1) 25 | options = {} 26 | options['sample_shape'] = (train.shape[1],train.shape[2]) 27 | options['num_labels'] = labtrain.shape[1] 28 | options['batch_size'] = batch_size 29 | options['C'] = train.shape[2] 30 | options['T'] = train.shape[1] 31 | options['K'] = 1 32 | options['Nt'] = 40 33 | options['pool_size'] = 40 34 | options['dropout_rate'] = 0.0 35 | options['cl_Wy'] = int(np.ceil(float(options['T'])/float(options['pool_size'])) * options['K']) 36 | tf.reset_default_graph() 37 | graph = tf.get_default_graph() 38 | gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.4) 39 | 40 | model = SyncNetModel(options) 41 | sess = tf.Session(graph = graph, config=tf.ConfigProto(gpu_options=gpu_options)) 42 | tf.global_variables_initializer().run(session = sess) 43 | gen_batches = utils.batch_generator([train, labtrain], options['batch_size']) 44 | 45 | print('Training...') 46 | for i in range(1, num_steps + 1): 47 | p = float(i) / num_steps 48 | #lr = 0.002 #/ (1. + 10 * p)**0.75 49 | lr = 0.002 50 | X, y = gen_batches.next() 51 | _, batch_loss, y_acc = \ 52 | sess.run([model.train_ops, model.y_loss, model.y_acc], 53 | feed_dict={model.X: X, model.y: y, model.lr: lr}) 54 | _ = sess.run(model.beta_op, feed_dict = {}) 55 | if i % 100 == 0: 56 | print 'iter %d loss: %f p_acc: %f lr: %f' % \ 57 | (i, batch_loss, y_acc, lr) 58 | 59 | 60 | if i % valid_steps == 0: 61 | train_pred, train_acc = sess.run([model.y_pred, model.y_acc], feed_dict = {model.X: train, model.y:labtrain}) 62 | 63 | val_pred, val_acc = sess.run([model.y_pred, model.y_acc], feed_dict = {model.X: val, model.y:labval}) 64 | 65 | test_pred, test_acc = sess.run([model.y_pred, model.y_acc], feed_dict = {model.X: test, model.y:labtest}) 66 | 67 | print 'train: %.4f valid: %.4f test: %.4f ' % \ 68 | (train_acc, val_acc, test_acc) 69 | params = utils.get_params(sess) 70 | np.save('./result.npy',params) 71 | #result = utils.get_params(sess) 72 | #result_for_save = {} 73 | #keys = result.keys() 74 | #for i in range(len(keys)): 75 | # temp = keys[i].split('/') 76 | # temp = temp[1] 77 | # temp = temp[:-2] 78 | # result_for_save[temp] = result[keys[i]] 79 | #scipy.io.savemat('./result.mat',{'b':result_for_save['b'], 'phi':result_for_save['phi']}) 80 | # -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import matplotlib.patches as mpatches 4 | import matplotlib.pyplot as plt 5 | from mpl_toolkits.axes_grid1 import ImageGrid 6 | import scipy 7 | # Model construction utilities below adapted from 8 | # https://www.tensorflow.org/versions/r0.8/tutorials/mnist/pros/index.html#deep-mnist-for-experts 9 | 10 | def get_params(sess): 11 | variables = tf.trainable_variables() 12 | params = {} 13 | for i in range(len(variables)): 14 | name = variables[i].name 15 | params[name] = sess.run(variables[i]) 16 | return params 17 | 18 | 19 | def to_one_hot(x, N = -1): 20 | x = x.astype('int32') 21 | if np.min(x) !=0 and N == -1: 22 | x = x - np.min(x) 23 | x = x.reshape(-1) 24 | if N == -1: 25 | N = np.max(x) + 1 26 | label = np.zeros((x.shape[0],N)) 27 | idx = range(x.shape[0]) 28 | label[idx,x] = 1 29 | return label.astype('float32') 30 | 31 | def image_mean(x): 32 | x_mean = x.mean((0, 1, 2)) 33 | return x_mean 34 | 35 | def shape(tensor): 36 | """ 37 | Get the shape of a tensor. This is a compile-time operation, 38 | meaning that it runs when building the graph, not running it. 39 | This means that it cannot know the shape of any placeholders 40 | or variables with shape determined by feed_dict. 41 | """ 42 | return tuple([d.value for d in tensor.get_shape()]) 43 | 44 | 45 | def fully_connected_layer(in_tensor, out_units): 46 | """ 47 | Add a fully connected layer to the default graph, taking as input `in_tensor`, and 48 | creating a hidden layer of `out_units` neurons. This should be done in a new variable 49 | scope. Creates variables W and b, and computes activation_function(in * W + b). 50 | """ 51 | _, num_features = shape(in_tensor) 52 | weights = tf.get_variable(name = "weights", shape = [num_features, out_units], initializer = tf.truncated_normal_initializer(stddev=0.1)) 53 | biases = tf.get_variable( name = "biases", shape = [out_units], initializer=tf.constant_initializer(0.1)) 54 | return tf.matmul(in_tensor, weights) + biases 55 | 56 | 57 | def conv2d(in_tensor, filter_shape, out_channels): 58 | """ 59 | Creates a conv2d layer. The input image (whish should already be shaped like an image, 60 | a 4D tensor [N, W, H, C]) is convolved with `out_channels` filters, each with shape 61 | `filter_shape` (a width and height). The ReLU activation function is used on the 62 | output of the convolution. 63 | """ 64 | _, _, _, channels = shape(in_tensor) 65 | W_shape = filter_shape + [channels, out_channels] 66 | 67 | # create variables 68 | weights = tf.get_variable(name = "weights", shape = W_shape, initializer=tf.truncated_normal_initializer(stddev=0.1)) 69 | biases = tf.get_variable(name = "biases", shape = [out_channels], initializer= tf.constant_initializer(0.1)) 70 | conv = tf.nn.conv2d( in_tensor, weights, strides=[1, 1, 1, 1], padding='SAME') 71 | h_conv = conv + biases 72 | return h_conv 73 | 74 | 75 | #def conv1d(in_tensor, filter_shape, out_channels): 76 | # _, _, channels = shape(in_tensor) 77 | # W_shape = [filter_shape, channels, out_channels] 78 | # 79 | # W = tf.truncated_normal(W_shape, dtype = tf.float32, stddev = 0.1) 80 | # weights = tf.Variable(W, name = "weights") 81 | # b = tf.truncated_normal([out_channels], dtype = tf.float32, stddev = 0.1) 82 | # biases = tf.Variable(b, name = "biases") 83 | # conv = tf.nn.conv1d(in_tensor, weights, stride=1, padding='SAME') 84 | # h_conv = conv + biases 85 | # return h_conv 86 | 87 | def vars_from_scopes(scopes): 88 | """ 89 | Returns list of all variables from all listed scopes. Operates within the current scope, 90 | so if current scope is "scope1", then passing in ["weights", "biases"] will find 91 | all variables in scopes "scope1/weights" and "scope1/biases". 92 | """ 93 | current_scope = tf.get_variable_scope().name 94 | #print(current_scope) 95 | if current_scope != '': 96 | scopes = [current_scope + '/' + scope for scope in scopes] 97 | var = [] 98 | for scope in scopes: 99 | for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=scope): 100 | var.append(v) 101 | return var 102 | 103 | def tfvar2str(tf_vars): 104 | names = [] 105 | for i in range(len(tf_vars)): 106 | names.append(tf_vars[i].name) 107 | return names 108 | 109 | 110 | def shuffle_aligned_list(data): 111 | """Shuffle arrays in a list by shuffling each array identically.""" 112 | num = data[0].shape[0] 113 | p = np.random.permutation(num) 114 | return [d[p] for d in data] 115 | 116 | 117 | def batch_generator(data, batch_size, shuffle=True): 118 | """Generate batches of data. 119 | 120 | Given a list of array-like objects, generate batches of a given 121 | size by yielding a list of array-like objects corresponding to the 122 | same slice of each input. 123 | """ 124 | if shuffle: 125 | data = shuffle_aligned_list(data) 126 | 127 | batch_count = 0 128 | while True: 129 | if batch_count * batch_size + batch_size >= len(data[0]): 130 | batch_count = 0 131 | 132 | if shuffle: 133 | data = shuffle_aligned_list(data) 134 | 135 | start = batch_count * batch_size 136 | end = start + batch_size 137 | batch_count += 1 138 | yield [d[start:end] for d in data] 139 | 140 | 141 | 142 | 143 | def predictor_accuracy(predictions, labels): 144 | """ 145 | Returns a number in [0, 1] indicating the percentage of `labels` predicted 146 | correctly (i.e., assigned max logit) by `predictions`. 147 | """ 148 | return tf.reduce_mean(tf.cast(tf.equal(tf.argmax(predictions, 1), tf.argmax(labels, 1)),tf.float32)) 149 | 150 | def dic2list(sources, targets): 151 | names_dic = {} 152 | for key in sources: 153 | names_dic[sources[key]] = key 154 | for key in targets: 155 | names_dic[targets[key]] = key 156 | names = [] 157 | for i in range(len(names_dic)): 158 | names.append(names_dic[i]) 159 | return names 160 | 161 | def softmax(x): 162 | """Compute softmax values for each sets of scores in x.""" 163 | e_x = np.exp(x - np.max(x)) 164 | return e_x / e_x.sum(axis=0) 165 | 166 | def norm_matrix(X, l): 167 | Y = np.zeros(X.shape); 168 | for i in range(X.shape[0]): 169 | Y[i] = X[i]/np.linalg.norm(X[i],l) 170 | return Y 171 | 172 | 173 | def description(sources, targets): 174 | source_names = sources.keys() 175 | target_names = targets.keys() 176 | N = min(len(source_names), 4) 177 | description = source_names[0] 178 | for i in range(1,N): 179 | description = description + '_' + source_names[i] 180 | description = description + '-' + target_names[0] 181 | return description 182 | 183 | def channel_dropout(X, p): 184 | if p == 0: 185 | return X 186 | mask = tf.random_uniform(shape = [tf.shape(X)[0], tf.shape(X)[2]]) 187 | mask = mask + 1 - p 188 | mask = tf.floor(mask) 189 | dropout = tf.expand_dims(mask,axis = 1) * X/(1-p) 190 | return dropout 191 | 192 | def sigmoid(x): 193 | return 1 / (1 + np.exp(-x)) --------------------------------------------------------------------------------