├── README.md ├── grey10_icbm_3mm_bin.nii.gz ├── nips3mm.py ├── nips3mm_h38.py ├── nips3mm_recovery.py ├── nips3mm_scarcity.py ├── nips3mm_serial.py └── nips3mm_vanilla.py /README.md: -------------------------------------------------------------------------------- 1 | NIPS 2015 2 | ========= 3 | 4 | Code for reproducing the key results of our NIPS 2015 paper on semi-supervised low-rank logistic regression models for large functional neuroimaging datasets. 5 | 6 | Bzdok D, Eickenberg M, Grisel O, Thirion B, Varoquaux G. 7 | **Semi-supervised Factored Logistic Regression for High-Dimensional Neuroimaging Data** 8 | Advances in Neural Information Processing Systems (**NIPS 2015**), Montreal. 9 | [Paper on ResearchGate](https://www.researchgate.net/publication/281490102_Semi-Supervised_Factored_Logistic_Regression_for_High-Dimensional_Neuroimaging_Data) 10 | 11 | Please cite this paper when using the code for your research. 12 | 13 | To follow established conventions of scikit-learn estimators, the ``SSEncoder`` class exposes the functions fit(), predict(), and score(). 14 | This should allow for seamless integration into other scikit-learn-enabled machine-learning pipelines. 15 | 16 | For questions and bug reports, please send me an e-mail at _danilobzdok[at]gmail.com_. 17 | 18 | ## Prerequisites 19 | 20 | 1. Make sure that recent versions of the following packages are available: 21 | - Python (version 2.7 or higher) 22 | - Numpy (e.g. `pip install numpy`) 23 | - Theano (e.g. `pip install Theano`) 24 | - Nilearn (e.g., `pip install nilearn`) 25 | - Nibabel (e.g., `pip install nibabel`) 26 | 27 | 2. Set `floatX = float32` in the `[global]` section of Theano config (usually `~/.theanorc`). Alternatively you could prepend `THEANO_FLAGS=floatX=float32 ` to the python commands. 28 | 29 | 3. Clone this repository, e.g.: 30 | ```sh 31 | git clone https://github.com/banilo/nips2015.git 32 | ``` 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /grey10_icbm_3mm_bin.nii.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/banilo/nips2015/050695f025c359ed39242ce889ebf246c20ae2aa/grey10_icbm_3mm_bin.nii.gz -------------------------------------------------------------------------------- /nips3mm.py: -------------------------------------------------------------------------------- 1 | """ 2 | HCP: Semi-supervised network decomposition by low-rank logistic regression 3 | """ 4 | 5 | print __doc__ 6 | 7 | import os 8 | import os.path as op 9 | import numpy as np 10 | import glob 11 | from scipy.linalg import norm 12 | import nibabel as nib 13 | from sklearn.grid_search import RandomizedSearchCV 14 | from sklearn.base import BaseEstimator 15 | from sklearn.preprocessing import StandardScaler 16 | from nilearn.input_data import NiftiMasker 17 | from sklearn.metrics import precision_recall_fscore_support 18 | from sklearn.metrics import confusion_matrix 19 | import theano 20 | import theano.tensor as T 21 | from matplotlib import pylab as plt 22 | print('Running THEANO on %s' % theano.config.device) 23 | from nilearn.image import concat_imgs 24 | import joblib 25 | import time 26 | 27 | RES_NAME = 'nips3mm' 28 | WRITE_DIR = op.join(os.getcwd(), RES_NAME) 29 | if not op.exists(WRITE_DIR): 30 | os.mkdir(WRITE_DIR) 31 | 32 | ############################################################################## 33 | # load+preprocess data 34 | ############################################################################## 35 | 36 | mask_img = 'grey10_icbm_3mm_bin.nii.gz' 37 | nifti_masker = NiftiMasker(mask_img=mask_img, smoothing_fwhm=False, 38 | standardize=False) 39 | nifti_masker.fit() 40 | mask_nvox = nifti_masker.mask_img_.get_data().sum() 41 | 42 | print('Loading data...') 43 | 44 | # ARCHI task 45 | X_task, labels = joblib.load('preload_HT_3mm') 46 | 47 | labels = np.int32(labels) 48 | 49 | # contrasts are IN ORDER -> shuffle! 50 | new_inds = np.arange(0, X_task.shape[0]) 51 | np.random.shuffle(new_inds) 52 | X_task = X_task[new_inds] 53 | labels = labels[new_inds] 54 | # subs = subs[new_inds] 55 | 56 | # rest 57 | # X_rest = nifti_masker.transform('preload_HR20persub_10mm_ero2.nii') 58 | # X_rest = nifti_masker.transform('dump_rs_spca_s12_tmp') 59 | rs_spca_data = joblib.load('dump_rs_spca_s12_tmp') 60 | rs_spca_niis = nib.Nifti1Image(rs_spca_data, 61 | nifti_masker.mask_img_.get_affine()) 62 | X_rest = nifti_masker.transform(rs_spca_niis) 63 | del rs_spca_niis 64 | del rs_spca_data 65 | 66 | X_task = StandardScaler().fit_transform(X_task) 67 | X_rest = StandardScaler().fit_transform(X_rest) 68 | 69 | # ARCHI task 70 | AT_niis, AT_labels, AT_subs = joblib.load('preload_AT_3mm') 71 | AT_X = nifti_masker.transform(AT_niis) 72 | AT_X = StandardScaler().fit_transform(AT_X) 73 | print('done :)') 74 | 75 | ############################################################################## 76 | # define computation graph 77 | ############################################################################## 78 | 79 | class SSEncoder(BaseEstimator): 80 | def __init__(self, n_hidden, gain1, learning_rate, max_epochs=100, 81 | l1=0.1, l2=0.1, lambda_param=.5): 82 | """ 83 | Parameters 84 | ---------- 85 | lambda : float 86 | Mediates between AE and LR. lambda==1 equates with LR only. 87 | """ 88 | self.n_hidden = n_hidden 89 | self.gain1 = gain1 90 | self.max_epochs = max_epochs 91 | self.learning_rate = np.float32(learning_rate) 92 | self.penalty_l1 = np.float32(l1) 93 | self.penalty_l2 = np.float32(l2) 94 | self.lambda_param = np.float32(lambda_param) 95 | 96 | # def rectify(X): 97 | # return T.maximum(0., X) 98 | 99 | from theano.tensor.shared_randomstreams import RandomStreams 100 | 101 | def RMSprop(self, cost, params, lr=0.001, rho=0.9, epsilon=1e-6): 102 | grads = T.grad(cost=cost, wrt=params) 103 | updates = [] 104 | for p, g in zip(params, grads): 105 | acc = theano.shared(p.get_value() * 0.) 106 | acc_new = rho * acc + (1 - rho) * g ** 2 107 | gradient_scaling = T.sqrt(acc_new + epsilon) 108 | g = g / gradient_scaling 109 | updates.append((acc, acc_new)) 110 | updates.append((p, p - lr * g)) 111 | return updates 112 | 113 | def get_param_pool(self): 114 | cur_params = ( 115 | self.V1s, self.bV0, self.bV1, 116 | self.W0s, self.W1s, self.bW0s, self.bW1s 117 | ) 118 | return cur_params 119 | 120 | def test_performance_in_other_dataset(self): 121 | from sklearn.linear_model import LogisticRegression 122 | from sklearn.cross_validation import StratifiedShuffleSplit 123 | 124 | compr_matrix = self.W0s.get_value().T # currently best compression 125 | AT_X_compr = np.dot(compr_matrix, AT_X.T).T 126 | clf = LogisticRegression(penalty='l1') 127 | folder = StratifiedShuffleSplit(y=AT_labels, n_iter=5, test_size=0.2, 128 | random_state=42) 129 | 130 | acc_list = [] 131 | prfs_list = [] 132 | for (train_inds, test_inds) in folder: 133 | clf.fit(AT_X_compr[train_inds, :], AT_labels[train_inds]) 134 | pred_y = clf.predict(AT_X_compr[test_inds, :]) 135 | 136 | acc = (pred_y == AT_labels[test_inds]).mean() 137 | prfs_list.append(precision_recall_fscore_support( 138 | AT_labels[test_inds], pred_y)) 139 | 140 | acc_list.append(acc) 141 | 142 | compr_mean_acc = np.mean(acc_list) 143 | prfs = np.asarray(prfs_list).mean(axis=0) 144 | return compr_mean_acc, prfs 145 | 146 | def fit(self, X_rest, X_task, y): 147 | DEBUG_FLAG = True 148 | 149 | # self.max_epochs = 333 150 | self.batch_size = 100 151 | n_input = X_rest.shape[1] # sklearn-like structure 152 | n_output = n_input 153 | rng = np.random.RandomState(42) 154 | self.input_taskdata = T.matrix(dtype='float32', name='input_taskdata') 155 | self.input_restdata = T.matrix(dtype='float32', name='input_restdata') 156 | self.params_from_last_iters = [] 157 | 158 | index = T.iscalar(name='index') 159 | 160 | # prepare data for theano computation 161 | if not DEBUG_FLAG: 162 | X_train_s = theano.shared( 163 | value=np.float32(X_task), name='X_train_s') 164 | y_train_s = theano.shared( 165 | value=np.int32(y), name='y_train_s') 166 | lr_train_samples = len(X_task) 167 | else: 168 | from sklearn.cross_validation import StratifiedShuffleSplit 169 | folder = StratifiedShuffleSplit(y, n_iter=1, test_size=0.20) 170 | new_trains, inds_val = iter(folder).next() 171 | X_train, X_val = X_task[new_trains], X_task[inds_val] 172 | y_train, y_val = y[new_trains], y[inds_val] 173 | 174 | X_train_s = theano.shared(value=np.float32(X_train), 175 | name='X_train_s', borrow=False) 176 | y_train_s = theano.shared(value=np.int32(y_train), 177 | name='y_train_s', borrow=False) 178 | # X_val_s = theano.shared(value=np.float32(X_val), 179 | # name='X_train_s', borrow=False) 180 | # y_val_s = theano.shared(value=np.int32(y_val), 181 | # name='y_cal_s', borrow=False) 182 | lr_train_samples = len(X_train) 183 | self.dbg_epochs_ = list() 184 | self.dbg_acc_train_ = list() 185 | self.dbg_acc_val_ = list() 186 | self.dbg_ae_cost_ = list() 187 | self.dbg_lr_cost_ = list() 188 | self.dbg_ae_nonimprovesteps = list() 189 | self.dbg_acc_other_ds_ = list() 190 | self.dbg_combined_cost_ = list() 191 | self.dbg_prfs_ = list() 192 | self.dbg_prfs_other_ds_ = list() 193 | X_rest_s = theano.shared(value=np.float32(X_rest), name='X_rest_s') 194 | ae_train_samples = len(X_rest) 195 | 196 | # V -> supervised / logistic regression 197 | # W -> unsupervised / auto-encoder 198 | 199 | # computational graph: auto-encoder 200 | W0_vals = rng.randn(n_input, self.n_hidden).astype(np.float32) * self.gain1 201 | 202 | self.W0s = theano.shared(W0_vals) 203 | self.W1s = self.W0s.T # tied 204 | bW0_vals = np.zeros(self.n_hidden).astype(np.float32) 205 | self.bW0s = theano.shared(value=bW0_vals, name='bW0') 206 | bW1_vals = np.zeros(n_output).astype(np.float32) 207 | self.bW1s = theano.shared(value=bW1_vals, name='bW1') 208 | 209 | givens_ae = { 210 | self.input_restdata: X_rest_s[ 211 | index * self.batch_size:(index + 1) * self.batch_size] 212 | } 213 | 214 | encoding = (self.input_restdata.dot(self.W0s) + self.bW0s).dot(self.W1s) + self.bW1s 215 | 216 | self.ae_loss = T.sum((self.input_restdata - encoding) ** 2, axis=1) 217 | 218 | self.ae_cost = ( 219 | T.mean(self.ae_loss) / n_input 220 | ) 221 | 222 | # params1 = [self.W0s, self.bW0s, self.bW1s] 223 | # gparams1 = [T.grad(cost=self.ae_cost, wrt=param1) for param1 in params1] 224 | # 225 | # lr = self.learning_rate 226 | # updates = self.RMSprop(cost=self.ae_cost, params=params1, 227 | # lr=self.learning_rate) 228 | 229 | # f_train_ae = theano.function( 230 | # [index], 231 | # [self.ae_cost], 232 | # givens=givens_ae, 233 | # updates=updates) 234 | 235 | # computation graph: logistic regression 236 | clf_n_output = 18 # number of labels 237 | my_y = T.ivector(name='y') 238 | 239 | bV0_vals = np.zeros(self.n_hidden).astype(np.float32) 240 | self.bV0 = theano.shared(value=bV0_vals, name='bV0') 241 | bV1_vals = np.zeros(clf_n_output).astype(np.float32) 242 | self.bV1 = theano.shared(value=bV1_vals, name='bV1') 243 | 244 | # V0_vals = rng.randn(n_input, self.n_hidden).astype(np.float32) * self.gain1 245 | V1_vals = rng.randn(self.n_hidden, clf_n_output).astype(np.float32) * self.gain1 246 | # self.V0s = theano.shared(V0_vals) 247 | self.V1s = theano.shared(V1_vals) 248 | 249 | self.p_y_given_x = T.nnet.softmax( 250 | # T.dot(T.dot(self.input_taskdata, self.V0s) + self.bV0, self.V1s) + self.bV1 251 | T.dot(T.dot(self.input_taskdata, self.W0s) + self.bV0, self.V1s) + self.bV1 252 | ) 253 | self.lr_cost = -T.mean(T.log(self.p_y_given_x)[T.arange(my_y.shape[0]), my_y]) 254 | self.lr_cost = ( 255 | self.lr_cost + 256 | T.mean(abs(self.W0s)) * self.penalty_l1 + 257 | # T.mean(abs(self.V0s)) * self.penalty_l1 + 258 | T.mean(abs(self.bV0)) * self.penalty_l1 + 259 | T.mean(abs(self.V1s)) * self.penalty_l1 + 260 | T.mean(abs(self.bV1)) * self.penalty_l1 + 261 | 262 | T.mean((self.W0s ** np.float32(2))) * self.penalty_l2 + 263 | # T.mean((self.V0s ** 2)) * self.penalty_l2 + 264 | T.mean((self.bV0 ** np.float32(2))) * self.penalty_l2 + 265 | T.mean((self.V1s ** np.float32(2))) * self.penalty_l2 + 266 | T.mean((self.bV1 ** np.float32(2))) * self.penalty_l2 267 | ) 268 | self.y_pred = T.argmax(self.p_y_given_x, axis=1) 269 | 270 | givens_lr = { 271 | self.input_taskdata: X_train_s[index * self.batch_size:(index + 1) * self.batch_size], 272 | my_y: y_train_s[index * self.batch_size:(index + 1) * self.batch_size] 273 | } 274 | 275 | # params2 = [self.V0s, self.bV0, self.V1s, self.bV1] 276 | # params2 = [self.W0s, self.bV0, self.V1s, self.bV1] 277 | # updates2 = self.RMSprop(cost=self.lr_cost, params=params2, 278 | # lr=self.learning_rate) 279 | 280 | # f_train_lr = theano.function( 281 | # [index], 282 | # [self.lr_cost], 283 | # givens=givens_lr, 284 | # updates=updates2) 285 | 286 | # combined loss for AE and LR 287 | combined_params = [self.W0s, self.bW0s, self.bW1s, 288 | # self.V0s, self.V1s, self.bV0, self.bV1] 289 | self.V1s, self.bV0, self.bV1] 290 | self.combined_cost = ( 291 | (np.float32(1) - self.lambda_param) * self.ae_cost + 292 | self.lambda_param * self.lr_cost 293 | ) 294 | combined_updates = self.RMSprop( 295 | cost=self.combined_cost, 296 | params=combined_params, 297 | lr=self.learning_rate) 298 | givens_combined = { 299 | self.input_restdata: X_rest_s[index * self.batch_size:(index + 1) * self.batch_size], 300 | self.input_taskdata: X_train_s[index * self.batch_size:(index + 1) * self.batch_size], 301 | my_y: y_train_s[index * self.batch_size:(index + 1) * self.batch_size] 302 | } 303 | f_train_combined = theano.function( 304 | [index], 305 | [self.combined_cost, self.ae_cost, self.lr_cost], 306 | givens=givens_combined, 307 | updates=combined_updates, allow_input_downcast=True) 308 | 309 | # optimization loop 310 | start_time = time.time() 311 | ae_last_cost = np.inf 312 | lr_last_cost = np.inf 313 | no_improve_steps = 0 314 | acc_train, acc_val = 0., 0. 315 | for i_epoch in range(self.max_epochs): 316 | if i_epoch == 1: 317 | epoch_dur = time.time() - start_time 318 | total_mins = (epoch_dur * self.max_epochs) / 60 319 | hs, mins = divmod(total_mins, 60) 320 | print("Max estimated duration: %i hours and %i minutes" % (hs, mins)) 321 | 322 | # AE 323 | # if i_epoch % 2 == 0: # every second time 324 | #if False: 325 | # auto-encoder 326 | ae_n_batches = ae_train_samples // self.batch_size 327 | lr_n_batches = lr_train_samples // self.batch_size 328 | # for i in range(lr_n_batches): 329 | # for i in range(max(ae_n_batches, lr_n_batches)): 330 | # if i < ae_n_batches: 331 | # ae_cur_cost = float(f_train_ae(i)[0]) 332 | # ae_cur_cost = 0 333 | # if i < lr_n_batches: 334 | # lr_cur_cost = float(f_train_lr(i)[0]) 335 | # for i in range(lr_n_batches): 336 | for i in range(min(ae_n_batches, lr_n_batches)): 337 | # lr_cur_cost = f_train_lr(i)[0] 338 | # ae_cur_cost = lr_cur_cost 339 | combined_cost, ae_cur_cost, lr_cur_cost = f_train_combined(i) 340 | 341 | # evaluate epoch cost 342 | if ae_last_cost - ae_cur_cost < 0.1: 343 | no_improve_steps += 1 344 | else: 345 | ae_last_cost = ae_cur_cost 346 | no_improve_steps = 0 347 | 348 | # logistic 349 | lr_last_cost = lr_cur_cost 350 | acc_train = self.score(X_train, y_train) 351 | acc_val, prfs_val = self.score(X_val, y_val, return_prfs=True) 352 | 353 | print('E:%i, ae_cost:%.4f, lr_cost:%.4f, train_score:%.2f, vald_score:%.2f, ae_badsteps:%i' % ( 354 | i_epoch + 1, ae_cur_cost, lr_cur_cost, acc_train, acc_val, no_improve_steps)) 355 | 356 | if (i_epoch % 10 == 0): 357 | self.dbg_ae_cost_.append(ae_cur_cost) 358 | self.dbg_lr_cost_.append(lr_cur_cost) 359 | self.dbg_combined_cost_.append(combined_cost) 360 | 361 | self.dbg_epochs_.append(i_epoch + 1) 362 | self.dbg_ae_nonimprovesteps.append(no_improve_steps) 363 | self.dbg_acc_train_.append(acc_train) 364 | self.dbg_acc_val_.append(acc_val) 365 | self.dbg_prfs_.append(prfs_val) 366 | 367 | # test out-of-dataset performance 368 | od_acc, prfs_other = self.test_performance_in_other_dataset() 369 | self.dbg_acc_other_ds_.append(od_acc) 370 | self.dbg_prfs_other_ds_.append(prfs_other) 371 | print('out-of-dataset acc: %.2f' % od_acc) 372 | 373 | # save paramters from last 100 iterations 374 | if i_epoch > (self.max_epochs - 100): 375 | print('Param pool!') 376 | param_pool = self.get_param_pool() 377 | self.params_from_last_iters.append(param_pool) 378 | 379 | total_mins = (time.time() - start_time) / 60 380 | hs, mins = divmod(total_mins, 60) 381 | print("Final duration: %i hours and %i minutes" % (hs, mins)) 382 | 383 | return self 384 | 385 | def predict(self, X): 386 | X_test_s = theano.shared(value=np.float32(X), name='X_test_s', borrow=True) 387 | 388 | givens_te = { 389 | self.input_taskdata: X_test_s 390 | } 391 | 392 | f_test = theano.function( 393 | [], 394 | [self.y_pred], 395 | givens=givens_te) 396 | predictions = f_test() 397 | del X_test_s 398 | del givens_te 399 | return predictions[0] 400 | 401 | def score(self, X, y, return_prfs=False): 402 | pred_y = self.predict(X) 403 | acc = np.mean(pred_y == y) 404 | prfs = precision_recall_fscore_support(pred_y, y) 405 | if return_prfs: 406 | return acc, prfs 407 | else: 408 | return acc 409 | 410 | 411 | ############################################################################## 412 | # plot figures 413 | ############################################################################## 414 | 415 | def dump_comps(masker, compressor, components, threshold=2, fwhm=None, 416 | perc=None): 417 | from scipy.stats import zscore 418 | from nilearn.plotting import plot_stat_map 419 | from nilearn.image import smooth_img 420 | from scipy.stats import scoreatpercentile 421 | 422 | if isinstance(compressor, basestring): 423 | comp_name = compressor 424 | else: 425 | comp_name = compressor.__str__().split('(')[0] 426 | 427 | for i_c, comp in enumerate(components): 428 | path_mask = op.join(WRITE_DIR, '%s_%i-%i' % (comp_name, 429 | n_comp, i_c + 1)) 430 | nii_raw = masker.inverse_transform(comp) 431 | nii_raw.to_filename(path_mask + '.nii.gz') 432 | 433 | comp_z = zscore(comp) 434 | 435 | if perc is not None: 436 | cur_thresh = scoreatpercentile(np.abs(comp_z), per=perc) 437 | path_mask += '_perc%i' % perc 438 | print('Applying percentile %.2f (threshold: %.2f)' % (perc, cur_thresh)) 439 | else: 440 | cur_thresh = threshold 441 | path_mask += '_thr%.2f' % cur_thresh 442 | print('Applying threshold: %.2f' % cur_thresh) 443 | 444 | nii_z = masker.inverse_transform(comp_z) 445 | gz_path = path_mask + '_zmap.nii.gz' 446 | nii_z.to_filename(gz_path) 447 | plot_stat_map(gz_path, bg_img='colin.nii', threshold=cur_thresh, 448 | cut_coords=(0, -2, 0), draw_cross=False, 449 | output_file=path_mask + 'zmap.png') 450 | 451 | # optional: do smoothing 452 | if fwhm is not None: 453 | nii_z_fwhm = smooth_img(nii_z, fwhm=fwhm) 454 | plot_stat_map(nii_z_fwhm, bg_img='colin.nii', threshold=cur_thresh, 455 | cut_coords=(0, -2, 0), draw_cross=False, 456 | output_file=path_mask + 457 | ('zmap_%imm.png' % fwhm)) 458 | 459 | n_comps = [20] 460 | # n_comps = [40, 30, 20, 10, 5] 461 | for n_comp in n_comps: 462 | # for lambda_param in [0]: 463 | for lambda_param in [0.50]: 464 | l1 = 0.1 465 | l2 = 0.1 466 | my_title = r'Low-rank LR + AE (combined loss, shared decomp): n_comp=%i L1=%.1f L2=%.1f lambda=%.2f res=3mm spca20RS' % ( 467 | n_comp, l1, l2, lambda_param 468 | ) 469 | print(my_title) 470 | estimator = SSEncoder( 471 | n_hidden=n_comp, 472 | gain1=0.004, # empirically determined by CV 473 | learning_rate = np.float32(0.00001), # empirically determined by CV, 474 | max_epochs=500, l1=l1, l2=l2, lambda_param=lambda_param) 475 | 476 | estimator.fit(X_rest, X_task, labels) 477 | 478 | fname = my_title.replace(' ', '_').replace('+', '').replace(':', '').replace('__', '_').replace('%', '') 479 | cur_path = op.join(WRITE_DIR, fname) 480 | joblib.dump(estimator, cur_path) 481 | # estimator = joblib.load(cur_path) 482 | # plt.savefig(cur_path + '_SUMMARY.png', dpi=200) 483 | 484 | # dump data also as numpy array 485 | np.save(cur_path + 'dbg_epochs_', np.array(estimator.dbg_epochs_)) 486 | np.save(cur_path + 'dbg_acc_train_', np.array(estimator.dbg_acc_train_)) 487 | np.save(cur_path + 'dbg_acc_val_', np.array(estimator.dbg_acc_val_)) 488 | np.save(cur_path + 'dbg_ae_cost_', np.array(estimator.dbg_ae_cost_)) 489 | np.save(cur_path + 'dbg_lr_cost_', np.array(estimator.dbg_lr_cost_)) 490 | np.save(cur_path + 'dbg_ae_nonimprovesteps', np.array(estimator.dbg_ae_nonimprovesteps)) 491 | np.save(cur_path + 'dbg_acc_other_ds_', np.array(estimator.dbg_acc_other_ds_)) 492 | np.save(cur_path + 'dbg_combined_cost_', np.array(estimator.dbg_combined_cost_)) 493 | np.save(cur_path + 'dbg_prfs_', np.array(estimator.dbg_prfs_)) 494 | np.save(cur_path + 'dbg_prfs_other_ds_', np.array(estimator.dbg_prfs_other_ds_)) 495 | 496 | W0_mat = estimator.W0s.get_value().T 497 | np.save(cur_path + 'W0comps', W0_mat) 498 | 499 | V1_mat = estimator.V1s.get_value().T 500 | np.save(cur_path + 'V1comps', V1_mat) 501 | # dump_comps(nifti_masker, fname, comps, threshold=0.5) 502 | 503 | STOP_CALCULATION 504 | 505 | # equally scaled plots 506 | import re 507 | pkgs = glob.glob(RES_NAME + '/*dbg_epochs*.npy') 508 | dbg_epochs_ = np.load(pkgs[0]) 509 | dbg_epochs_ = np.load(pkgs[0]) 510 | 511 | d = { 512 | 'training accuracy': '/*dbg_acc_train*.npy', 513 | 'accuracy val': '/*dbg_acc_val_*.npy', 514 | 'accuracy other ds': '/*dbg_acc_other_ds_*.npy', 515 | 'loss ae': '/*dbg_ae_cost_*.npy', 516 | 'loss lr': '/*dbg_lr_cost_*.npy', 517 | 'loss combined': '/*dbg_combined_cost_*.npy' 518 | } 519 | n_comps = [20] 520 | 521 | path_vanilla = 'nips3mm_vanilla' 522 | 523 | for k, v in d.iteritems(): 524 | pkgs = glob.glob(RES_NAME + v) 525 | for n_comp in n_comps: 526 | plt.figure() 527 | for p in pkgs: 528 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 529 | # n_hidden = int(re.search('comp=(?P.{1,2,3})_', p).group('comp')) 530 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 531 | if n_comp != n_hidden: 532 | continue 533 | 534 | dbg_acc_train_ = np.load(p) 535 | 536 | cur_label = 'n_comp=%i' % n_hidden 537 | cur_label += '/' 538 | cur_label += 'lambda=%.2f' % lambda_param 539 | cur_label += '/' 540 | if not '_AE' in p: 541 | cur_label += 'LR only!' 542 | elif 'subRS' in p: 543 | cur_label += 'RSnormal' 544 | elif 'spca20RS' in p: 545 | cur_label += 'RSspca20' 546 | elif 'pca20RS' in p: 547 | cur_label += 'RSpca20' 548 | cur_label += '/' 549 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 550 | cur_label += '' if '_AE' in p else '/LR only!' 551 | plt.plot( 552 | dbg_epochs_, 553 | dbg_acc_train_, 554 | label=cur_label) 555 | if k == 'training accuracy' or k == 'accuracy val': 556 | van_pkgs = glob.glob(path_vanilla + v) 557 | vanilla_values = np.load(van_pkgs[0]) 558 | plt.plot( 559 | dbg_epochs_, 560 | vanilla_values, 561 | label='LR') 562 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss') 563 | plt.legend(loc='lower right', fontsize=9) 564 | plt.yticks(np.linspace(0., 1., 11)) 565 | plt.ylabel(k) 566 | plt.xlabel('epochs') 567 | plt.ylim(0., 1.05) 568 | plt.grid(True) 569 | plt.show() 570 | plt.savefig(op.join(WRITE_DIR, 571 | k.replace(' ', '_') + '_%icomps.png' % n_comp)) 572 | 573 | pkgs = glob.glob(RES_NAME + '/*dbg_acc_val_*.npy') 574 | for n_comp in n_comps: # 575 | plt.figure() 576 | for p in pkgs: 577 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 578 | # n_hidden = int(re.search('comp=(?P.{1,2,3})_', p).group('comp')) 579 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 580 | if n_comp != n_hidden: 581 | continue 582 | 583 | dbg_acc_val_ = np.load(p) 584 | 585 | cur_label = 'n_comp=%i' % n_hidden 586 | cur_label += '/' 587 | cur_label += 'lambda=%.2f' % lambda_param 588 | cur_label += '/' 589 | if not '_AE' in p: 590 | cur_label += 'LR only!' 591 | elif 'subRS' in p: 592 | cur_label += 'RSnormal' 593 | elif 'pca20RS' in p: 594 | cur_label += 'RSpca20' 595 | cur_label += '/' 596 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 597 | plt.plot( 598 | dbg_epochs_, 599 | dbg_acc_val_, 600 | label=cur_label) 601 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss') 602 | plt.legend(loc='lower right', fontsize=9) 603 | plt.yticks(np.linspace(0., 1., 11)) 604 | plt.ylabel('validation set accuracy') 605 | plt.ylim(0., 1.05) 606 | plt.xlabel('epochs') 607 | plt.grid(True) 608 | plt.show() 609 | plt.savefig(op.join(WRITE_DIR, 'accuracy_val_%icomps.png' % n_comp)) 610 | 611 | pkgs = glob.glob(RES_NAME + '/*dbg_acc_other_ds_*.npy') 612 | for n_comp in n_comps: # 613 | plt.figure() 614 | for p in pkgs: 615 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 616 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 617 | if n_comp != n_hidden: 618 | continue 619 | 620 | dbg_acc_other_ds_ = np.load(p) 621 | 622 | cur_label = 'n_comp=%i' % n_hidden 623 | cur_label += '/' 624 | cur_label += 'lambda=%.2f' % lambda_param 625 | cur_label += '/' 626 | if not '_AE' in p: 627 | cur_label += 'LR only!' 628 | elif 'subRS' in p: 629 | cur_label += 'RSnormal' 630 | elif 'pca20RS' in p: 631 | cur_label += 'RSpca20' 632 | cur_label += '/' 633 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 634 | plt.plot( 635 | dbg_epochs_, 636 | dbg_acc_other_ds_, 637 | label=cur_label) 638 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss') 639 | plt.legend(loc='lower right', fontsize=9) 640 | plt.yticks(np.linspace(0., 1., 11)) 641 | plt.ylabel('ARCHI dataset accuracy') 642 | plt.ylim(0., 1.05) 643 | plt.xlabel('epochs') 644 | plt.grid(True) 645 | plt.show() 646 | plt.savefig(op.join(WRITE_DIR, 'accuracy_archi_%icomps.png' % n_comp)) 647 | 648 | pkgs = glob.glob(RES_NAME + '/*dbg_ae_cost_*.npy') 649 | for n_comp in n_comps: # AE 650 | plt.figure() 651 | for p in pkgs: 652 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 653 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 654 | if n_comp != n_hidden: 655 | continue 656 | 657 | dbg_ae_cost_ = np.load(p) 658 | 659 | cur_label = 'n_comp=%i' % n_hidden 660 | cur_label += '/' 661 | cur_label += 'lambda=%.2f' % lambda_param 662 | cur_label += '/' 663 | if not '_AE' in p: 664 | cur_label += 'LR only!' 665 | elif 'subRS' in p: 666 | cur_label += 'RSnormal' 667 | elif 'pca20RS' in p: 668 | cur_label += 'RSpca20' 669 | cur_label += '/' 670 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 671 | plt.plot( 672 | dbg_epochs_, 673 | dbg_ae_cost_, 674 | label=cur_label) 675 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss') 676 | plt.legend(loc='lower right', fontsize=9) 677 | # plt.yticks(np.linspace(0., 1., 11)) 678 | plt.ylabel('AE loss') 679 | plt.xlabel('epochs') 680 | plt.grid(True) 681 | plt.show() 682 | plt.savefig(op.join(WRITE_DIR, 'loss_ae_%icomps.png' % n_comp)) 683 | 684 | pkgs = glob.glob(RES_NAME + '/*dbg_lr_cost_*.npy') # LR cost 685 | for n_comp in n_comps: # AE 686 | plt.figure() 687 | for p in pkgs: 688 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 689 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 690 | if n_comp != n_hidden: 691 | continue 692 | 693 | dbg_lr_cost_ = np.load(p) 694 | 695 | cur_label = 'n_comp=%i' % n_hidden 696 | cur_label += '/' 697 | cur_label += 'lambda=%.2f' % lambda_param 698 | cur_label += '/' 699 | if not '_AE' in p: 700 | cur_label += 'LR only!' 701 | elif 'subRS' in p: 702 | cur_label += 'RSnormal' 703 | elif 'pca20RS' in p: 704 | cur_label += 'RSpca20' 705 | cur_label += '/' 706 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 707 | plt.plot( 708 | dbg_epochs_, 709 | dbg_lr_cost_, 710 | label=cur_label) 711 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss') 712 | plt.legend(loc='lower right', fontsize=9) 713 | # plt.yticks(np.linspace(0., 1., 11)) 714 | plt.ylabel('LR loss') 715 | plt.xlabel('epochs') 716 | plt.grid(True) 717 | plt.show() 718 | plt.savefig(op.join(WRITE_DIR, 'loss_lr_%icomps.png' % n_comp)) 719 | 720 | pkgs = glob.glob(RES_NAME + '/*dbg_combined_cost_*.npy') # combined loss 721 | for n_comp in n_comps: # AE 722 | plt.figure() 723 | for p in pkgs: 724 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 725 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 726 | if n_comp != n_hidden: 727 | continue 728 | 729 | dbg_combined_cost_ = np.load(p) 730 | 731 | cur_label = 'n_comp=%i' % n_hidden 732 | cur_label += '/' 733 | cur_label += 'lambda=%.2f' % lambda_param 734 | cur_label += '/' 735 | if not '_AE' in p: 736 | cur_label += 'LR only!' 737 | elif 'subRS' in p: 738 | cur_label += 'RSnormal' 739 | elif 'pca20RS' in p: 740 | cur_label += 'RSpca20' 741 | cur_label += '/' 742 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 743 | plt.plot( 744 | dbg_epochs_, 745 | dbg_combined_cost_, 746 | label=cur_label) 747 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss') 748 | plt.legend(loc='lower right', fontsize=9) 749 | # plt.yticks(np.linspace(0., 1., 11)) 750 | plt.ylabel('combined loss') 751 | plt.xlabel('epochs') 752 | plt.grid(True) 753 | plt.show() 754 | plt.savefig(op.join(WRITE_DIR, 'loss_combined_%icomps.png' % n_comp)) 755 | 756 | # precision / recall / f1 757 | target_lambda = 0.5 758 | 759 | pkgs = glob.glob(RES_NAME + '/*lambda=%.2f*dbg_prfs_.npy' % target_lambda) 760 | for n_comp in n_comps: 761 | plt.figure() 762 | for p in pkgs: 763 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 764 | n_hidden = int(re.search('comp=(?P.{1,3})_', p).group('comp')) 765 | if n_comp != n_hidden: 766 | continue 767 | 768 | dbg_prfs_ = np.load(p) 769 | 770 | cur_label = 'n_comp=%i' % n_hidden 771 | cur_label += '/' 772 | cur_label += 'lambda=%.2f' % lambda_param 773 | cur_label += '/' 774 | if not '_AE' in p: 775 | cur_label += 'LR only!' 776 | elif 'subRS' in p: 777 | cur_label += 'RSnormal' 778 | elif 'pca20RS' in p: 779 | cur_label += 'RSpca20' 780 | cur_label += '/' 781 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 782 | for i in np.arange(18): 783 | plt.plot( 784 | dbg_epochs_, 785 | np.array(dbg_prfs_)[:, 0, i], 786 | label='task %i' % (i + 1)) 787 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss lambda=%.2f' % 788 | target_lambda) 789 | plt.legend(loc='lower right', fontsize=9) 790 | # plt.yticks(np.linspace(0., 1., 11)) 791 | plt.ylabel('in-dataset precisions') 792 | plt.ylim(0., 1.05) 793 | plt.xlabel('epochs') 794 | plt.grid(True) 795 | plt.show() 796 | plt.savefig(op.join(WRITE_DIR, 'prec_inds_lambda=%0.2f_%icomps.png' % 797 | (target_lambda, n_comp))) 798 | 799 | # in-dataset recall at lambda=0.5 800 | pkgs = glob.glob(RES_NAME + '/*lambda=%.2f*dbg_prfs_.npy' % target_lambda) 801 | for n_comp in n_comps: 802 | plt.figure() 803 | for p in pkgs: 804 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 805 | n_hidden = int(re.search('comp=(?P.{1,3})_', p).group('comp')) 806 | if n_comp != n_hidden: 807 | continue 808 | 809 | dbg_prfs_ = np.load(p) 810 | 811 | dbg_prfs_ = np.load(p) 812 | cur_label = 'n_comp=%i' % n_hidden 813 | cur_label += '/' 814 | cur_label += 'lambda=%.2f' % lambda_param 815 | cur_label += '/' 816 | if not '_AE' in p: 817 | cur_label += 'LR only!' 818 | elif 'subRS' in p: 819 | cur_label += 'RSnormal' 820 | elif 'pca20RS' in p: 821 | cur_label += 'RSpca20' 822 | cur_label += '/' 823 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 824 | for i in np.arange(18): 825 | plt.plot( 826 | dbg_epochs_, 827 | np.array(dbg_prfs_)[:, 1, i], 828 | label='task %i' % (i + 1)) 829 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss lambda=%.2f' % 830 | target_lambda) 831 | plt.legend(loc='lower right', fontsize=9) 832 | # plt.yticks(np.linspace(0., 1., 11)) 833 | plt.ylabel('in-dataset recall') 834 | plt.ylim(0., 1.05) 835 | plt.xlabel('epochs') 836 | plt.grid(True) 837 | plt.show() 838 | plt.savefig(op.join(WRITE_DIR, 'rec_inds_lambda=%0.2f_%icomps.png' % 839 | (target_lambda, n_comp))) 840 | 841 | # in-dataset f1 at lambda=0.5 842 | pkgs = glob.glob(RES_NAME + '/*lambda=%.2f*dbg_prfs_.npy' % target_lambda) 843 | for n_comp in n_comps: 844 | plt.figure() 845 | for p in pkgs: 846 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 847 | n_hidden = int(re.search('comp=(?P.{1,3})_', p).group('comp')) 848 | if n_comp != n_hidden: 849 | continue 850 | 851 | dbg_prfs_ = np.load(p) 852 | 853 | cur_label = 'n_comp=%i' % n_hidden 854 | cur_label += '/' 855 | cur_label += 'lambda=%.2f' % lambda_param 856 | cur_label += '/' 857 | if not '_AE' in p: 858 | cur_label += 'LR only!' 859 | elif 'subRS' in p: 860 | cur_label += 'RSnormal' 861 | elif 'pca20RS' in p: 862 | cur_label += 'RSpca20' 863 | cur_label += '/' 864 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 865 | for i in np.arange(18): 866 | plt.plot( 867 | dbg_epochs_, 868 | np.array(dbg_prfs_)[:, 2, i], 869 | label='task %i' % (i + 1)) 870 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss lambda=%.2f' % 871 | target_lambda) 872 | plt.legend(loc='lower right', fontsize=9) 873 | # plt.yticks(np.linspace(0., 1., 11)) 874 | plt.ylabel('in-dataset f1 score') 875 | plt.ylim(0., 1.05) 876 | plt.xlabel('epochs') 877 | plt.grid(True) 878 | plt.show() 879 | plt.savefig(op.join(WRITE_DIR, 'f1_inds_lambda=%0.2f_%icomps.png' % 880 | (target_lambda, n_comp))) 881 | 882 | # out-of-dataset precision at lambda=0.5 883 | pkgs = glob.glob(RES_NAME + '/*lambda=%.2f*dbg_prfs_other_ds_.npy' % target_lambda) 884 | for n_comp in n_comps: 885 | plt.figure() 886 | for p in pkgs: 887 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 888 | n_hidden = int(re.search('comp=(?P.{1,3})_', p).group('comp')) 889 | if n_comp != n_hidden: 890 | continue 891 | 892 | dbg_prfs_other_ds_ = np.load(p) 893 | 894 | cur_label = 'n_comp=%i' % n_hidden 895 | cur_label += '/' 896 | cur_label += 'lambda=%.2f' % lambda_param 897 | cur_label += '/' 898 | if not '_AE' in p: 899 | cur_label += 'LR only!' 900 | elif 'subRS' in p: 901 | cur_label += 'RSnormal' 902 | elif 'pca20RS' in p: 903 | cur_label += 'RSpca20' 904 | cur_label += '/' 905 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 906 | for i in np.arange(18): 907 | plt.plot( 908 | dbg_epochs_, 909 | np.array(dbg_prfs_other_ds_)[:, 0, i], 910 | label='task %i' % (i + 1)) 911 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss lambda=%.2f' % 912 | target_lambda) 913 | plt.legend(loc='lower right', fontsize=9) 914 | # plt.yticks(np.linspace(0., 1., 11)) 915 | plt.ylabel('out-of-dataset precisions') 916 | plt.ylim(0., 1.05) 917 | plt.xlabel('epochs') 918 | plt.grid(True) 919 | plt.show() 920 | plt.savefig(op.join(WRITE_DIR, 'prec_oods_lambda=%0.2f_%icomps.png' % 921 | (target_lambda, n_comp))) 922 | 923 | # out-of-dataset recall at lambda=0.5 924 | pkgs = glob.glob(RES_NAME + '/*lambda=%.2f*dbg_prfs_other_ds_.npy' % target_lambda) 925 | for n_comp in n_comps: 926 | plt.figure() 927 | for p in pkgs: 928 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 929 | n_hidden = int(re.search('comp=(?P.{1,3})_', p).group('comp')) 930 | if n_comp != n_hidden: 931 | continue 932 | 933 | dbg_prfs_other_ds_ = np.load(p) 934 | 935 | cur_label = 'n_comp=%i' % n_hidden 936 | cur_label += '/' 937 | cur_label += 'lambda=%.2f' % lambda_param 938 | cur_label += '/' 939 | if not '_AE' in p: 940 | cur_label += 'LR only!' 941 | elif 'subRS' in p: 942 | cur_label += 'RSnormal' 943 | elif 'pca20RS' in p: 944 | cur_label += 'RSpca20' 945 | cur_label += '/' 946 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 947 | for i in np.arange(18): 948 | plt.plot( 949 | dbg_epochs_, 950 | np.array(dbg_prfs_other_ds_)[:, 1, i], 951 | label='task %i' % (i + 1)) 952 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss lambda=%.2f' % 953 | target_lambda) 954 | plt.legend(loc='lower right', fontsize=9) 955 | # plt.yticks(np.linspace(0., 1., 11)) 956 | plt.ylabel('out-of-dataset recall') 957 | plt.ylim(0., 1.05) 958 | plt.xlabel('epochs') 959 | plt.grid(True) 960 | plt.show() 961 | plt.savefig(op.join(WRITE_DIR, 'rec_oods_lambda=%0.2f_%icomps.png' % 962 | (target_lambda, n_comp))) 963 | 964 | # out-of-dataset f1 at lambda=0.5 965 | pkgs = glob.glob(RES_NAME + '/*lambda=%.2f*dbg_prfs_other_ds_.npy' % target_lambda) 966 | for n_comp in n_comps: 967 | plt.figure() 968 | for p in pkgs: 969 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 970 | n_hidden = int(re.search('comp=(?P.{1,3})_', p).group('comp')) 971 | if n_comp != n_hidden: 972 | continue 973 | 974 | dbg_prfs_other_ds_ = np.load(p) 975 | 976 | cur_label = 'n_comp=%i' % n_hidden 977 | cur_label += '/' 978 | cur_label += 'lambda=%.2f' % lambda_param 979 | cur_label += '/' 980 | if not '_AE' in p: 981 | cur_label += 'LR only!' 982 | elif 'subRS' in p: 983 | cur_label += 'RSnormal' 984 | elif 'pca20RS' in p: 985 | cur_label += 'RSpca20' 986 | cur_label += '/' 987 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 988 | for i in np.arange(18): 989 | plt.plot( 990 | dbg_epochs_, 991 | np.array(dbg_prfs_other_ds_)[:, 2, i], 992 | label='task %i' % (i + 1)) 993 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss lambda=%.2f' % 994 | target_lambda) 995 | plt.legend(loc='lower right', fontsize=9) 996 | # plt.yticks(np.linspace(0., 1., 11)) 997 | plt.ylabel('out-of-dataset f1 score') 998 | plt.ylim(0., 1.05) 999 | plt.xlabel('epochs') 1000 | plt.grid(True) 1001 | plt.show() 1002 | plt.savefig(op.join(WRITE_DIR, 'f1_oods_lambda=%0.2f_%icomps.png' % 1003 | (target_lambda, n_comp))) 1004 | 1005 | # print network components (1st layer) 1006 | from nilearn.image import smooth_img 1007 | n_comp = 20 1008 | lmbd = 0.25 1009 | pkgs = glob.glob(RES_NAME + '/*W0comps.npy') 1010 | for p in pkgs: 1011 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 1012 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 1013 | if n_comp != n_hidden or lambda_param != lmbd: 1014 | continue 1015 | 1016 | new_fname = 'comps_n=%i_lambda=%.2f_th0.0' % (n_hidden, lambda_param) 1017 | comps = np.load(p) 1018 | dump_comps(nifti_masker, new_fname, comps, threshold=0.0) 1019 | 1020 | # print class weights 1021 | pkgs = glob.glob(RES_NAME + '/*W0comps.npy') 1022 | n_comp = 20 1023 | lmbd = 0.5 1024 | for p in pkgs: 1025 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 1026 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 1027 | if n_comp != n_hidden or lambda_param != lmbd: 1028 | continue 1029 | print p 1030 | 1031 | q = p.replace('W0', 'V1') 1032 | comps = np.dot(np.load(q), np.load(p)) 1033 | 1034 | new_fname = 'comps_n=%i_lambda=%.2f' % (n_hidden, lambda_param) 1035 | dump_comps(nifti_masker, new_fname, comps, threshold=0.0, fwhm=None, 1036 | perc=75) 1037 | 1038 | 1039 | # print LR decision matrix (2nd layer) 1040 | n_comp = 20 1041 | lmbd = 0.5 1042 | pkgs = glob.glob(RES_NAME + '/*V1comps.npy') 1043 | for p in pkgs: 1044 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 1045 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 1046 | if n_comp != n_hidden or lambda_param != lmbd: 1047 | continue 1048 | print p 1049 | 1050 | cur_mat = np.load(p) 1051 | 1052 | if n_comp == 20: 1053 | fs = (8, 6) 1054 | elif n_comp == 100: 1055 | fs = (12, 8) 1056 | 1057 | 1058 | plt.figure(figsize=fs) 1059 | masked_data = np.ma.masked_where(cur_mat != 0., cur_mat) 1060 | plt.imshow(masked_data, interpolation='nearest', cmap=plt.cm.gray_r) 1061 | masked_data = np.ma.masked_where(cur_mat == 0., cur_mat) 1062 | plt.imshow(masked_data, interpolation='nearest', cmap=plt.cm.RdBu_r) 1063 | plt.show() 1064 | 1065 | # plt.xticks(range(n_comp)[::5], (np.arange(n_comp) + 1)[::5]) 1066 | # plt.xlabel('hidden components') 1067 | # plt.yticks(range(18), np.arange(18) + 1) 1068 | # plt.ylabel('tasks') 1069 | # plt.title('Linear combinations of component per task') 1070 | # plt.colorbar() 1071 | 1072 | new_fname = 'comps_n=%i_lambda=%.2f_V1_net2task.png' % (n_hidden, lambda_param) 1073 | plt.savefig(op.join(WRITE_DIR, new_fname)) 1074 | 1075 | # out-of-dataset f1 score summary plots 1076 | for n_comp in [20, 50, 100]: 1077 | f1_mean_per_lambda = list() 1078 | f1_std_per_lambda = list() 1079 | lambs = [0.25, 0.5, 0.75, 1.0] 1080 | for target_lambda in lambs: 1081 | pkgs = glob.glob(RES_NAME + '/*n_comp=%i*lambda=%.2f*dbg_prfs_other_ds_.npy' % 1082 | (n_comp, target_lambda)) 1083 | print pkgs 1084 | dbg_prfs_other_ds_ = np.load(pkgs[0]) 1085 | cur_mean = np.mean(dbg_prfs_other_ds_[-1, 2, :]) 1086 | f1_mean_per_lambda.append(cur_mean) 1087 | cur_std = np.std(dbg_prfs_other_ds_[-1, 2, :]) 1088 | f1_std_per_lambda.append(cur_std) 1089 | print('F1 means: %.2f +/- %.2f (SD)' % (cur_mean, cur_std)) 1090 | 1091 | f1_mean_per_lambda = np.array(f1_mean_per_lambda) 1092 | f1_std_per_lambda = np.array(f1_std_per_lambda) 1093 | 1094 | plt.figure() 1095 | ind = np.arange(4) 1096 | width = 1. 1097 | colors = [#(7., 116., 242.), #(7., 176., 242.) 1098 | #(7., 136., 217.), (7., 40., 164.), (1., 4., 64.)] 1099 | (7., 176., 242.), (7., 136., 217.), (7., 40., 164.), (1., 4., 64.)] 1100 | my_colors = [(x/256, y/256, z/256) for x, y, z in colors] 1101 | plt.bar(ind, f1_mean_per_lambda, yerr=f1_std_per_lambda, 1102 | width=width, color=my_colors) 1103 | plt.ylabel('mean F1 score (+/- SD)') 1104 | plt.title('out-of-dataset performance\n' 1105 | '%i components' % n_comp) 1106 | tick_strs = [u'low-rank $\lambda=%.2f$' % val for val in lambs] 1107 | plt.xticks(ind + width / 2., tick_strs, rotation=320) 1108 | plt.ylim(.5, 1.0) 1109 | plt.grid(True) 1110 | plt.yticks(np.linspace(0.5, 1., 11), np.linspace(0.5, 1., 11)) 1111 | plt.tight_layout() 1112 | out_path2 = op.join(WRITE_DIR, 'f1_bars_comp=%i.png' % n_comp) 1113 | plt.savefig(out_path2) 1114 | -------------------------------------------------------------------------------- /nips3mm_h38.py: -------------------------------------------------------------------------------- 1 | """ 2 | HCP: Semi-supervised network decomposition by low-rank logistic regression 3 | """ 4 | 5 | print __doc__ 6 | 7 | import os 8 | import os.path as op 9 | import numpy as np 10 | import glob 11 | from scipy.linalg import norm 12 | import nibabel as nib 13 | from matplotlib import pylab as plt 14 | from sklearn.grid_search import RandomizedSearchCV 15 | from sklearn.base import BaseEstimator 16 | from sklearn.preprocessing import StandardScaler 17 | from nilearn.input_data import NiftiMasker 18 | from sklearn.metrics import precision_recall_fscore_support 19 | from sklearn.metrics import confusion_matrix 20 | import theano 21 | import theano.tensor as T 22 | print('Running THEANO on %s' % theano.config.device) 23 | from nilearn.image import concat_imgs 24 | import joblib 25 | import time 26 | 27 | RES_NAME = 'nips3mm_h38' 28 | WRITE_DIR = op.join(os.getcwd(), RES_NAME) 29 | if not op.exists(WRITE_DIR): 30 | os.mkdir(WRITE_DIR) 31 | 32 | ############################################################################## 33 | # load+preprocess data 34 | ############################################################################## 35 | 36 | mask_img = 'grey10_icbm_3mm_bin.nii.gz' 37 | nifti_masker = NiftiMasker(mask_img=mask_img, smoothing_fwhm=False, 38 | standardize=False) 39 | nifti_masker.fit() 40 | mask_nvox = nifti_masker.mask_img_.get_data().sum() 41 | 42 | print('Loading data...') 43 | 44 | # ARCHI task 45 | X_task, labels = joblib.load('preload_HT38_3mm') 46 | 47 | labels = np.int32(labels) 48 | 49 | # contrasts are IN ORDER -> shuffle! 50 | new_inds = np.arange(0, X_task.shape[0]) 51 | np.random.shuffle(new_inds) 52 | X_task = X_task[new_inds] 53 | labels = labels[new_inds] 54 | # subs = subs[new_inds] 55 | 56 | # rest 57 | # X_rest = nifti_masker.transform('preload_HR20persub_10mm_ero2.nii') 58 | # X_rest = nifti_masker.transform('dump_rs_spca_s12_tmp') 59 | rs_spca_data = joblib.load('dump_rs_spca_s12_tmp') 60 | rs_spca_niis = nib.Nifti1Image(rs_spca_data, 61 | nifti_masker.mask_img_.get_affine()) 62 | X_rest = nifti_masker.transform(rs_spca_niis) 63 | del rs_spca_niis 64 | del rs_spca_data 65 | 66 | X_task = StandardScaler().fit_transform(X_task) 67 | 68 | # ARCHI task 69 | AT_niis, AT_labels, AT_subs = joblib.load('preload_AT_3mm') 70 | AT_X = nifti_masker.transform(AT_niis) 71 | AT_X = StandardScaler().fit_transform(AT_X) 72 | print('done :)') 73 | 74 | ############################################################################## 75 | # define computation graph 76 | ############################################################################## 77 | 78 | class SSEncoder(BaseEstimator): 79 | def __init__(self, n_hidden, gain1, learning_rate, max_epochs=100, 80 | l1=0.1, l2=0.1, lambda_param=.5): 81 | """ 82 | Parameters 83 | ---------- 84 | lambda : float 85 | Mediates between AE and LR. lambda==1 equates with LR only. 86 | """ 87 | self.n_hidden = n_hidden 88 | self.gain1 = gain1 89 | self.max_epochs = max_epochs 90 | self.learning_rate = np.float32(learning_rate) 91 | self.penalty_l1 = np.float32(l1) 92 | self.penalty_l2 = np.float32(l2) 93 | self.lambda_param = np.float32(lambda_param) 94 | 95 | # def rectify(X): 96 | # return T.maximum(0., X) 97 | 98 | from theano.tensor.shared_randomstreams import RandomStreams 99 | 100 | def RMSprop(self, cost, params, lr=0.001, rho=0.9, epsilon=1e-6): 101 | grads = T.grad(cost=cost, wrt=params) 102 | updates = [] 103 | for p, g in zip(params, grads): 104 | acc = theano.shared(p.get_value() * 0.) 105 | acc_new = rho * acc + (1 - rho) * g ** 2 106 | gradient_scaling = T.sqrt(acc_new + epsilon) 107 | g = g / gradient_scaling 108 | updates.append((acc, acc_new)) 109 | updates.append((p, p - lr * g)) 110 | return updates 111 | 112 | def get_param_pool(self): 113 | cur_params = ( 114 | self.V1s, self.bV0, self.bV1, 115 | self.W0s, self.W1s, self.bW0s, self.bW1s 116 | ) 117 | return cur_params 118 | 119 | def test_performance_in_other_dataset(self): 120 | from sklearn.linear_model import LogisticRegression 121 | from sklearn.cross_validation import StratifiedShuffleSplit 122 | 123 | compr_matrix = self.W0s.get_value().T # currently best compression 124 | AT_X_compr = np.dot(compr_matrix, AT_X.T).T 125 | clf = LogisticRegression(penalty='l1') 126 | folder = StratifiedShuffleSplit(y=AT_labels, n_iter=5, test_size=0.2) 127 | 128 | acc_list = [] 129 | prfs_list = [] 130 | for (train_inds, test_inds) in folder: 131 | clf.fit(AT_X_compr[train_inds, :], AT_labels[train_inds]) 132 | pred_y = clf.predict(AT_X_compr[test_inds, :]) 133 | 134 | acc = (pred_y == AT_labels[test_inds]).mean() 135 | prfs_list.append(precision_recall_fscore_support( 136 | AT_labels[test_inds], pred_y)) 137 | 138 | acc_list.append(acc) 139 | 140 | compr_mean_acc = np.mean(acc_list) 141 | prfs = np.asarray(prfs_list).mean(axis=0) 142 | return compr_mean_acc, prfs 143 | 144 | def fit(self, X_rest, X_task, y): 145 | DEBUG_FLAG = True 146 | 147 | # self.max_epochs = 333 148 | self.batch_size = 100 149 | n_input = X_rest.shape[1] # sklearn-like structure 150 | n_output = n_input 151 | rng = np.random.RandomState(42) 152 | self.input_taskdata = T.matrix(dtype='float32', name='input_taskdata') 153 | self.input_restdata = T.matrix(dtype='float32', name='input_restdata') 154 | self.params_from_last_iters = [] 155 | 156 | index = T.iscalar(name='index') 157 | 158 | # prepare data for theano computation 159 | if not DEBUG_FLAG: 160 | X_train_s = theano.shared( 161 | value=np.float32(X_task), name='X_train_s') 162 | y_train_s = theano.shared( 163 | value=np.int32(y), name='y_train_s') 164 | lr_train_samples = len(X_task) 165 | else: 166 | from sklearn.cross_validation import StratifiedShuffleSplit 167 | folder = StratifiedShuffleSplit(y, n_iter=1, test_size=0.20) 168 | new_trains, inds_val = iter(folder).next() 169 | X_train, X_val = X_task[new_trains], X_task[inds_val] 170 | y_train, y_val = y[new_trains], y[inds_val] 171 | 172 | X_train_s = theano.shared(value=np.float32(X_train), 173 | name='X_train_s', borrow=False) 174 | y_train_s = theano.shared(value=np.int32(y_train), 175 | name='y_train_s', borrow=False) 176 | # X_val_s = theano.shared(value=np.float32(X_val), 177 | # name='X_train_s', borrow=False) 178 | # y_val_s = theano.shared(value=np.int32(y_val), 179 | # name='y_cal_s', borrow=False) 180 | lr_train_samples = len(X_train) 181 | self.dbg_epochs_ = list() 182 | self.dbg_acc_train_ = list() 183 | self.dbg_acc_val_ = list() 184 | self.dbg_ae_cost_ = list() 185 | self.dbg_lr_cost_ = list() 186 | self.dbg_ae_nonimprovesteps = list() 187 | self.dbg_acc_other_ds_ = list() 188 | self.dbg_combined_cost_ = list() 189 | self.dbg_prfs_ = list() 190 | self.dbg_prfs_other_ds_ = list() 191 | X_rest_s = theano.shared(value=np.float32(X_rest), name='X_rest_s') 192 | ae_train_samples = len(X_rest) 193 | 194 | # V -> supervised / logistic regression 195 | # W -> unsupervised / auto-encoder 196 | 197 | # computational graph: auto-encoder 198 | W0_vals = rng.randn(n_input, self.n_hidden).astype(np.float32) * self.gain1 199 | 200 | self.W0s = theano.shared(W0_vals) 201 | self.W1s = self.W0s.T # tied 202 | bW0_vals = np.zeros(self.n_hidden).astype(np.float32) 203 | self.bW0s = theano.shared(value=bW0_vals, name='bW0') 204 | bW1_vals = np.zeros(n_output).astype(np.float32) 205 | self.bW1s = theano.shared(value=bW1_vals, name='bW1') 206 | 207 | givens_ae = { 208 | self.input_restdata: X_rest_s[ 209 | index * self.batch_size:(index + 1) * self.batch_size] 210 | } 211 | 212 | encoding = (self.input_restdata.dot(self.W0s) + self.bW0s).dot(self.W1s) + self.bW1s 213 | 214 | self.ae_loss = T.sum((self.input_restdata - encoding) ** 2, axis=1) 215 | 216 | self.ae_cost = ( 217 | T.mean(self.ae_loss) / n_input 218 | ) 219 | 220 | # params1 = [self.W0s, self.bW0s, self.bW1s] 221 | # gparams1 = [T.grad(cost=self.ae_cost, wrt=param1) for param1 in params1] 222 | # 223 | # lr = self.learning_rate 224 | # updates = self.RMSprop(cost=self.ae_cost, params=params1, 225 | # lr=self.learning_rate) 226 | 227 | # f_train_ae = theano.function( 228 | # [index], 229 | # [self.ae_cost], 230 | # givens=givens_ae, 231 | # updates=updates) 232 | 233 | # computation graph: logistic regression 234 | clf_n_output = 38 # number of labels 235 | my_y = T.ivector(name='y') 236 | 237 | bV0_vals = np.zeros(self.n_hidden).astype(np.float32) 238 | self.bV0 = theano.shared(value=bV0_vals, name='bV0') 239 | bV1_vals = np.zeros(clf_n_output).astype(np.float32) 240 | self.bV1 = theano.shared(value=bV1_vals, name='bV1') 241 | 242 | # V0_vals = rng.randn(n_input, self.n_hidden).astype(np.float32) * self.gain1 243 | V1_vals = rng.randn(self.n_hidden, clf_n_output).astype(np.float32) * self.gain1 244 | # self.V0s = theano.shared(V0_vals) 245 | self.V1s = theano.shared(V1_vals) 246 | 247 | self.p_y_given_x = T.nnet.softmax( 248 | # T.dot(T.dot(self.input_taskdata, self.V0s) + self.bV0, self.V1s) + self.bV1 249 | T.dot(T.dot(self.input_taskdata, self.W0s) + self.bV0, self.V1s) + self.bV1 250 | ) 251 | self.lr_cost = -T.mean(T.log(self.p_y_given_x)[T.arange(my_y.shape[0]), my_y]) 252 | self.lr_cost = ( 253 | self.lr_cost + 254 | T.mean(abs(self.W0s)) * self.penalty_l1 + 255 | # T.mean(abs(self.V0s)) * self.penalty_l1 + 256 | T.mean(abs(self.bV0)) * self.penalty_l1 + 257 | T.mean(abs(self.V1s)) * self.penalty_l1 + 258 | T.mean(abs(self.bV1)) * self.penalty_l1 + 259 | 260 | T.mean((self.W0s ** np.float32(2))) * self.penalty_l2 + 261 | # T.mean((self.V0s ** 2)) * self.penalty_l2 + 262 | T.mean((self.bV0 ** np.float32(2))) * self.penalty_l2 + 263 | T.mean((self.V1s ** np.float32(2))) * self.penalty_l2 + 264 | T.mean((self.bV1 ** np.float32(2))) * self.penalty_l2 265 | ) 266 | self.y_pred = T.argmax(self.p_y_given_x, axis=1) 267 | 268 | givens_lr = { 269 | self.input_taskdata: X_train_s[index * self.batch_size:(index + 1) * self.batch_size], 270 | my_y: y_train_s[index * self.batch_size:(index + 1) * self.batch_size] 271 | } 272 | 273 | # params2 = [self.V0s, self.bV0, self.V1s, self.bV1] 274 | # params2 = [self.W0s, self.bV0, self.V1s, self.bV1] 275 | # updates2 = self.RMSprop(cost=self.lr_cost, params=params2, 276 | # lr=self.learning_rate) 277 | 278 | # f_train_lr = theano.function( 279 | # [index], 280 | # [self.lr_cost], 281 | # givens=givens_lr, 282 | # updates=updates2) 283 | 284 | # combined loss for AE and LR 285 | combined_params = [self.W0s, self.bW0s, self.bW1s, 286 | # self.V0s, self.V1s, self.bV0, self.bV1] 287 | self.V1s, self.bV0, self.bV1] 288 | self.combined_cost = ( 289 | (np.float32(1) - self.lambda_param) * self.ae_cost + 290 | self.lambda_param * self.lr_cost 291 | ) 292 | combined_updates = self.RMSprop( 293 | cost=self.combined_cost, 294 | params=combined_params, 295 | lr=self.learning_rate) 296 | givens_combined = { 297 | self.input_restdata: X_rest_s[index * self.batch_size:(index + 1) * self.batch_size], 298 | self.input_taskdata: X_train_s[index * self.batch_size:(index + 1) * self.batch_size], 299 | my_y: y_train_s[index * self.batch_size:(index + 1) * self.batch_size] 300 | } 301 | f_train_combined = theano.function( 302 | [index], 303 | [self.combined_cost, self.ae_cost, self.lr_cost], 304 | givens=givens_combined, 305 | updates=combined_updates, allow_input_downcast=True) 306 | 307 | # optimization loop 308 | start_time = time.time() 309 | ae_last_cost = np.inf 310 | lr_last_cost = np.inf 311 | no_improve_steps = 0 312 | acc_train, acc_val = 0., 0. 313 | for i_epoch in range(self.max_epochs): 314 | if i_epoch == 1: 315 | epoch_dur = time.time() - start_time 316 | total_mins = (epoch_dur * self.max_epochs) / 60 317 | hs, mins = divmod(total_mins, 60) 318 | print("Max estimated duration: %i hours and %i minutes" % (hs, mins)) 319 | 320 | # AE 321 | # if i_epoch % 2 == 0: # every second time 322 | #if False: 323 | # auto-encoder 324 | ae_n_batches = ae_train_samples // self.batch_size 325 | lr_n_batches = lr_train_samples // self.batch_size 326 | # for i in range(lr_n_batches): 327 | # for i in range(max(ae_n_batches, lr_n_batches)): 328 | # if i < ae_n_batches: 329 | # ae_cur_cost = float(f_train_ae(i)[0]) 330 | # ae_cur_cost = 0 331 | # if i < lr_n_batches: 332 | # lr_cur_cost = float(f_train_lr(i)[0]) 333 | # for i in range(lr_n_batches): 334 | for i in range(min(ae_n_batches, lr_n_batches)): 335 | # lr_cur_cost = f_train_lr(i)[0] 336 | # ae_cur_cost = lr_cur_cost 337 | combined_cost, ae_cur_cost, lr_cur_cost = f_train_combined(i) 338 | 339 | # evaluate epoch cost 340 | if ae_last_cost - ae_cur_cost < 0.1: 341 | no_improve_steps += 1 342 | else: 343 | ae_last_cost = ae_cur_cost 344 | no_improve_steps = 0 345 | 346 | # logistic 347 | lr_last_cost = lr_cur_cost 348 | acc_train = self.score(X_train, y_train) 349 | acc_val, prfs_val = self.score(X_val, y_val, return_prfs=True) 350 | 351 | print('E:%i, ae_cost:%.4f, lr_cost:%.4f, train_score:%.2f, vald_score:%.2f, ae_badsteps:%i' % ( 352 | i_epoch + 1, ae_cur_cost, lr_cur_cost, acc_train, acc_val, no_improve_steps)) 353 | 354 | if (i_epoch % 10 == 0): 355 | self.dbg_ae_cost_.append(ae_cur_cost) 356 | self.dbg_lr_cost_.append(lr_cur_cost) 357 | self.dbg_combined_cost_.append(combined_cost) 358 | 359 | self.dbg_epochs_.append(i_epoch + 1) 360 | self.dbg_ae_nonimprovesteps.append(no_improve_steps) 361 | self.dbg_acc_train_.append(acc_train) 362 | self.dbg_acc_val_.append(acc_val) 363 | self.dbg_prfs_.append(prfs_val) 364 | 365 | # test out-of-dataset performance 366 | od_acc, prfs_other = self.test_performance_in_other_dataset() 367 | self.dbg_acc_other_ds_.append(od_acc) 368 | self.dbg_prfs_other_ds_.append(prfs_other) 369 | print('out-of-dataset acc: %.2f' % od_acc) 370 | 371 | # save paramters from last 100 iterations 372 | if i_epoch > (self.max_epochs - 499): 373 | print('Param pool!') 374 | param_pool = self.get_param_pool() 375 | self.params_from_last_iters.append(param_pool) 376 | 377 | total_mins = (time.time() - start_time) / 60 378 | hs, mins = divmod(total_mins, 60) 379 | print("Final duration: %i hours and %i minutes" % (hs, mins)) 380 | 381 | return self 382 | 383 | def predict(self, X): 384 | X_test_s = theano.shared(value=np.float32(X), name='X_test_s', borrow=True) 385 | 386 | givens_te = { 387 | self.input_taskdata: X_test_s 388 | } 389 | 390 | f_test = theano.function( 391 | [], 392 | [self.y_pred], 393 | givens=givens_te) 394 | predictions = f_test() 395 | del X_test_s 396 | del givens_te 397 | return predictions[0] 398 | 399 | def score(self, X, y, return_prfs=False): 400 | pred_y = self.predict(X) 401 | acc = np.mean(pred_y == y) 402 | prfs = precision_recall_fscore_support(pred_y, y) 403 | if return_prfs: 404 | return acc, prfs 405 | else: 406 | return acc 407 | 408 | 409 | ############################################################################## 410 | # plot figures 411 | ############################################################################## 412 | 413 | def dump_comps(masker, compressor, components, threshold=2): 414 | from scipy.stats import zscore 415 | from nilearn.plotting import plot_stat_map 416 | 417 | if isinstance(compressor, basestring): 418 | comp_name = compressor 419 | else: 420 | comp_name = compressor.__str__().split('(')[0] 421 | 422 | for i_c, comp in enumerate(components): 423 | path_mask = op.join(WRITE_DIR, '%s_%i-%i' % (comp_name, 424 | n_comp, i_c + 1)) 425 | nii_raw = masker.inverse_transform(comp) 426 | nii_raw.to_filename(path_mask + '.nii.gz') 427 | 428 | nii_z = masker.inverse_transform(zscore(comp)) 429 | gz_path = path_mask + '_zmap.nii.gz' 430 | nii_z.to_filename(gz_path) 431 | plot_stat_map(gz_path, bg_img='colin.nii', threshold=threshold, 432 | cut_coords=(0, -2, 0), draw_cross=False, 433 | output_file=path_mask + 'zmap.png') 434 | 435 | n_comps = [100] 436 | # n_comps = [40, 30, 20, 10, 5] 437 | for n_comp in n_comps: 438 | # for lambda_param in [0]: 439 | for lambda_param in [1.0]: 440 | l1 = 0.1 441 | l2 = 0.1 442 | my_title = r'Low-rank LR + AE (combined loss, shared decomp): n_comp=%i L1=%.1f L2=%.1f lambda=%.2f res=3mm spca20RS' % ( 443 | n_comp, l1, l2, lambda_param 444 | ) 445 | print(my_title) 446 | estimator = SSEncoder( 447 | n_hidden=n_comp, 448 | gain1=0.004, # empirically determined by CV 449 | learning_rate = np.float32(0.00001), # empirically determined by CV, 450 | max_epochs=300, l1=l1, l2=l2, lambda_param=lambda_param) 451 | 452 | estimator.fit(X_rest, X_task, labels) 453 | 454 | # my_title = r'Low-rank LR: n_comp=%i L1=%.1f L2=%.1f res=10mm pca20RS' % ( 455 | # n_comp, l1, l2 456 | # ) 457 | # FONTS = 12 458 | # plt.figure(facecolor='white', figsize=(8, 6)) 459 | # plt.plot(np.log(estimator.dbg_ae_cost_), label='cost autoencoder') 460 | # plt.plot(estimator.dbg_lr_cost_, label='cost logistic') 461 | # plt.plot(estimator.dbg_acc_train_, label='score training set') 462 | # plt.plot(estimator.dbg_acc_val_, label='score validation set') 463 | # plt.plot(estimator.dbg_acc_other_ds_, label='other-datset acc') 464 | # plt.legend(loc='best', fontsize=12) 465 | # plt.xlabel('epoch', fontsize=FONTS) 466 | # plt.ylabel('misc', fontsize=FONTS) 467 | # plt.yticks(np.arange(12), np.arange(12)) 468 | # plt.grid(True) 469 | # plt.title(my_title) 470 | # plt.show() 471 | 472 | fname = my_title.replace(' ', '_').replace('+', '').replace(':', '').replace('__', '_').replace('%', '') 473 | cur_path = op.join(WRITE_DIR, fname) 474 | joblib.dump(estimator, cur_path) 475 | # plt.savefig(cur_path + '_SUMMARY.png', dpi=200) 476 | 477 | # dump data also as numpy array 478 | np.save(cur_path + 'dbg_epochs_', np.array(estimator.dbg_epochs_)) 479 | np.save(cur_path + 'dbg_acc_train_', np.array(estimator.dbg_acc_train_)) 480 | np.save(cur_path + 'dbg_acc_val_', np.array(estimator.dbg_acc_val_)) 481 | np.save(cur_path + 'dbg_ae_cost_', np.array(estimator.dbg_ae_cost_)) 482 | np.save(cur_path + 'dbg_lr_cost_', np.array(estimator.dbg_lr_cost_)) 483 | np.save(cur_path + 'dbg_ae_nonimprovesteps', np.array(estimator.dbg_ae_nonimprovesteps)) 484 | np.save(cur_path + 'dbg_acc_other_ds_', np.array(estimator.dbg_acc_other_ds_)) 485 | np.save(cur_path + 'dbg_combined_cost_', np.array(estimator.dbg_combined_cost_)) 486 | np.save(cur_path + 'dbg_prfs_', np.array(estimator.dbg_prfs_)) 487 | np.save(cur_path + 'dbg_prfs_other_ds_', np.array(estimator.dbg_prfs_other_ds_)) 488 | 489 | W0_mat = estimator.W0s.get_value().T 490 | np.save(cur_path + 'W0comps', W0_mat) 491 | 492 | V1_mat = estimator.V1s.get_value().T 493 | np.save(cur_path + 'V1comps', V1_mat) 494 | # dump_comps(nifti_masker, fname, comps, threshold=0.5) 495 | 496 | STOP_CALCULATION 497 | 498 | # equally scaled plots 499 | import re 500 | pkgs = glob.glob(RES_NAME + '/*dbg_epochs_*.npy') 501 | dbg_epochs_ = np.load(pkgs[0]) 502 | n_comps = [20] 503 | pkgs = glob.glob(RES_NAME + '/*dbg_acc_train_*.npy') 504 | for n_comp in n_comps: 505 | plt.figure() 506 | for p in pkgs: 507 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 508 | n_hidden = int(re.search('comp=(?P.{1,3})_', p).group('comp')) 509 | if n_comp != n_hidden: 510 | continue 511 | 512 | dbg_acc_train_ = np.load(p) 513 | 514 | cur_label = 'n_comp=%i' % n_hidden 515 | cur_label += '/' 516 | cur_label += 'lambda=%.1f' % lambda_param 517 | cur_label += '/' 518 | if not '_AE' in p: 519 | cur_label += 'LR only!' 520 | elif 'subRS' in p: 521 | cur_label += 'RSnormal' 522 | elif 'pca20RS' in p: 523 | cur_label += 'RSpca20' 524 | cur_label += '/' 525 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 526 | cur_label += '' if '_AE' in p else '/LR only!' 527 | plt.plot( 528 | dbg_epochs_, 529 | dbg_acc_train_, 530 | label=cur_label) 531 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=10mm combined-loss') 532 | plt.legend(loc='lower right', fontsize=9) 533 | plt.yticks(np.linspace(0., 1., 11)) 534 | plt.ylabel('training accuracy') 535 | plt.xlabel('epochs') 536 | plt.ylim(0., 1.) 537 | plt.grid(True) 538 | plt.show() 539 | plt.savefig(op.join(WRITE_DIR, 'accuracy_train_%icomps.png' % n_comp)) 540 | 541 | pkgs = glob.glob(RES_NAME + '/*dbg_acc_val_*.npy') 542 | for n_comp in n_comps: # 543 | plt.figure() 544 | for p in pkgs: 545 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 546 | n_hidden = int(re.search('comp=(?P.{1,3})_', p).group('comp')) 547 | if n_comp != n_hidden: 548 | continue 549 | 550 | dbg_acc_val_ = np.load(p) 551 | 552 | cur_label = 'n_comp=%i' % n_hidden 553 | cur_label += '/' 554 | cur_label += 'lambda=%.1f' % lambda_param 555 | cur_label += '/' 556 | if not '_AE' in p: 557 | cur_label += 'LR only!' 558 | elif 'subRS' in p: 559 | cur_label += 'RSnormal' 560 | elif 'pca20RS' in p: 561 | cur_label += 'RSpca20' 562 | cur_label += '/' 563 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 564 | plt.plot( 565 | dbg_epochs_, 566 | dbg_acc_val_, 567 | label=cur_label) 568 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss') 569 | plt.legend(loc='lower right', fontsize=9) 570 | plt.yticks(np.linspace(0., 1., 11)) 571 | plt.ylabel('validation set accuracy') 572 | plt.ylim(0., 1.) 573 | plt.xlabel('epochs') 574 | plt.grid(True) 575 | plt.show() 576 | plt.savefig(op.join(WRITE_DIR, 'accuracy_val_%icomps.png' % n_comp)) 577 | 578 | pkgs = glob.glob(RES_NAME + '/*dbg_acc_other_ds_*.npy') 579 | for n_comp in n_comps: # 580 | plt.figure() 581 | for p in pkgs: 582 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 583 | n_hidden = int(re.search('comp=(?P.{1,2})_', p).group('comp')) 584 | if n_comp != n_hidden: 585 | continue 586 | 587 | dbg_acc_other_ds_ = np.load(p) 588 | 589 | cur_label = 'n_comp=%i' % n_hidden 590 | cur_label += '/' 591 | cur_label += 'lambda=%.1f' % lambda_param 592 | cur_label += '/' 593 | if not '_AE' in p: 594 | cur_label += 'LR only!' 595 | elif 'subRS' in p: 596 | cur_label += 'RSnormal' 597 | elif 'pca20RS' in p: 598 | cur_label += 'RSpca20' 599 | cur_label += '/' 600 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 601 | plt.plot( 602 | dbg_epochs_, 603 | dbg_acc_other_ds_, 604 | label=cur_label) 605 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss') 606 | plt.legend(loc='lower right', fontsize=9) 607 | plt.yticks(np.linspace(0., 1., 11)) 608 | plt.ylabel('ARCHI dataset accuracy') 609 | plt.ylim(0., 1.) 610 | plt.xlabel('epochs') 611 | plt.grid(True) 612 | plt.show() 613 | plt.savefig(op.join(WRITE_DIR, 'accuracy_archi_%icomps.png' % n_comp)) 614 | 615 | pkgs = glob.glob(RES_NAME + '/*dbg_ae_cost_*.npy') 616 | for n_comp in n_comps: # AE 617 | plt.figure() 618 | for p in pkgs: 619 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 620 | n_hidden = int(re.search('comp=(?P.{1,2})_', p).group('comp')) 621 | if n_comp != n_hidden: 622 | continue 623 | 624 | dbg_ae_cost_ = np.load(p) 625 | 626 | cur_label = 'n_comp=%i' % n_hidden 627 | cur_label += '/' 628 | cur_label += 'lambda=%.1f' % lambda_param 629 | cur_label += '/' 630 | if not '_AE' in p: 631 | cur_label += 'LR only!' 632 | elif 'subRS' in p: 633 | cur_label += 'RSnormal' 634 | elif 'pca20RS' in p: 635 | cur_label += 'RSpca20' 636 | cur_label += '/' 637 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 638 | plt.plot( 639 | dbg_epochs_, 640 | dbg_ae_cost_, 641 | label=cur_label) 642 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss') 643 | plt.legend(loc='lower right', fontsize=9) 644 | # plt.yticks(np.linspace(0., 1., 11)) 645 | plt.ylabel('AE loss') 646 | plt.xlabel('epochs') 647 | plt.grid(True) 648 | plt.show() 649 | plt.savefig(op.join(WRITE_DIR, 'loss_ae_%icomps.png' % n_comp)) 650 | 651 | pkgs = glob.glob(RES_NAME + '/*dbg_lr_cost_*.npy') # LR cost 652 | for n_comp in n_comps: # AE 653 | plt.figure() 654 | for p in pkgs: 655 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 656 | n_hidden = int(re.search('comp=(?P.{1,2})_', p).group('comp')) 657 | if n_comp != n_hidden: 658 | continue 659 | 660 | dbg_lr_cost_ = np.load(p) 661 | 662 | cur_label = 'n_comp=%i' % n_hidden 663 | cur_label += '/' 664 | cur_label += 'lambda=%.1f' % lambda_param 665 | cur_label += '/' 666 | if not '_AE' in p: 667 | cur_label += 'LR only!' 668 | elif 'subRS' in p: 669 | cur_label += 'RSnormal' 670 | elif 'pca20RS' in p: 671 | cur_label += 'RSpca20' 672 | cur_label += '/' 673 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 674 | plt.plot( 675 | dbg_epochs_, 676 | dbg_lr_cost_, 677 | label=cur_label) 678 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss') 679 | plt.legend(loc='lower right', fontsize=9) 680 | # plt.yticks(np.linspace(0., 1., 11)) 681 | plt.ylabel('LR loss') 682 | plt.xlabel('epochs') 683 | plt.grid(True) 684 | plt.show() 685 | plt.savefig(op.join(WRITE_DIR, 'loss_lr_%icomps.png' % n_comp)) 686 | 687 | pkgs = glob.glob(RES_NAME + '/*dbg_combined_cost_*.npy') # combined loss 688 | for n_comp in n_comps: # AE 689 | plt.figure() 690 | for p in pkgs: 691 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 692 | n_hidden = int(re.search('comp=(?P.{1,2})_', p).group('comp')) 693 | if n_comp != n_hidden: 694 | continue 695 | 696 | dbg_combined_cost_ = np.load(p) 697 | 698 | cur_label = 'n_comp=%i' % n_hidden 699 | cur_label += '/' 700 | cur_label += 'lambda=%.1f' % lambda_param 701 | cur_label += '/' 702 | if not '_AE' in p: 703 | cur_label += 'LR only!' 704 | elif 'subRS' in p: 705 | cur_label += 'RSnormal' 706 | elif 'pca20RS' in p: 707 | cur_label += 'RSpca20' 708 | cur_label += '/' 709 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 710 | plt.plot( 711 | dbg_epochs_, 712 | dbg_combined_cost_, 713 | label=cur_label) 714 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss') 715 | plt.legend(loc='lower right', fontsize=9) 716 | # plt.yticks(np.linspace(0., 1., 11)) 717 | plt.ylabel('combined loss') 718 | plt.xlabel('epochs') 719 | plt.grid(True) 720 | plt.show() 721 | plt.savefig(op.join(WRITE_DIR, 'loss_combined_%icomps.png' % n_comp)) 722 | 723 | pkgs = glob.glob(RES_NAME + '/*lambda=0.25*dbg_prfs_.npy') 724 | for n_comp in n_comps: 725 | plt.figure() 726 | for p in pkgs: 727 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 728 | n_hidden = int(re.search('comp=(?P.{1,2})_', p).group('comp')) 729 | if n_comp != n_hidden: 730 | continue 731 | 732 | dbg_prfs_ = np.load(p) 733 | 734 | cur_label = 'n_comp=%i' % n_hidden 735 | cur_label += '/' 736 | cur_label += 'lambda=%.1f' % lambda_param 737 | cur_label += '/' 738 | if not '_AE' in p: 739 | cur_label += 'LR only!' 740 | elif 'subRS' in p: 741 | cur_label += 'RSnormal' 742 | elif 'pca20RS' in p: 743 | cur_label += 'RSpca20' 744 | cur_label += '/' 745 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 746 | for i in np.arange(38): 747 | plt.plot( 748 | dbg_epochs_, 749 | np.array(dbg_prfs_)[:, 0, i], 750 | label='task %i' % (i + 1)) 751 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=10mm combined-loss lambda=0.5') 752 | plt.legend(loc='lower right', fontsize=9) 753 | # plt.yticks(np.linspace(0., 1., 11)) 754 | plt.ylabel('in-dataset precisions') 755 | plt.ylim(0., 1.) 756 | plt.xlabel('epochs') 757 | plt.grid(True) 758 | plt.show() 759 | plt.savefig(op.join(WRITE_DIR, 'prec_inds_%icomps.png' % n_comp)) 760 | 761 | # in-dataset recall at lambda=0.5 762 | pkgs = glob.glob(RES_NAME + '/*lambda=0.25*dbg_prfs_.npy') 763 | for n_comp in n_comps: 764 | plt.figure() 765 | for p in pkgs: 766 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 767 | n_hidden = int(re.search('comp=(?P.{1,2})_', p).group('comp')) 768 | if n_comp != n_hidden: 769 | continue 770 | 771 | dbg_prfs_ = np.load(p) 772 | 773 | dbg_prfs_ = np.load(p) 774 | cur_label = 'n_comp=%i' % n_hidden 775 | cur_label += '/' 776 | cur_label += 'lambda=%.1f' % lambda_param 777 | cur_label += '/' 778 | if not '_AE' in p: 779 | cur_label += 'LR only!' 780 | elif 'subRS' in p: 781 | cur_label += 'RSnormal' 782 | elif 'pca20RS' in p: 783 | cur_label += 'RSpca20' 784 | cur_label += '/' 785 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 786 | for i in np.arange(38): 787 | plt.plot( 788 | dbg_epochs_, 789 | np.array(dbg_prfs_)[:, 1, i], 790 | label='task %i' % (i + 1)) 791 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=10mm combined-loss lambda=0.5') 792 | plt.legend(loc='lower right', fontsize=9) 793 | # plt.yticks(np.linspace(0., 1., 11)) 794 | plt.ylabel('in-dataset recall') 795 | plt.ylim(0., 1.) 796 | plt.xlabel('epochs') 797 | plt.grid(True) 798 | plt.show() 799 | plt.savefig(op.join(WRITE_DIR, 'rec_inds_%icomps.png' % n_comp)) 800 | 801 | # in-dataset f1 802 | # pkgs = glob.glob(RES_NAME + '/*lambda=1.00*dbg_prfs_.npy') 803 | pkgs = glob.glob(RES_NAME + '/*dbg_prfs_.npy') 804 | for n_comp in n_comps: 805 | plt.figure() 806 | p_ep = glob.glob(RES_NAME + '/*comp=%i*dbg_epochs_*.npy' % n_comp) 807 | dbg_epochs_ = np.load(p_ep[0]) 808 | for p in pkgs: 809 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 810 | n_hidden = int(re.search('comp=(?P.{1,3})_', p).group('comp')) 811 | if n_comp != n_hidden: 812 | continue 813 | 814 | dbg_prfs_ = np.load(p) 815 | 816 | cur_label = 'n_comp=%i' % n_hidden 817 | cur_label += '/' 818 | cur_label += 'lambda=%.1f' % lambda_param 819 | cur_label += '/' 820 | if not '_AE' in p: 821 | cur_label += 'LR only!' 822 | elif 'subRS' in p: 823 | cur_label += 'RSnormal' 824 | elif 'pca20RS' in p: 825 | cur_label += 'RSpca20' 826 | cur_label += '/' 827 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 828 | cur_label = 'Logistic regression' 829 | for i in np.arange(38): 830 | plt.plot( 831 | dbg_epochs_[:len(np.array(dbg_prfs_)[:, 2, i])], 832 | np.array(dbg_prfs_)[:, 2, i], 833 | label='task %i' % (i + 1)) 834 | # plt.title('Low-rank logistic regression ($n=%i$, $\lambda=1.0$)' % n_comp) 835 | plt.title('Logistic regression (voxel space)') 836 | # plt.title('Classification performance for 38 tasks') 837 | # plt.legend(loc='lower right', fontsize=9) 838 | # plt.yticks(np.linspace(0., 1., 11)) 839 | plt.ylabel('f1 score (test set)') 840 | plt.ylim(0., 1.05) 841 | plt.xlabel('epochs') 842 | plt.grid(True) 843 | plt.show() 844 | plt.savefig(op.join(WRITE_DIR, 'f1_inds_%icomps2.png' % n_comp)) 845 | 846 | # out-of-dataset precision at lambda=0.5 847 | pkgs = glob.glob(RES_NAME + '/*lambda=0.25*dbg_prfs_other_ds_.npy') 848 | for n_comp in n_comps: 849 | plt.figure() 850 | for p in pkgs: 851 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 852 | n_hidden = int(re.search('comp=(?P.{1,2})_', p).group('comp')) 853 | if n_comp != n_hidden: 854 | continue 855 | 856 | dbg_prfs_other_ds_ = np.load(p) 857 | 858 | cur_label = 'n_comp=%i' % n_hidden 859 | cur_label += '/' 860 | cur_label += 'lambda=%.1f' % lambda_param 861 | cur_label += '/' 862 | if not '_AE' in p: 863 | cur_label += 'LR only!' 864 | elif 'subRS' in p: 865 | cur_label += 'RSnormal' 866 | elif 'pca20RS' in p: 867 | cur_label += 'RSpca20' 868 | cur_label += '/' 869 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 870 | for i in np.arange(18): 871 | plt.plot( 872 | dbg_epochs_, 873 | np.array(dbg_prfs_other_ds_)[:, 0, i], 874 | label='task %i' % (i + 1)) 875 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=10mm combined-loss lambda=0.5') 876 | plt.legend(loc='lower right', fontsize=9) 877 | # plt.yticks(np.linspace(0., 1., 11)) 878 | plt.ylabel('out-of-dataset precisions') 879 | plt.ylim(0., 1.) 880 | plt.xlabel('epochs') 881 | plt.grid(True) 882 | plt.show() 883 | plt.savefig(op.join(WRITE_DIR, 'prec_oods_%icomps.png' % n_comp)) 884 | 885 | # out-of-dataset recall at lambda=0.5 886 | pkgs = glob.glob(RES_NAME + '/*lambda=0.25*dbg_prfs_other_ds_.npy') 887 | for n_comp in n_comps: 888 | plt.figure() 889 | for p in pkgs: 890 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 891 | n_hidden = int(re.search('comp=(?P.{1,2})_', p).group('comp')) 892 | if n_comp != n_hidden: 893 | continue 894 | 895 | dbg_prfs_other_ds_ = np.load(p) 896 | 897 | cur_label = 'n_comp=%i' % n_hidden 898 | cur_label += '/' 899 | cur_label += 'lambda=%.1f' % lambda_param 900 | cur_label += '/' 901 | if not '_AE' in p: 902 | cur_label += 'LR only!' 903 | elif 'subRS' in p: 904 | cur_label += 'RSnormal' 905 | elif 'pca20RS' in p: 906 | cur_label += 'RSpca20' 907 | cur_label += '/' 908 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 909 | for i in np.arange(18): 910 | plt.plot( 911 | dbg_epochs_, 912 | np.array(dbg_prfs_other_ds_)[:, 1, i], 913 | label='task %i' % (i + 1)) 914 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=10mm combined-loss lambda=0.5') 915 | plt.legend(loc='lower right', fontsize=9) 916 | # plt.yticks(np.linspace(0., 1., 11)) 917 | plt.ylabel('out-of-dataset recall') 918 | plt.ylim(0., 1.) 919 | plt.xlabel('epochs') 920 | plt.grid(True) 921 | plt.show() 922 | plt.savefig(op.join(WRITE_DIR, 'rec_oods_%icomps.png' % n_comp)) 923 | 924 | # out-of-dataset f1 at lambda=0.5 925 | pkgs = glob.glob(RES_NAME + '/*lambda=1.00*dbg_prfs_other_ds_.npy') 926 | for n_comp in n_comps: 927 | plt.figure() 928 | for p in pkgs: 929 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 930 | n_hidden = int(re.search('comp=(?P.{1,2})_', p).group('comp')) 931 | if n_comp != n_hidden: 932 | continue 933 | 934 | dbg_prfs_other_ds_ = np.load(p) 935 | 936 | cur_label = 'n_comp=%i' % n_hidden 937 | cur_label += '/' 938 | cur_label += 'lambda=%.1f' % lambda_param 939 | cur_label += '/' 940 | if not '_AE' in p: 941 | cur_label += 'LR only!' 942 | elif 'subRS' in p: 943 | cur_label += 'RSnormal' 944 | elif 'pca20RS' in p: 945 | cur_label += 'RSpca20' 946 | cur_label += '/' 947 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 948 | for i in np.arange(18): 949 | plt.plot( 950 | dbg_epochs_, 951 | np.array(dbg_prfs_other_ds_)[:, 2, i], 952 | label='task %i' % (i + 1)) 953 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=10mm combined-loss lambda=0.5') 954 | plt.legend(loc='lower right', fontsize=9) 955 | # plt.yticks(np.linspace(0., 1., 11)) 956 | plt.ylabel('out-of-dataset f1 score') 957 | plt.ylim(0., 1.) 958 | plt.xlabel('epochs') 959 | plt.grid(True) 960 | plt.show() 961 | plt.savefig(op.join(WRITE_DIR, 'f1_oods_%icomps.png' % n_comp)) 962 | 963 | # scatter plot against vanilla 964 | plt.close('all') 965 | newmeth1 = np.load(op.join(WRITE_DIR, 'Low-rank_LR_AE_(combined_loss,_shared_decomp)_n_comp=20_L1=0.1_L2=0.1_lambda=0.25_res=3mm_spca20RSdbg_prfs_.npy')) 966 | newmeth2 = np.load(op.join(WRITE_DIR, 'Low-rank_LR_AE_(combined_loss,_shared_decomp)_n_comp=100_L1=0.1_L2=0.1_lambda=0.25_res=3mm_spca20RSdbg_prfs_.npy')) 967 | ordinary = np.load(op.join('nips3mm_h38_vanilla', 'LR_L1=0.1_L2=0.1_res=3mmdbg_prfs_.npy')) 968 | 969 | plt.figure() 970 | ep_points = np.arange(min(len(ordinary), len(newmeth1))) 971 | for i in ep_points[10:30]: 972 | plt.scatter(ordinary[i, 2, :], newmeth1[i, 2, :], s=60) 973 | plt.show() 974 | plt.plot(np.linspace(0, 1., 521), np.linspace(0, 1., 521), 975 | '--', color='black') 976 | plt.xlim(0, 1.) 977 | plt.ylim(0, 1.) 978 | plt.xlabel('Plain Vanilla LogReg') 979 | plt.ylabel('Low-Rank LogReg (n=20)') 980 | plt.savefig(op.join(WRITE_DIR, 'scattern_againstLR_%icomps.png' % 20)) 981 | 982 | plt.figure() 983 | ep_points = np.arange(min(len(ordinary), len(newmeth2))) 984 | for i in ep_points[10:30]: 985 | plt.scatter(ordinary[i, 2, :], newmeth2[i, 2, :], s=60) 986 | plt.show() 987 | plt.plot(np.linspace(0, 1., 521), np.linspace(0, 1., 521), 988 | '--', color='black') 989 | plt.xlim(0, 1.) 990 | plt.ylim(0, 1.) 991 | plt.xlabel('Plain Vanilla LogReg') 992 | plt.ylabel('Low-Rank LogReg (n=100)') 993 | plt.savefig(op.join(WRITE_DIR, 'scattern_againstLR_%icomps.png' % 100)) 994 | 995 | # barplot against vanilla 996 | plt.close('all') 997 | newmeth1 = np.load(op.join(WRITE_DIR, 'Low-rank_LR_AE_(combined_loss,_shared_decomp)_n_comp=20_L1=0.1_L2=0.1_lambda=1.00_res=3mm_spca20RSdbg_prfs_.npy')) 998 | newmeth2 = np.load(op.join(WRITE_DIR, 'Low-rank_LR_AE_(combined_loss,_shared_decomp)_n_comp=100_L1=0.1_L2=0.1_lambda=1.00_res=3mm_spca20RSdbg_prfs_.npy')) 999 | newmeth = (newmeth1, newmeth2) 1000 | ordinary = np.load(op.join('nips3mm_h38_vanilla', 'LR_L1=0.1_L2=0.1_res=3mmdbg_prfs_.npy')) 1001 | n_comps = [20, 100] 1002 | for n_comp, newmeth in zip(n_comps, newmeth): 1003 | plt.figure() 1004 | my_width = 0.2 1005 | x_pos = 0 1006 | pos_list = list() 1007 | c_sslr = '#404040' 1008 | c_normal = '#51A2E0' 1009 | my_fs = 16 1010 | inds = np.argsort(newmeth[-1, 2, :]) 1011 | for i in np.arange(38): 1012 | if (i == 0): 1013 | plt.bar(x_pos, newmeth[-1, 2, inds[i]], width=my_width, color=c_sslr, 1014 | label='Factored Logistic Regression') 1015 | plt.bar(x_pos, ordinary[-1, 2, inds[i]], width=my_width, color=c_normal, 1016 | label='Ordinary Logistic Regression') 1017 | else: 1018 | plt.bar(x_pos, newmeth[-1, 2, inds[i]], width=my_width, color=c_sslr) 1019 | plt.bar(x_pos, ordinary[-1, 2, inds[i]], width=my_width, color=c_normal) 1020 | pos_list.append(x_pos) 1021 | x_pos += my_width 1022 | 1023 | plt.ylim(0, 1.05) 1024 | plt.xlim(0, x_pos) 1025 | plt.xlabel('mental task') 1026 | plt.title('$n=%i$' % n_comp, fontsize=my_fs) 1027 | plt.xticks( 1028 | np.array(pos_list[::2]) + my_width / 2, 1029 | np.arange(0, 38)[::2] + 1, 1030 | fontsize=my_fs) 1031 | plt.yticks(np.linspace(0, 1, 11), fontsize=my_fs) 1032 | plt.ylabel('f1 score') 1033 | plt.xlabel('task', fontsize=my_fs) 1034 | plt.ylabel('f1 score', fontsize=my_fs) 1035 | plt.legend(loc='lower right', fontsize=my_fs + 1) 1036 | plt.grid(True) 1037 | plt.show() 1038 | plt.savefig(op.join(WRITE_DIR, 'bars_against_LR_%icomps_sorted.png' % n_comp)) 1039 | 1040 | plt.xlim(0, 1.) 1041 | plt.ylim(0, 1.) 1042 | plt.savefig(op.join(WRITE_DIR, 'scattern_againstLR_%icomps.png' % 100)) 1043 | 1044 | # print components 1045 | pkgs = glob.glob(RES_NAME + '/*comps.npy') 1046 | for p in pkgs: 1047 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 1048 | n_hidden = int(re.search('comp=(?P.{1,2})_', p).group('comp')) 1049 | if n_comp != n_hidden: 1050 | continue 1051 | 1052 | new_fname = 'comps_n=%i_lambda=%.2f' % (n_hidden, lambda_param) 1053 | comps = np.load(p) 1054 | dump_comps(nifti_masker, new_fname, comps, threshold=0.5) -------------------------------------------------------------------------------- /nips3mm_recovery.py: -------------------------------------------------------------------------------- 1 | """ 2 | HCP: Semi-supervised network decomposition by low-rank logistic regression 3 | """ 4 | 5 | print __doc__ 6 | 7 | import os 8 | import os.path as op 9 | import numpy as np 10 | import glob 11 | from scipy.linalg import norm 12 | import nibabel as nib 13 | from sklearn.grid_search import RandomizedSearchCV 14 | from sklearn.base import BaseEstimator 15 | from sklearn.preprocessing import StandardScaler 16 | from nilearn.input_data import NiftiMasker 17 | from sklearn.metrics import precision_recall_fscore_support 18 | from sklearn.metrics import confusion_matrix 19 | import theano 20 | import theano.tensor as T 21 | from matplotlib import pylab as plt 22 | print('Running THEANO on %s' % theano.config.device) 23 | from nilearn.image import concat_imgs 24 | import joblib 25 | import time 26 | from scipy.stats import zscore 27 | 28 | LR_AE_DIR = 'nips3mm' 29 | LR_DIR = 'nips3mm_vanilla' 30 | 31 | ############################################################################## 32 | # abc 33 | ############################################################################## 34 | 35 | class SSEncoder(BaseEstimator): 36 | def __init__(self, gain1, learning_rate, max_epochs=100, 37 | l1=0.1, l2=0.1): 38 | """ 39 | Parameters 40 | ---------- 41 | lambda : float 42 | Mediates between AE and LR. lambda==1 equates with LR only. 43 | """ 44 | self.gain1 = gain1 45 | self.max_epochs = max_epochs 46 | self.learning_rate = np.float32(learning_rate) 47 | self.penalty_l1 = np.float32(l1) 48 | self.penalty_l2 = np.float32(l2) 49 | 50 | # def rectify(X): 51 | # return T.maximum(0., X) 52 | 53 | from theano.tensor.shared_randomstreams import RandomStreams 54 | 55 | def RMSprop(self, cost, params, lr=0.001, rho=0.9, epsilon=1e-6): 56 | grads = T.grad(cost=cost, wrt=params) 57 | updates = [] 58 | for p, g in zip(params, grads): 59 | acc = theano.shared(p.get_value() * 0.) 60 | acc_new = rho * acc + (1 - rho) * g ** 2 61 | gradient_scaling = T.sqrt(acc_new + epsilon) 62 | g = g / gradient_scaling 63 | updates.append((acc, acc_new)) 64 | updates.append((p, p - lr * g)) 65 | return updates 66 | 67 | def get_param_pool(self): 68 | cur_params = ( 69 | self.V0s, self.bV0 70 | ) 71 | return cur_params 72 | 73 | def fit(self, X_task, y): 74 | DEBUG_FLAG = True 75 | 76 | # self.max_epochs = 333 77 | self.batch_size = 100 78 | rng = np.random.RandomState(42) 79 | self.input_taskdata = T.matrix(dtype='float32', name='input_taskdata') 80 | self.input_restdata = T.matrix(dtype='float32', name='input_restdata') 81 | self.params_from_last_iters = [] 82 | n_input = X_task.shape[1] 83 | 84 | index = T.iscalar(name='index') 85 | 86 | # prepare data for theano computation 87 | if not DEBUG_FLAG: 88 | X_train_s = theano.shared( 89 | value=np.float32(X_task), name='X_train_s') 90 | y_train_s = theano.shared( 91 | value=np.int32(y), name='y_train_s') 92 | lr_train_samples = len(X_task) 93 | else: 94 | from sklearn.cross_validation import StratifiedShuffleSplit 95 | folder = StratifiedShuffleSplit(y, n_iter=1, test_size=0.20) 96 | new_trains, inds_val = iter(folder).next() 97 | X_train, X_val = X_task[new_trains], X_task[inds_val] 98 | y_train, y_val = y[new_trains], y[inds_val] 99 | 100 | X_train_s = theano.shared(value=np.float32(X_train), 101 | name='X_train_s', borrow=False) 102 | y_train_s = theano.shared(value=np.int32(y_train), 103 | name='y_train_s', borrow=False) 104 | # X_val_s = theano.shared(value=np.float32(X_val), 105 | # name='X_train_s', borrow=False) 106 | # y_val_s = theano.shared(value=np.int32(y_val), 107 | # name='y_cal_s', borrow=False) 108 | lr_train_samples = len(X_train) 109 | self.dbg_epochs_ = list() 110 | self.dbg_acc_train_ = list() 111 | self.dbg_acc_val_ = list() 112 | self.dbg_ae_cost_ = list() 113 | self.dbg_lr_cost_ = list() 114 | self.dbg_ae_nonimprovesteps = list() 115 | self.dbg_acc_other_ds_ = list() 116 | self.dbg_prfs_ = list() 117 | self.dbg_prfs_other_ds_ = list() 118 | 119 | # computation graph: logistic regression 120 | clf_n_output = 18 # number of labels 121 | my_y = T.ivector(name='y') 122 | 123 | bV0_vals = np.zeros(clf_n_output).astype(np.float32) 124 | self.bV0 = theano.shared(value=bV0_vals, name='bV0') 125 | 126 | V0_vals = rng.randn(n_input, clf_n_output).astype(np.float32) * self.gain1 127 | self.V0s = theano.shared(V0_vals) 128 | 129 | self.p_y_given_x = T.nnet.softmax( 130 | T.dot(self.input_taskdata, self.V0s) + self.bV0 131 | ) 132 | self.lr_cost = -T.mean(T.log(self.p_y_given_x)[T.arange(my_y.shape[0]), my_y]) 133 | self.lr_cost = ( 134 | self.lr_cost + 135 | T.mean(abs(self.V0s)) * self.penalty_l1 + 136 | T.mean(abs(self.bV0)) * self.penalty_l1 + 137 | 138 | T.mean((self.V0s ** np.float32(2))) * self.penalty_l2 + 139 | T.mean((self.bV0 ** np.float32(2))) * self.penalty_l2 140 | ) 141 | self.y_pred = T.argmax(self.p_y_given_x, axis=1) 142 | 143 | givens_lr = { 144 | self.input_taskdata: X_train_s[index * self.batch_size:(index + 1) * self.batch_size], 145 | my_y: y_train_s[index * self.batch_size:(index + 1) * self.batch_size] 146 | } 147 | 148 | params = [self.V0s, self.bV0] 149 | updates = self.RMSprop(cost=self.lr_cost, params=params, 150 | lr=self.learning_rate) 151 | 152 | f_train_lr = theano.function( 153 | [index], 154 | [self.lr_cost], 155 | givens=givens_lr, 156 | updates=updates) 157 | 158 | # optimization loop 159 | start_time = time.time() 160 | lr_last_cost = np.inf 161 | ae_cur_cost = np.inf 162 | no_improve_steps = 0 163 | acc_train, acc_val = 0., 0. 164 | for i_epoch in range(self.max_epochs): 165 | if i_epoch == 1: 166 | epoch_dur = time.time() - start_time 167 | total_mins = (epoch_dur * self.max_epochs) / 60 168 | hs, mins = divmod(total_mins, 60) 169 | print("Max estimated duration: %i hours and %i minutes" % (hs, mins)) 170 | 171 | lr_n_batches = lr_train_samples // self.batch_size 172 | for i in range(lr_n_batches): 173 | lr_cur_cost = f_train_lr(i)[0] 174 | 175 | # evaluate epoch cost 176 | if lr_last_cost - lr_cur_cost < 0.1: 177 | no_improve_steps += 1 178 | else: 179 | lr_last_cost = lr_cur_cost 180 | no_improve_steps = 0 181 | 182 | # logistic 183 | lr_last_cost = lr_cur_cost 184 | acc_train = self.score(X_train, y_train) 185 | acc_val, prfs_val = self.score(X_val, y_val, return_prfs=True) 186 | 187 | print('E:%i, ae_cost:%.4f, lr_cost:%.4f, train_score:%.2f, vald_score:%.2f, ae_badsteps:%i' % ( 188 | i_epoch + 1, ae_cur_cost, lr_cur_cost, acc_train, acc_val, no_improve_steps)) 189 | 190 | if (i_epoch % 10 == 0): 191 | self.dbg_ae_cost_.append(ae_cur_cost) 192 | self.dbg_lr_cost_.append(lr_cur_cost) 193 | 194 | self.dbg_epochs_.append(i_epoch + 1) 195 | self.dbg_ae_nonimprovesteps.append(no_improve_steps) 196 | self.dbg_acc_train_.append(acc_train) 197 | self.dbg_acc_val_.append(acc_val) 198 | self.dbg_prfs_.append(prfs_val) 199 | 200 | # if i_epoch > (self.max_epochs - 100): 201 | param_pool = self.get_param_pool() 202 | self.params_from_last_iters.append(param_pool) 203 | 204 | total_mins = (time.time() - start_time) / 60 205 | hs, mins = divmod(total_mins, 60) 206 | print("Final duration: %i hours and %i minutes" % (hs, mins)) 207 | 208 | return self 209 | 210 | def predict(self, X): 211 | X_test_s = theano.shared(value=np.float32(X), name='X_test_s', borrow=True) 212 | 213 | givens_te = { 214 | self.input_taskdata: X_test_s 215 | } 216 | 217 | f_test = theano.function( 218 | [], 219 | [self.y_pred], 220 | givens=givens_te) 221 | predictions = f_test() 222 | del X_test_s 223 | del givens_te 224 | return predictions[0] 225 | 226 | def score(self, X, y, return_prfs=False): 227 | pred_y = self.predict(X) 228 | acc = np.mean(pred_y == y) 229 | prfs = precision_recall_fscore_support(pred_y, y) 230 | if return_prfs: 231 | return acc, prfs 232 | else: 233 | return acc 234 | 235 | ############################################################################## 236 | # abc 237 | ############################################################################## 238 | 239 | 240 | 241 | means_path = '/git/cohort/archi/compr2task_means' 242 | 243 | mask_img = 'grey10_icbm_3mm_bin.nii.gz' 244 | nifti_masker = NiftiMasker(mask_img=mask_img, smoothing_fwhm=False, 245 | standardize=False) 246 | nifti_masker.fit() 247 | mask_nvox = nifti_masker.mask_img_.get_data().sum() 248 | 249 | # get HCP data 250 | HCP_contrasts = [ 251 | 'REWARD-PUNISH', 'PUNISH-REWARD', 'SHAPES-FACES', 'FACES-SHAPES', 252 | 'RANDOM-TOM', 'TOM-RANDOM', 253 | 254 | 'MATH-STORY', 'STORY-MATH', 255 | 'T-AVG', 'F-H', 'H-F', 256 | 'MATCH-REL', 'REL-MATCH', 257 | 258 | 'BODY-AVG', 'FACE-AVG', 'PLACE-AVG', 'TOOL-AVG', 259 | '2BK-0BK' 260 | ] 261 | mean_supp = np.zeros((18, mask_nvox)) 262 | from nilearn.image import resample_img 263 | for itask, task in enumerate(HCP_contrasts): 264 | cur_nii = op.join(means_path, 'mean_%s.nii.gz' % (task)) 265 | print(cur_nii) 266 | res_nii = resample_img(cur_nii, 267 | target_affine=nifti_masker.mask_img_.get_affine(), 268 | target_shape=nifti_masker.mask_img_.shape) 269 | task_mean = nifti_masker.transform(res_nii) 270 | mean_supp[itask, :] = task_mean 271 | mean_supp_z = zscore(mean_supp, axis=1) 272 | 273 | # get classification weights 274 | lr_supp = np.load(op.join(LR_DIR, 'V0comps.npy')) 275 | lr_supp_z = zscore(lr_supp, axis=1) 276 | 277 | # get LR/AE weights 278 | WRITE_DIR = 'nips3mm_recovery' 279 | lambs = [0.25, 0.5, 0.75, 1] 280 | import re 281 | from scipy.stats import pearsonr 282 | for n_comp in [5]: 283 | corr_means_lr = np.zeros((len(lambs), 18)) 284 | corr_means_lr_ae = np.zeros((len(lambs), 18)) 285 | for ilamb, lamb in enumerate(lambs): 286 | pkgs = glob.glob(op.join(LR_AE_DIR, '*comp=%i_*V1comps*' % n_comp)) 287 | for p in pkgs: 288 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 289 | # n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 290 | if lamb != lambda_param: 291 | continue 292 | 293 | print(p) 294 | q = p.replace('V1comp', 'W0comp') 295 | 296 | try: 297 | mat_V1 = np.load(p) 298 | mat_W0 = np.load(q) 299 | except: 300 | print('Did not find %s oder %s!' % (p, q)) 301 | 302 | lr_ae_supp = np.dot(mat_V1, mat_W0) 303 | lr_ae_supp_z = zscore(lr_ae_supp, axis=1) 304 | 305 | for i in np.arange(18): 306 | r1, p1 = pearsonr(lr_ae_supp_z[i, :], mean_supp_z[i, :]) 307 | print('r/lrae: %.4f' % r1) 308 | corr_means_lr_ae[ilamb, i] = r1 309 | 310 | r2, p2 = pearsonr(lr_supp_z[i, :], mean_supp_z[i, :]) 311 | print('r/lr: %.4f' % r2) 312 | corr_means_lr[ilamb, i] = r2 313 | 314 | # boxplot 315 | plt.figure() 316 | corrs = np.vstack((corr_means_lr[0, :], corr_means_lr_ae)) 317 | plt.boxplot(corrs.T) 318 | plt.ylabel('correlation r') 319 | plt.title('Support Recovery: normal versus low-rank logistic regression\n' 320 | '%i components' % n_comp) 321 | tick_strs = [u'normal'] + [u'low-rank lambda=%.2f' % val for val in lambs] 322 | plt.xticks(np.arange(6) + 1, tick_strs, rotation=320) 323 | plt.ylim(0, 1.0) 324 | plt.yticks(np.linspace(0, 1., 11), np.linspace(0, 1., 11)) 325 | plt.tight_layout() 326 | 327 | out_path = op.join(WRITE_DIR, 'supp_recov_comp=%i.png' % n_comp) 328 | plt.savefig(out_path) 329 | 330 | # barplot 331 | plt.figure() 332 | ind = np.arange(5) 333 | width = 1. 334 | colors = [(242., 62., 22.), #(7., 196., 255.), 335 | (7., 176., 242.), (7., 136., 217.), (7., 40., 164.), (1., 4., 64.)] 336 | my_colors = [(x/256, y/256, z/256) for x, y, z in colors] 337 | plt.bar(ind, np.mean(corrs, axis=1), yerr=np.std(corrs, axis=1), 338 | width=width, color=my_colors) 339 | plt.ylabel('correlation r (+/- SD)') 340 | plt.title('Support Recovery: normal versus low-rank logistic regression\n' 341 | '%i components' % n_comp) 342 | plt.xticks(ind + width / 2., tick_strs, rotation=320) 343 | plt.ylim(0, 1.0) 344 | plt.yticks(np.linspace(0, 1., 11), np.linspace(0, 1., 11)) 345 | plt.tight_layout() 346 | out_path2 = out_path.replace('.png', '_bars.png') 347 | plt.savefig(out_path2) 348 | 349 | 350 | -------------------------------------------------------------------------------- /nips3mm_scarcity.py: -------------------------------------------------------------------------------- 1 | """ 2 | HCP: Semi-supervised network decomposition by low-rank logistic regression 3 | """ 4 | 5 | print __doc__ 6 | 7 | import os 8 | import os.path as op 9 | import numpy as np 10 | import glob 11 | from scipy.linalg import norm 12 | import nibabel as nib 13 | from sklearn.grid_search import RandomizedSearchCV 14 | from sklearn.base import BaseEstimator 15 | from sklearn.preprocessing import StandardScaler 16 | from nilearn.input_data import NiftiMasker 17 | from sklearn.metrics import precision_recall_fscore_support 18 | from sklearn.metrics import confusion_matrix 19 | import theano 20 | import theano.tensor as T 21 | from matplotlib import pylab as plt 22 | print('Running THEANO on %s' % theano.config.device) 23 | from nilearn.image import concat_imgs 24 | import joblib 25 | import time 26 | from nilearn.image import smooth_img 27 | 28 | RES_NAME = 'nips3mm_scarcity' 29 | WRITE_DIR = op.join(os.getcwd(), RES_NAME) 30 | if not op.exists(WRITE_DIR): 31 | os.mkdir(WRITE_DIR) 32 | 33 | MAX_SAMPLES = 2000 34 | MAX_REST_SAMPLES = 100 35 | MAX_TASK_SAMPLES = 100 36 | 37 | ############################################################################## 38 | # load+preprocess data 39 | ############################################################################## 40 | 41 | mask_img = 'grey10_icbm_3mm_bin.nii.gz' 42 | nifti_masker = NiftiMasker(mask_img=mask_img, smoothing_fwhm=False, 43 | standardize=False) 44 | nifti_masker.fit() 45 | mask_nvox = nifti_masker.mask_img_.get_data().sum() 46 | 47 | print('Loading data...') 48 | 49 | # ARCHI task 50 | X_task, labels = joblib.load('preload_HT_3mm') 51 | 52 | labels = np.int32(labels) 53 | 54 | # contrasts are IN ORDER -> shuffle! 55 | new_inds = np.arange(0, X_task.shape[0]) 56 | np.random.shuffle(new_inds) 57 | X_task = X_task[new_inds] 58 | labels = labels[new_inds] 59 | # subs = subs[new_inds] 60 | 61 | # rest 62 | # X_rest = nifti_masker.transform('preload_HR20persub_10mm_ero2.nii') 63 | # X_rest = nifti_masker.transform('dump_rs_spca_s12_tmp') 64 | rs_spca_data = joblib.load('dump_rs_spca_s12_tmp') 65 | rs_spca_niis = nib.Nifti1Image(rs_spca_data, 66 | nifti_masker.mask_img_.get_affine()) 67 | X_rest = nifti_masker.transform(rs_spca_niis) 68 | del rs_spca_niis 69 | del rs_spca_data 70 | 71 | X_task = StandardScaler().fit_transform(X_task) 72 | X_rest = StandardScaler().fit_transform(X_rest) 73 | 74 | # ARCHI task 75 | AT_niis, AT_labels, AT_subs = joblib.load('preload_AT_3mm') 76 | AT_X = nifti_masker.transform(AT_niis) 77 | AT_X = StandardScaler().fit_transform(AT_X) 78 | print('done :)') 79 | 80 | def make_new_noise(masker): 81 | # CONST 82 | n_random_niis = 1000 83 | n_random_foci = 200, 300 # random amount of foci per study 84 | fwhm_range = (4, 20) 85 | 86 | print "Inventing NOISE images from Gaussians..." 87 | # generate some noise niftis 88 | inds = np.where(masker.mask_img_.get_data()) # grid of locations 89 | x_inds, y_inds, z_inds = inds # unpack tuple 90 | list_niis = [] 91 | for inii in xrange(n_random_niis): 92 | # if inii % 24 == 0: 93 | # print "%i/%i" % (inii + 1, n_random_niis) 94 | nfoci = np.random.randint(*n_random_foci) 95 | cur_img = np.zeros(masker.mask_img_.shape) 96 | for ifocus in xrange(nfoci): 97 | # find random point within mask 98 | i = np.random.randint(0, len(x_inds) - 1) 99 | x, y, z = x_inds[i], y_inds[i], z_inds[i] 100 | # put a dot there 101 | if np.random.randint(0, 2): 102 | cur_img[x, y, z] = 150 103 | else: 104 | cur_img[x, y, z] = -150 105 | 106 | # smooth current image of random foci 107 | cur_fwhm = np.random.randint(*fwhm_range) 108 | new_nii = smooth_img( 109 | nib.Nifti1Image(cur_img, masker.mask_img_.get_affine(), 110 | header=masker.mask_img_.get_header()), cur_fwhm) 111 | 112 | list_niis.append(new_nii) 113 | 114 | X_rest = masker.transform(list_niis) 115 | 116 | return np.vstack((X_rest, X_rest, X_rest, X_rest, 117 | X_rest, X_rest, X_rest, X_rest)) 118 | 119 | ############################################################################## 120 | # define computation graph 121 | ############################################################################## 122 | 123 | class SSEncoder(BaseEstimator): 124 | def __init__(self, n_hidden, gain1, learning_rate, max_epochs=100, 125 | l1=0.1, l2=0.1, lambda_param=.5): 126 | """ 127 | Parameters 128 | ---------- 129 | lambda : float 130 | Mediates between AE and LR. lambda==1 equates with LR only. 131 | """ 132 | self.n_hidden = n_hidden 133 | self.gain1 = gain1 134 | self.max_epochs = max_epochs 135 | self.learning_rate = np.float32(learning_rate) 136 | self.penalty_l1 = np.float32(l1) 137 | self.penalty_l2 = np.float32(l2) 138 | self.lambda_param = np.float32(lambda_param) 139 | 140 | # def rectify(X): 141 | # return T.maximum(0., X) 142 | 143 | from theano.tensor.shared_randomstreams import RandomStreams 144 | 145 | def RMSprop(self, cost, params, lr=0.001, rho=0.9, epsilon=1e-6): 146 | grads = T.grad(cost=cost, wrt=params) 147 | updates = [] 148 | for p, g in zip(params, grads): 149 | acc = theano.shared(p.get_value() * 0.) 150 | acc_new = rho * acc + (1 - rho) * g ** 2 151 | gradient_scaling = T.sqrt(acc_new + epsilon) 152 | g = g / gradient_scaling 153 | updates.append((acc, acc_new)) 154 | updates.append((p, p - lr * g)) 155 | return updates 156 | 157 | def get_param_pool(self): 158 | cur_params = ( 159 | self.V1s, self.bV0, self.bV1, 160 | self.W0s, self.W1s, self.bW0s, self.bW1s 161 | ) 162 | return cur_params 163 | 164 | def test_performance_in_other_dataset(self): 165 | from sklearn.linear_model import LogisticRegression 166 | from sklearn.cross_validation import StratifiedShuffleSplit 167 | 168 | compr_matrix = self.W0s.get_value().T # currently best compression 169 | AT_X_compr = np.dot(compr_matrix, AT_X.T).T 170 | clf = LogisticRegression(penalty='l1') 171 | folder = StratifiedShuffleSplit(y=AT_labels, n_iter=5, test_size=0.2) 172 | 173 | acc_list = [] 174 | prfs_list = [] 175 | for (train_inds, test_inds) in folder: 176 | clf.fit(AT_X_compr[train_inds, :], AT_labels[train_inds]) 177 | pred_y = clf.predict(AT_X_compr[test_inds, :]) 178 | 179 | acc = (pred_y == AT_labels[test_inds]).mean() 180 | prfs_list.append(precision_recall_fscore_support( 181 | AT_labels[test_inds], pred_y)) 182 | 183 | acc_list.append(acc) 184 | 185 | compr_mean_acc = np.mean(acc_list) 186 | prfs = np.asarray(prfs_list).mean(axis=0) 187 | return compr_mean_acc, prfs 188 | 189 | def fit(self, X_rest, X_task, y): 190 | DEBUG_FLAG = True 191 | 192 | # self.max_epochs = 333 193 | self.batch_size = 100 194 | n_input = X_rest.shape[1] # sklearn-like structure 195 | n_output = n_input 196 | rng = np.random.RandomState(42) 197 | self.input_taskdata = T.matrix(dtype='float32', name='input_taskdata') 198 | self.input_restdata = T.matrix(dtype='float32', name='input_restdata') 199 | self.params_from_last_iters = [] 200 | 201 | index = T.iscalar(name='index') 202 | 203 | # prepare data for theano computation 204 | if not DEBUG_FLAG: 205 | X_train_s = theano.shared( 206 | value=np.float32(X_task), name='X_train_s') 207 | y_train_s = theano.shared( 208 | value=np.int32(y), name='y_train_s') 209 | lr_train_samples = len(X_task) 210 | else: 211 | from sklearn.cross_validation import StratifiedShuffleSplit 212 | folder = StratifiedShuffleSplit(y, n_iter=1, test_size=0.20) 213 | new_trains, inds_val = iter(folder).next() 214 | X_train, X_val = X_task[new_trains], X_task[inds_val] 215 | y_train, y_val = y[new_trains], y[inds_val] 216 | 217 | X_train_s = theano.shared(value=np.float32(X_train), 218 | name='X_train_s', borrow=False) 219 | y_train_s = theano.shared(value=np.int32(y_train), 220 | name='y_train_s', borrow=False) 221 | # X_val_s = theano.shared(value=np.float32(X_val), 222 | # name='X_train_s', borrow=False) 223 | # y_val_s = theano.shared(value=np.int32(y_val), 224 | # name='y_cal_s', borrow=False) 225 | lr_train_samples = len(X_train) 226 | self.dbg_epochs_ = list() 227 | self.dbg_acc_train_ = list() 228 | self.dbg_acc_val_ = list() 229 | self.dbg_ae_cost_ = list() 230 | self.dbg_lr_cost_ = list() 231 | self.dbg_ae_nonimprovesteps = list() 232 | self.dbg_acc_other_ds_ = list() 233 | self.dbg_combined_cost_ = list() 234 | self.dbg_prfs_ = list() 235 | self.dbg_prfs_other_ds_ = list() 236 | X_rest_s = theano.shared(value=np.float32(X_rest), name='X_rest_s') 237 | ae_train_samples = len(X_rest) 238 | 239 | # V -> supervised / logistic regression 240 | # W -> unsupervised / auto-encoder 241 | 242 | # computational graph: auto-encoder 243 | W0_vals = rng.randn(n_input, self.n_hidden).astype(np.float32) * self.gain1 244 | 245 | self.W0s = theano.shared(W0_vals) 246 | self.W1s = self.W0s.T # tied 247 | bW0_vals = np.zeros(self.n_hidden).astype(np.float32) 248 | self.bW0s = theano.shared(value=bW0_vals, name='bW0') 249 | bW1_vals = np.zeros(n_output).astype(np.float32) 250 | self.bW1s = theano.shared(value=bW1_vals, name='bW1') 251 | 252 | givens_ae = { 253 | self.input_restdata: X_rest_s[ 254 | index * self.batch_size:(index + 1) * self.batch_size] 255 | } 256 | 257 | encoding = (self.input_restdata.dot(self.W0s) + self.bW0s).dot(self.W1s) + self.bW1s 258 | 259 | self.ae_loss = T.sum((self.input_restdata - encoding) ** 2, axis=1) 260 | 261 | self.ae_cost = ( 262 | T.mean(self.ae_loss) / n_input 263 | ) 264 | 265 | # params1 = [self.W0s, self.bW0s, self.bW1s] 266 | # gparams1 = [T.grad(cost=self.ae_cost, wrt=param1) for param1 in params1] 267 | # 268 | # lr = self.learning_rate 269 | # updates = self.RMSprop(cost=self.ae_cost, params=params1, 270 | # lr=self.learning_rate) 271 | 272 | # f_train_ae = theano.function( 273 | # [index], 274 | # [self.ae_cost], 275 | # givens=givens_ae, 276 | # updates=updates) 277 | 278 | # computation graph: logistic regression 279 | clf_n_output = 18 # number of labels 280 | my_y = T.ivector(name='y') 281 | 282 | bV0_vals = np.zeros(self.n_hidden).astype(np.float32) 283 | self.bV0 = theano.shared(value=bV0_vals, name='bV0') 284 | bV1_vals = np.zeros(clf_n_output).astype(np.float32) 285 | self.bV1 = theano.shared(value=bV1_vals, name='bV1') 286 | 287 | # V0_vals = rng.randn(n_input, self.n_hidden).astype(np.float32) * self.gain1 288 | V1_vals = rng.randn(self.n_hidden, clf_n_output).astype(np.float32) * self.gain1 289 | # self.V0s = theano.shared(V0_vals) 290 | self.V1s = theano.shared(V1_vals) 291 | 292 | self.p_y_given_x = T.nnet.softmax( 293 | # T.dot(T.dot(self.input_taskdata, self.V0s) + self.bV0, self.V1s) + self.bV1 294 | T.dot(T.dot(self.input_taskdata, self.W0s) + self.bV0, self.V1s) + self.bV1 295 | ) 296 | self.lr_cost = -T.mean(T.log(self.p_y_given_x)[T.arange(my_y.shape[0]), my_y]) 297 | self.lr_cost = ( 298 | self.lr_cost + 299 | T.mean(abs(self.W0s)) * self.penalty_l1 + 300 | # T.mean(abs(self.V0s)) * self.penalty_l1 + 301 | T.mean(abs(self.bV0)) * self.penalty_l1 + 302 | T.mean(abs(self.V1s)) * self.penalty_l1 + 303 | T.mean(abs(self.bV1)) * self.penalty_l1 + 304 | 305 | T.mean((self.W0s ** np.float32(2))) * self.penalty_l2 + 306 | # T.mean((self.V0s ** 2)) * self.penalty_l2 + 307 | T.mean((self.bV0 ** np.float32(2))) * self.penalty_l2 + 308 | T.mean((self.V1s ** np.float32(2))) * self.penalty_l2 + 309 | T.mean((self.bV1 ** np.float32(2))) * self.penalty_l2 310 | ) 311 | self.y_pred = T.argmax(self.p_y_given_x, axis=1) 312 | 313 | givens_lr = { 314 | self.input_taskdata: X_train_s[index * self.batch_size:(index + 1) * self.batch_size], 315 | my_y: y_train_s[index * self.batch_size:(index + 1) * self.batch_size] 316 | } 317 | 318 | # params2 = [self.V0s, self.bV0, self.V1s, self.bV1] 319 | # params2 = [self.W0s, self.bV0, self.V1s, self.bV1] 320 | # updates2 = self.RMSprop(cost=self.lr_cost, params=params2, 321 | # lr=self.learning_rate) 322 | 323 | # f_train_lr = theano.function( 324 | # [index], 325 | # [self.lr_cost], 326 | # givens=givens_lr, 327 | # updates=updates2) 328 | 329 | # combined loss for AE and LR 330 | combined_params = [self.W0s, self.bW0s, self.bW1s, 331 | # self.V0s, self.V1s, self.bV0, self.bV1] 332 | self.V1s, self.bV0, self.bV1] 333 | self.combined_cost = ( 334 | (np.float32(1) - self.lambda_param) * self.ae_cost + 335 | self.lambda_param * self.lr_cost 336 | ) 337 | combined_updates = self.RMSprop( 338 | cost=self.combined_cost, 339 | params=combined_params, 340 | lr=self.learning_rate) 341 | givens_combined = { 342 | self.input_restdata: X_rest_s[index * self.batch_size:(index + 1) * self.batch_size], 343 | self.input_taskdata: X_train_s[index * self.batch_size:(index + 1) * self.batch_size], 344 | my_y: y_train_s[index * self.batch_size:(index + 1) * self.batch_size] 345 | } 346 | f_train_combined = theano.function( 347 | [index], 348 | [self.combined_cost, self.ae_cost, self.lr_cost], 349 | givens=givens_combined, 350 | updates=combined_updates, allow_input_downcast=True) 351 | 352 | # optimization loop 353 | start_time = time.time() 354 | ae_last_cost = np.inf 355 | lr_last_cost = np.inf 356 | no_improve_steps = 0 357 | acc_train, acc_val = 0., 0. 358 | for i_epoch in range(self.max_epochs): 359 | if i_epoch == 1: 360 | epoch_dur = time.time() - start_time 361 | total_mins = (epoch_dur * self.max_epochs) / 60 362 | hs, mins = divmod(total_mins, 60) 363 | print("Max estimated duration: %i hours and %i minutes" % (hs, mins)) 364 | 365 | # resample + restrict task and rest data 366 | inds_rest = np.random.randint(0, MAX_REST_SAMPLES, MAX_SAMPLES) 367 | assert np.bincount(inds_rest).sum() == MAX_SAMPLES 368 | 369 | X_rest_s.set_value(make_new_noise(nifti_masker), borrow=False) 370 | # X_rest_s.set_value(np.float32(X_rest[inds_rest]), borrow=False) 371 | 372 | ae_train_samples = MAX_SAMPLES 373 | 374 | inds_task = np.random.randint(0, MAX_TASK_SAMPLES, MAX_SAMPLES) 375 | assert np.bincount(inds_task).sum() == MAX_SAMPLES 376 | X_train_s.set_value(np.float32(X_train[inds_task]), borrow=False) 377 | y_train_s.set_value(np.int32(y_train[inds_task]), borrow=False) 378 | lr_train_samples = MAX_SAMPLES 379 | 380 | # AE 381 | # if i_epoch % 2 == 0: # every second time 382 | #if False: 383 | # auto-encoder 384 | ae_n_batches = ae_train_samples // self.batch_size 385 | lr_n_batches = lr_train_samples // self.batch_size 386 | # for i in range(lr_n_batches): 387 | # for i in range(max(ae_n_batches, lr_n_batches)): 388 | # if i < ae_n_batches: 389 | # ae_cur_cost = float(f_train_ae(i)[0]) 390 | # ae_cur_cost = 0 391 | # if i < lr_n_batches: 392 | # lr_cur_cost = float(f_train_lr(i)[0]) 393 | # for i in range(lr_n_batches): 394 | for i in range(min(ae_n_batches, lr_n_batches)): 395 | # lr_cur_cost = f_train_lr(i)[0] 396 | # ae_cur_cost = lr_cur_cost 397 | combined_cost, ae_cur_cost, lr_cur_cost = f_train_combined(i) 398 | 399 | # evaluate epoch cost 400 | if ae_last_cost - ae_cur_cost < 0.1: 401 | no_improve_steps += 1 402 | else: 403 | ae_last_cost = ae_cur_cost 404 | no_improve_steps = 0 405 | 406 | # logistic 407 | lr_last_cost = lr_cur_cost 408 | acc_train = self.score(X_train, y_train) 409 | acc_val, prfs_val = self.score(X_val, y_val, return_prfs=True) 410 | 411 | print('E:%i, ae_cost:%.4f, lr_cost:%.4f, train_score:%.2f, vald_score:%.2f, ae_badsteps:%i' % ( 412 | i_epoch + 1, ae_cur_cost, lr_cur_cost, acc_train, acc_val, no_improve_steps)) 413 | 414 | if (i_epoch % 10 == 0): 415 | self.dbg_ae_cost_.append(ae_cur_cost) 416 | self.dbg_lr_cost_.append(lr_cur_cost) 417 | self.dbg_combined_cost_.append(combined_cost) 418 | 419 | self.dbg_epochs_.append(i_epoch + 1) 420 | self.dbg_ae_nonimprovesteps.append(no_improve_steps) 421 | self.dbg_acc_train_.append(acc_train) 422 | self.dbg_acc_val_.append(acc_val) 423 | self.dbg_prfs_.append(prfs_val) 424 | 425 | # test out-of-dataset performance 426 | od_acc, prfs_other = self.test_performance_in_other_dataset() 427 | self.dbg_acc_other_ds_.append(od_acc) 428 | self.dbg_prfs_other_ds_.append(prfs_other) 429 | print('out-of-dataset acc: %.2f' % od_acc) 430 | 431 | # save paramters from last 100 iterations 432 | if i_epoch > (self.max_epochs - 100): 433 | print('Param pool!') 434 | param_pool = self.get_param_pool() 435 | self.params_from_last_iters.append(param_pool) 436 | 437 | total_mins = (time.time() - start_time) / 60 438 | hs, mins = divmod(total_mins, 60) 439 | print("Final duration: %i hours and %i minutes" % (hs, mins)) 440 | 441 | return self 442 | 443 | def predict(self, X): 444 | X_test_s = theano.shared(value=np.float32(X), name='X_test_s', borrow=True) 445 | 446 | givens_te = { 447 | self.input_taskdata: X_test_s 448 | } 449 | 450 | f_test = theano.function( 451 | [], 452 | [self.y_pred], 453 | givens=givens_te) 454 | predictions = f_test() 455 | del X_test_s 456 | del givens_te 457 | return predictions[0] 458 | 459 | def score(self, X, y, return_prfs=False): 460 | pred_y = self.predict(X) 461 | acc = np.mean(pred_y == y) 462 | prfs = precision_recall_fscore_support(pred_y, y) 463 | if return_prfs: 464 | return acc, prfs 465 | else: 466 | return acc 467 | 468 | 469 | ############################################################################## 470 | # plot figures 471 | ############################################################################## 472 | 473 | def dump_comps(masker, compressor, components, threshold=2): 474 | from scipy.stats import zscore 475 | from nilearn.plotting import plot_stat_map 476 | 477 | if isinstance(compressor, basestring): 478 | comp_name = compressor 479 | else: 480 | comp_name = compressor.__str__().split('(')[0] 481 | 482 | for i_c, comp in enumerate(components): 483 | path_mask = op.join(WRITE_DIR, '%s_%i-%i' % (comp_name, 484 | n_comp, i_c + 1)) 485 | nii_raw = masker.inverse_transform(comp) 486 | nii_raw.to_filename(path_mask + '.nii.gz') 487 | 488 | nii_z = masker.inverse_transform(zscore(comp)) 489 | gz_path = path_mask + '_zmap.nii.gz' 490 | nii_z.to_filename(gz_path) 491 | plot_stat_map(gz_path, bg_img='colin.nii', threshold=threshold, 492 | cut_coords=(0, -2, 0), draw_cross=False, 493 | output_file=path_mask + 'zmap.png') 494 | 495 | n_comps = [20] 496 | # n_comps = [40, 30, 20, 10, 5] 497 | for n_comp in n_comps: 498 | # for lambda_param in [0]: 499 | for lambda_param in [0.5]: 500 | l1 = 0.1 501 | l2 = 0.1 502 | my_title = r'Low-rank LR + AE (combined loss, shared decomp): n_comp=%i L1=%.1f L2=%.1f lambda=%.2f res=3mm spca20RS max%i r%i t%i' % ( 503 | n_comp, l1, l2, lambda_param, 504 | MAX_SAMPLES, 505 | MAX_REST_SAMPLES, 506 | MAX_TASK_SAMPLES 507 | ) 508 | print(my_title) 509 | estimator = SSEncoder( 510 | n_hidden=n_comp, 511 | gain1=0.004, # empirically determined by CV 512 | learning_rate = np.float32(0.00001), # empirically determined by CV, 513 | max_epochs=500, l1=l1, l2=l2, lambda_param=lambda_param) 514 | 515 | estimator.fit(X_rest, X_task, labels) 516 | 517 | fname = my_title.replace(' ', '_').replace('+', '').replace(':', '').replace('__', '_').replace('%', '') 518 | cur_path = op.join(WRITE_DIR, fname) 519 | joblib.dump(estimator, cur_path) 520 | # estimator = joblib.load(cur_path) 521 | # plt.savefig(cur_path + '_SUMMARY.png', dpi=200) 522 | 523 | # dump data also as numpy array 524 | np.save(cur_path + 'dbg_epochs_', np.array(estimator.dbg_epochs_)) 525 | np.save(cur_path + 'dbg_acc_train_', np.array(estimator.dbg_acc_train_)) 526 | np.save(cur_path + 'dbg_acc_val_', np.array(estimator.dbg_acc_val_)) 527 | np.save(cur_path + 'dbg_ae_cost_', np.array(estimator.dbg_ae_cost_)) 528 | np.save(cur_path + 'dbg_lr_cost_', np.array(estimator.dbg_lr_cost_)) 529 | np.save(cur_path + 'dbg_ae_nonimprovesteps', np.array(estimator.dbg_ae_nonimprovesteps)) 530 | np.save(cur_path + 'dbg_acc_other_ds_', np.array(estimator.dbg_acc_other_ds_)) 531 | np.save(cur_path + 'dbg_combined_cost_', np.array(estimator.dbg_combined_cost_)) 532 | np.save(cur_path + 'dbg_prfs_', np.array(estimator.dbg_prfs_)) 533 | np.save(cur_path + 'dbg_prfs_other_ds_', np.array(estimator.dbg_prfs_other_ds_)) 534 | 535 | W0_mat = estimator.W0s.get_value().T 536 | np.save(cur_path + 'W0comps', W0_mat) 537 | 538 | V1_mat = estimator.V1s.get_value().T 539 | np.save(cur_path + 'V1comps', V1_mat) 540 | # dump_comps(nifti_masker, fname, comps, threshold=0.5) 541 | 542 | STOP_CALCULATION 543 | 544 | # equally scaled plots 545 | import re 546 | pkgs = glob.glob(RES_NAME + '/*dbg_epochs*.npy') 547 | dbg_epochs_ = np.load(pkgs[0]) 548 | dbg_epochs_ = np.load(pkgs[0]) 549 | 550 | d = { 551 | 'training accuracy': '/*dbg_acc_train*.npy', 552 | 'accuracy val': '/*dbg_acc_val_*.npy', 553 | 'accuracy other ds': '/*dbg_acc_other_ds_*.npy', 554 | # 'loss ae': '/*dbg_ae_cost_*.npy', 555 | # 'loss lr': '/*dbg_lr_cost_*.npy', 556 | # 'loss combined': '/*dbg_combined_cost_*.npy' 557 | } 558 | 559 | path_main = 'nips3mm_scarcity' 560 | 561 | for k, v in d.iteritems(): 562 | pkgs = glob.glob(RES_NAME + v) 563 | for n_comp in n_comps: 564 | plt.figure() 565 | for p in pkgs: 566 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 567 | # n_hidden = int(re.search('comp=(?P.{1,2,3})_', p).group('comp')) 568 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 569 | max_samples = int(re.search('max(.{1,4})_', p).group(1)) 570 | rest_samples = int(re.search('_r(.{2,4})_', p).group(1)) 571 | task_samples = int(re.search('_t(.{2,4})dbg', p).group(1)) 572 | if n_comp != n_hidden: 573 | continue 574 | 575 | if (task_samples == 1000): 576 | continue 577 | 578 | cur_values = np.load(p) 579 | 580 | cur_label = 'n_comp=%i' % n_hidden 581 | cur_label += '/' 582 | cur_label += 'lambda=%.2f' % lambda_param 583 | cur_label += '/' 584 | cur_label += 'max=%i' % max_samples 585 | cur_label += '/' 586 | cur_label += 'nrest%i' % rest_samples 587 | cur_label += '/' 588 | cur_label += 'ntask=%i' % task_samples 589 | plt.plot( 590 | dbg_epochs_, 591 | cur_values, 592 | label=cur_label) 593 | # if k == 'training accuracy' or k == 'accuracy val': 594 | # van_pkgs = glob.glob(path_main + v) 595 | # vanilla_values = np.load(van_pkgs[0]) 596 | # plt.plot( 597 | # dbg_epochs_, 598 | # vanilla_values, 599 | # label='LR') 600 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss') 601 | plt.legend(loc='lower right', fontsize=9) 602 | plt.yticks(np.linspace(0., 1., 11)) 603 | plt.ylabel(k) 604 | plt.xlabel('epochs') 605 | plt.ylim(0., 1.05) 606 | plt.grid(True) 607 | plt.show() 608 | plt.savefig(op.join(WRITE_DIR, 609 | k.replace(' ', '_') + '_%icomps.png' % n_comp)) 610 | 611 | n_comps = [20] 612 | pkgs = glob.glob(RES_NAME + '/*dbg_acc_val_*.npy') 613 | for n_comp in n_comps: # 614 | plt.figure() 615 | for p in pkgs: 616 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 617 | # n_hidden = int(re.search('comp=(?P.{1,2,3})_', p).group('comp')) 618 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 619 | max_samples = int(re.search('max(.{1,4})_', p).group(1)) 620 | rest_samples = int(re.search('_r(.{2,4})_', p).group(1)) 621 | task_samples = int(re.search('_t(.{2,4})dbg', p).group(1)) 622 | 623 | if (task_samples == 1000): 624 | continue 625 | 626 | if n_comp != n_hidden: 627 | continue 628 | 629 | dbg_acc_val_ = np.load(p) 630 | 631 | cur_label = 'n_comp=%i' % n_hidden 632 | cur_label += '/' 633 | cur_label += 'lambda=%.2f' % lambda_param 634 | cur_label += '/' 635 | cur_label += 'max=%i' % max_samples 636 | cur_label += '/' 637 | cur_label += 'nrest%i' % rest_samples 638 | cur_label += '/' 639 | cur_label += 'ntask=%i' % task_samples 640 | plt.plot( 641 | dbg_epochs_, 642 | dbg_acc_val_, 643 | label=cur_label) 644 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss') 645 | plt.legend(loc='lower right', fontsize=9) 646 | plt.yticks(np.linspace(0., 1., 11)) 647 | plt.ylabel('validation set accuracy') 648 | plt.ylim(0., 1.05) 649 | plt.xlabel('epochs') 650 | plt.grid(True) 651 | plt.show() 652 | plt.savefig(op.join(WRITE_DIR, 'accuracy_val_%icomps.png' % n_comp)) 653 | 654 | pkgs = glob.glob(RES_NAME + '/*dbg_acc_other_ds_*.npy') 655 | for n_comp in n_comps: # 656 | plt.figure() 657 | for p in pkgs: 658 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 659 | # n_hidden = int(re.search('comp=(?P.{1,2,3})_', p).group('comp')) 660 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 661 | max_samples = int(re.search('max(.{1,4})_', p).group(1)) 662 | rest_samples = int(re.search('_r(.{4})_', p).group(1)) 663 | task_samples = int(re.search('_t(.{2,4})dbg', p).group(1)) 664 | 665 | if n_comp != n_hidden: 666 | continue 667 | 668 | dbg_acc_val_ = np.load(p) 669 | 670 | cur_label = 'n_comp=%i' % n_hidden 671 | cur_label += '/' 672 | cur_label += 'lambda=%.2f' % lambda_param 673 | cur_label += '/' 674 | cur_label += 'max=%i' % max_samples 675 | cur_label += '/' 676 | cur_label += 'nrest%i' % rest_samples 677 | cur_label += '/' 678 | cur_label += 'ntask=%i' % task_samples 679 | cur_label += '/' 680 | plt.plot( 681 | dbg_epochs_, 682 | dbg_acc_other_ds_, 683 | label=cur_label) 684 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss') 685 | plt.legend(loc='lower right', fontsize=9) 686 | plt.yticks(np.linspace(0., 1., 11)) 687 | plt.ylabel('ARCHI dataset accuracy') 688 | plt.ylim(0., 1.05) 689 | plt.xlabel('epochs') 690 | plt.grid(True) 691 | plt.show() 692 | plt.savefig(op.join(WRITE_DIR, 'accuracy_archi_%icomps.png' % n_comp)) 693 | 694 | pkgs = glob.glob(RES_NAME + '/*dbg_ae_cost_*.npy') 695 | for n_comp in n_comps: # AE 696 | plt.figure() 697 | for p in pkgs: 698 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 699 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 700 | if n_comp != n_hidden: 701 | continue 702 | 703 | dbg_ae_cost_ = np.load(p) 704 | 705 | cur_label = 'n_comp=%i' % n_hidden 706 | cur_label += '/' 707 | cur_label += 'lambda=%.2f' % lambda_param 708 | cur_label += '/' 709 | if not '_AE' in p: 710 | cur_label += 'LR only!' 711 | elif 'subRS' in p: 712 | cur_label += 'RSnormal' 713 | elif 'pca20RS' in p: 714 | cur_label += 'RSpca20' 715 | cur_label += '/' 716 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 717 | plt.plot( 718 | dbg_epochs_, 719 | dbg_ae_cost_, 720 | label=cur_label) 721 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss') 722 | plt.legend(loc='lower right', fontsize=9) 723 | # plt.yticks(np.linspace(0., 1., 11)) 724 | plt.ylabel('AE loss') 725 | plt.xlabel('epochs') 726 | plt.grid(True) 727 | plt.show() 728 | plt.savefig(op.join(WRITE_DIR, 'loss_ae_%icomps.png' % n_comp)) 729 | 730 | pkgs = glob.glob(RES_NAME + '/*dbg_lr_cost_*.npy') # LR cost 731 | for n_comp in n_comps: # AE 732 | plt.figure() 733 | for p in pkgs: 734 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 735 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 736 | if n_comp != n_hidden: 737 | continue 738 | 739 | dbg_lr_cost_ = np.load(p) 740 | 741 | cur_label = 'n_comp=%i' % n_hidden 742 | cur_label += '/' 743 | cur_label += 'lambda=%.2f' % lambda_param 744 | cur_label += '/' 745 | if not '_AE' in p: 746 | cur_label += 'LR only!' 747 | elif 'subRS' in p: 748 | cur_label += 'RSnormal' 749 | elif 'pca20RS' in p: 750 | cur_label += 'RSpca20' 751 | cur_label += '/' 752 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 753 | plt.plot( 754 | dbg_epochs_, 755 | dbg_lr_cost_, 756 | label=cur_label) 757 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss') 758 | plt.legend(loc='lower right', fontsize=9) 759 | # plt.yticks(np.linspace(0., 1., 11)) 760 | plt.ylabel('LR loss') 761 | plt.xlabel('epochs') 762 | plt.grid(True) 763 | plt.show() 764 | plt.savefig(op.join(WRITE_DIR, 'loss_lr_%icomps.png' % n_comp)) 765 | 766 | pkgs = glob.glob(RES_NAME + '/*dbg_combined_cost_*.npy') # combined loss 767 | for n_comp in n_comps: # AE 768 | plt.figure() 769 | for p in pkgs: 770 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 771 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 772 | if n_comp != n_hidden: 773 | continue 774 | 775 | dbg_combined_cost_ = np.load(p) 776 | 777 | cur_label = 'n_comp=%i' % n_hidden 778 | cur_label += '/' 779 | cur_label += 'lambda=%.2f' % lambda_param 780 | cur_label += '/' 781 | if not '_AE' in p: 782 | cur_label += 'LR only!' 783 | elif 'subRS' in p: 784 | cur_label += 'RSnormal' 785 | elif 'pca20RS' in p: 786 | cur_label += 'RSpca20' 787 | cur_label += '/' 788 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 789 | plt.plot( 790 | dbg_epochs_, 791 | dbg_combined_cost_, 792 | label=cur_label) 793 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss') 794 | plt.legend(loc='lower right', fontsize=9) 795 | # plt.yticks(np.linspace(0., 1., 11)) 796 | plt.ylabel('combined loss') 797 | plt.xlabel('epochs') 798 | plt.grid(True) 799 | plt.show() 800 | plt.savefig(op.join(WRITE_DIR, 'loss_combined_%icomps.png' % n_comp)) 801 | 802 | # precision / recall / f1 803 | target_lambda = 0.5 804 | 805 | pkgs = glob.glob(RES_NAME + '/*lambda=%.2f*dbg_prfs_.npy' % target_lambda) 806 | for n_comp in n_comps: 807 | plt.figure() 808 | for p in pkgs: 809 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 810 | n_hidden = int(re.search('comp=(?P.{1,3})_', p).group('comp')) 811 | if n_comp != n_hidden: 812 | continue 813 | 814 | dbg_prfs_ = np.load(p) 815 | 816 | cur_label = 'n_comp=%i' % n_hidden 817 | cur_label += '/' 818 | cur_label += 'lambda=%.2f' % lambda_param 819 | cur_label += '/' 820 | if not '_AE' in p: 821 | cur_label += 'LR only!' 822 | elif 'subRS' in p: 823 | cur_label += 'RSnormal' 824 | elif 'pca20RS' in p: 825 | cur_label += 'RSpca20' 826 | cur_label += '/' 827 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 828 | for i in np.arange(18): 829 | plt.plot( 830 | dbg_epochs_, 831 | np.array(dbg_prfs_)[:, 0, i], 832 | label='task %i' % (i + 1)) 833 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss lambda=%.2f' % 834 | target_lambda) 835 | plt.legend(loc='lower right', fontsize=9) 836 | # plt.yticks(np.linspace(0., 1., 11)) 837 | plt.ylabel('in-dataset precisions') 838 | plt.ylim(0., 1.05) 839 | plt.xlabel('epochs') 840 | plt.grid(True) 841 | plt.show() 842 | plt.savefig(op.join(WRITE_DIR, 'prec_inds_lambda=%0.2f_%icomps.png' % 843 | (target_lambda, n_comp))) 844 | 845 | # in-dataset recall at lambda=0.5 846 | pkgs = glob.glob(RES_NAME + '/*lambda=%.2f*dbg_prfs_.npy' % target_lambda) 847 | for n_comp in n_comps: 848 | plt.figure() 849 | for p in pkgs: 850 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 851 | n_hidden = int(re.search('comp=(?P.{1,3})_', p).group('comp')) 852 | if n_comp != n_hidden: 853 | continue 854 | 855 | dbg_prfs_ = np.load(p) 856 | 857 | dbg_prfs_ = np.load(p) 858 | cur_label = 'n_comp=%i' % n_hidden 859 | cur_label += '/' 860 | cur_label += 'lambda=%.2f' % lambda_param 861 | cur_label += '/' 862 | if not '_AE' in p: 863 | cur_label += 'LR only!' 864 | elif 'subRS' in p: 865 | cur_label += 'RSnormal' 866 | elif 'pca20RS' in p: 867 | cur_label += 'RSpca20' 868 | cur_label += '/' 869 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 870 | for i in np.arange(18): 871 | plt.plot( 872 | dbg_epochs_, 873 | np.array(dbg_prfs_)[:, 1, i], 874 | label='task %i' % (i + 1)) 875 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss lambda=%.2f' % 876 | target_lambda) 877 | plt.legend(loc='lower right', fontsize=9) 878 | # plt.yticks(np.linspace(0., 1., 11)) 879 | plt.ylabel('in-dataset recall') 880 | plt.ylim(0., 1.05) 881 | plt.xlabel('epochs') 882 | plt.grid(True) 883 | plt.show() 884 | plt.savefig(op.join(WRITE_DIR, 'rec_inds_lambda=%0.2f_%icomps.png' % 885 | (target_lambda, n_comp))) 886 | 887 | # in-dataset f1 at lambda=0.5 888 | pkgs = glob.glob(RES_NAME + '/*lambda=%.2f*dbg_prfs_.npy' % target_lambda) 889 | for n_comp in n_comps: 890 | plt.figure() 891 | for p in pkgs: 892 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 893 | n_hidden = int(re.search('comp=(?P.{1,3})_', p).group('comp')) 894 | if n_comp != n_hidden: 895 | continue 896 | 897 | dbg_prfs_ = np.load(p) 898 | 899 | cur_label = 'n_comp=%i' % n_hidden 900 | cur_label += '/' 901 | cur_label += 'lambda=%.2f' % lambda_param 902 | cur_label += '/' 903 | if not '_AE' in p: 904 | cur_label += 'LR only!' 905 | elif 'subRS' in p: 906 | cur_label += 'RSnormal' 907 | elif 'pca20RS' in p: 908 | cur_label += 'RSpca20' 909 | cur_label += '/' 910 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 911 | for i in np.arange(18): 912 | plt.plot( 913 | dbg_epochs_, 914 | np.array(dbg_prfs_)[:, 2, i], 915 | label='task %i' % (i + 1)) 916 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss lambda=%.2f' % 917 | target_lambda) 918 | plt.legend(loc='lower right', fontsize=9) 919 | # plt.yticks(np.linspace(0., 1., 11)) 920 | plt.ylabel('in-dataset f1 score') 921 | plt.ylim(0., 1.05) 922 | plt.xlabel('epochs') 923 | plt.grid(True) 924 | plt.show() 925 | plt.savefig(op.join(WRITE_DIR, 'f1_inds_lambda=%0.2f_%icomps.png' % 926 | (target_lambda, n_comp))) 927 | 928 | # out-of-dataset precision at lambda=0.5 929 | pkgs = glob.glob(RES_NAME + '/*lambda=%.2f*dbg_prfs_other_ds_.npy' % target_lambda) 930 | for n_comp in n_comps: 931 | plt.figure() 932 | for p in pkgs: 933 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 934 | n_hidden = int(re.search('comp=(?P.{1,3})_', p).group('comp')) 935 | if n_comp != n_hidden: 936 | continue 937 | 938 | dbg_prfs_other_ds_ = np.load(p) 939 | 940 | cur_label = 'n_comp=%i' % n_hidden 941 | cur_label += '/' 942 | cur_label += 'lambda=%.2f' % lambda_param 943 | cur_label += '/' 944 | if not '_AE' in p: 945 | cur_label += 'LR only!' 946 | elif 'subRS' in p: 947 | cur_label += 'RSnormal' 948 | elif 'pca20RS' in p: 949 | cur_label += 'RSpca20' 950 | cur_label += '/' 951 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 952 | for i in np.arange(18): 953 | plt.plot( 954 | dbg_epochs_, 955 | np.array(dbg_prfs_other_ds_)[:, 0, i], 956 | label='task %i' % (i + 1)) 957 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss lambda=%.2f' % 958 | target_lambda) 959 | plt.legend(loc='lower right', fontsize=9) 960 | # plt.yticks(np.linspace(0., 1., 11)) 961 | plt.ylabel('out-of-dataset precisions') 962 | plt.ylim(0., 1.05) 963 | plt.xlabel('epochs') 964 | plt.grid(True) 965 | plt.show() 966 | plt.savefig(op.join(WRITE_DIR, 'prec_oods_lambda=%0.2f_%icomps.png' % 967 | (target_lambda, n_comp))) 968 | 969 | # out-of-dataset recall at lambda=0.5 970 | pkgs = glob.glob(RES_NAME + '/*lambda=%.2f*dbg_prfs_other_ds_.npy' % target_lambda) 971 | for n_comp in n_comps: 972 | plt.figure() 973 | for p in pkgs: 974 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 975 | n_hidden = int(re.search('comp=(?P.{1,3})_', p).group('comp')) 976 | if n_comp != n_hidden: 977 | continue 978 | 979 | dbg_prfs_other_ds_ = np.load(p) 980 | 981 | cur_label = 'n_comp=%i' % n_hidden 982 | cur_label += '/' 983 | cur_label += 'lambda=%.2f' % lambda_param 984 | cur_label += '/' 985 | if not '_AE' in p: 986 | cur_label += 'LR only!' 987 | elif 'subRS' in p: 988 | cur_label += 'RSnormal' 989 | elif 'pca20RS' in p: 990 | cur_label += 'RSpca20' 991 | cur_label += '/' 992 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 993 | for i in np.arange(18): 994 | plt.plot( 995 | dbg_epochs_, 996 | np.array(dbg_prfs_other_ds_)[:, 1, i], 997 | label='task %i' % (i + 1)) 998 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss lambda=%.2f' % 999 | target_lambda) 1000 | plt.legend(loc='lower right', fontsize=9) 1001 | # plt.yticks(np.linspace(0., 1., 11)) 1002 | plt.ylabel('out-of-dataset recall') 1003 | plt.ylim(0., 1.05) 1004 | plt.xlabel('epochs') 1005 | plt.grid(True) 1006 | plt.show() 1007 | plt.savefig(op.join(WRITE_DIR, 'rec_oods_lambda=%0.2f_%icomps.png' % 1008 | (target_lambda, n_comp))) 1009 | 1010 | # out-of-dataset f1 at lambda=0.5 1011 | pkgs = glob.glob(RES_NAME + '/*lambda=%.2f*dbg_prfs_other_ds_.npy' % target_lambda) 1012 | for n_comp in n_comps: 1013 | plt.figure() 1014 | for p in pkgs: 1015 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 1016 | n_hidden = int(re.search('comp=(?P.{1,3})_', p).group('comp')) 1017 | if n_comp != n_hidden: 1018 | continue 1019 | 1020 | dbg_prfs_other_ds_ = np.load(p) 1021 | 1022 | cur_label = 'n_comp=%i' % n_hidden 1023 | cur_label += '/' 1024 | cur_label += 'lambda=%.2f' % lambda_param 1025 | cur_label += '/' 1026 | if not '_AE' in p: 1027 | cur_label += 'LR only!' 1028 | elif 'subRS' in p: 1029 | cur_label += 'RSnormal' 1030 | elif 'pca20RS' in p: 1031 | cur_label += 'RSpca20' 1032 | cur_label += '/' 1033 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 1034 | for i in np.arange(18): 1035 | plt.plot( 1036 | dbg_epochs_, 1037 | np.array(dbg_prfs_other_ds_)[:, 2, i], 1038 | label='task %i' % (i + 1)) 1039 | plt.title('Low-rank LR+AE L1=0.1 L2=0.1 res=3mm combined-loss lambda=%.2f' % 1040 | target_lambda) 1041 | plt.legend(loc='lower right', fontsize=9) 1042 | # plt.yticks(np.linspace(0., 1., 11)) 1043 | plt.ylabel('out-of-dataset f1 score') 1044 | plt.ylim(0., 1.05) 1045 | plt.xlabel('epochs') 1046 | plt.grid(True) 1047 | plt.show() 1048 | plt.savefig(op.join(WRITE_DIR, 'f1_oods_lambda=%0.2f_%icomps.png' % 1049 | (target_lambda, n_comp))) 1050 | 1051 | # print network components (1st layer) 1052 | pkgs = glob.glob(RES_NAME + '/*W0comps.npy') 1053 | for p in pkgs: 1054 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 1055 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 1056 | if n_comp != n_hidden: 1057 | continue 1058 | 1059 | new_fname = 'comps_n=%i_lambda=%.2f_th0.5' % (n_hidden, lambda_param) 1060 | comps = np.load(p) 1061 | dump_comps(nifti_masker, new_fname, comps, threshold=0.5) 1062 | 1063 | pkgs = glob.glob(RES_NAME + '/*comps*.npy') 1064 | for p in pkgs: 1065 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 1066 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 1067 | if 'V1' in p: 1068 | continue 1069 | if n_comp != n_hidden: 1070 | continue 1071 | print p 1072 | 1073 | new_fname = 'comps_n=%i_lambda=%.2f_th0.0' % (n_hidden, lambda_param) 1074 | comps = np.load(p) 1075 | dump_comps(nifti_masker, new_fname, comps, threshold=0.0) 1076 | 1077 | # print LR decision matrix (2nd layer) 1078 | pkgs = glob.glob(RES_NAME + '/*V1comps.npy') 1079 | for p in pkgs: 1080 | lambda_param = np.float(re.search('lambda=(.{4})', p).group(1)) 1081 | n_hidden = int(re.search('comp=(.{1,3})_', p).group(1)) 1082 | if n_comp != n_hidden: 1083 | continue 1084 | print p 1085 | 1086 | cur_mat = np.load(p) 1087 | 1088 | if n_comp == 20: 1089 | fs = (8, 6) 1090 | elif n_comp == 100: 1091 | fs = (12, 8) 1092 | 1093 | 1094 | plt.figure(figsize=fs) 1095 | masked_data = np.ma.masked_where(cur_mat != 0., cur_mat) 1096 | plt.imshow(masked_data, interpolation='nearest', cmap=plt.cm.gray_r) 1097 | masked_data = np.ma.masked_where(cur_mat == 0., cur_mat) 1098 | plt.imshow(masked_data, interpolation='nearest', cmap=plt.cm.RdBu_r) 1099 | plt.show() 1100 | 1101 | # plt.xticks(range(n_comp)[::5], (np.arange(n_comp) + 1)[::5]) 1102 | # plt.xlabel('hidden components') 1103 | # plt.yticks(range(18), np.arange(18) + 1) 1104 | # plt.ylabel('tasks') 1105 | # plt.title('Linear combinations of component per task') 1106 | # plt.colorbar() 1107 | 1108 | new_fname = 'comps_n=%i_lambda=%.2f_V1_net2task.png' % (n_hidden, lambda_param) 1109 | plt.savefig(op.join(WRITE_DIR, new_fname)) 1110 | 1111 | 1112 | # out-of-dataset f1 score summary plots 1113 | for n_comp in [20]: # [5, 20, 50, 100]: 1114 | f1_mean_per_lambda = list() 1115 | f1_std_per_lambda = list() 1116 | lambs = [0.25, 0.5, 0.75, 1.0] 1117 | for target_lambda in lambs: 1118 | pkgs = glob.glob(op.join(RES_NAME, '*n_comp=%i*lambda=%.2f*dbg_prfs_other_ds_.npy' % 1119 | (n_comp, target_lambda))) 1120 | print pkgs 1121 | dbg_prfs_other_ds_ = np.load(pkgs[0]) 1122 | cur_mean = np.mean(dbg_prfs_other_ds_[-1, 2, :]) 1123 | f1_mean_per_lambda.append(cur_mean) 1124 | cur_std = np.std(dbg_prfs_other_ds_[-1, 2, :]) 1125 | f1_std_per_lambda.append(cur_std) 1126 | print('F1 means: %.2f +/- %.2f (SD)' % (cur_mean, cur_std)) 1127 | 1128 | f1_mean_per_lambda = np.array(f1_mean_per_lambda) 1129 | f1_std_per_lambda = np.array(f1_std_per_lambda) 1130 | 1131 | plt.figure() 1132 | ind = np.arange(4) 1133 | width = 1. 1134 | colors = [ 1135 | (7., 176., 242.), (7., 136., 217.), (7., 40., 164.), (1., 4., 64.)] 1136 | my_colors = [(x/256, y/256, z/256) for x, y, z in colors] 1137 | plt.bar(ind, f1_mean_per_lambda, yerr=f1_std_per_lambda, 1138 | width=width, color=my_colors) 1139 | plt.ylabel('mean F1 score (+/- SD)') 1140 | plt.title('out-of-dataset performance\n' 1141 | '%i components' % n_comp) 1142 | tick_strs = [u'low-rank $\lambda=%.2f$' % val for val in lambs] 1143 | plt.xticks(ind + width / 2., tick_strs, rotation=320) 1144 | plt.ylim(.5, 1.0) 1145 | plt.yticks(np.linspace(0.5, 1., 6), np.linspace(0.5, 1., 6)) 1146 | plt.tight_layout() 1147 | out_path2 = op.join(WRITE_DIR, 'f1_bars_comp=%i.png' % n_comp) 1148 | plt.savefig(out_path2) 1149 | -------------------------------------------------------------------------------- /nips3mm_serial.py: -------------------------------------------------------------------------------- 1 | """ 2 | HCP: Semi-supervised network decomposition by low-rank logistic regression 3 | """ 4 | 5 | print __doc__ 6 | 7 | import os 8 | import os.path as op 9 | import numpy as np 10 | import glob 11 | from scipy.linalg import norm 12 | import nibabel as nib 13 | from sklearn.grid_search import RandomizedSearchCV 14 | from sklearn.base import BaseEstimator 15 | from sklearn.preprocessing import StandardScaler 16 | from nilearn.input_data import NiftiMasker 17 | from sklearn.metrics import precision_recall_fscore_support 18 | from sklearn.metrics import confusion_matrix 19 | import theano 20 | import theano.tensor as T 21 | from matplotlib import pylab as plt 22 | print('Running THEANO on %s' % theano.config.device) 23 | from nilearn.image import concat_imgs 24 | import joblib 25 | import time 26 | 27 | RES_NAME = 'nips3mm_serial' 28 | WRITE_DIR = op.join(os.getcwd(), RES_NAME) 29 | if not op.exists(WRITE_DIR): 30 | os.mkdir(WRITE_DIR) 31 | 32 | ############################################################################## 33 | # load+preprocess data 34 | ############################################################################## 35 | 36 | mask_img = 'grey10_icbm_3mm_bin.nii.gz' 37 | nifti_masker = NiftiMasker(mask_img=mask_img, smoothing_fwhm=False, 38 | standardize=False) 39 | nifti_masker.fit() 40 | mask_nvox = nifti_masker.mask_img_.get_data().sum() 41 | 42 | print('Loading data...') 43 | X_task, labels = joblib.load('preload_HT_3mm') 44 | 45 | labels = np.int32(labels) 46 | 47 | # contrasts are IN ORDER -> shuffle! 48 | new_inds = np.arange(0, X_task.shape[0]) 49 | np.random.shuffle(new_inds) 50 | X_task = X_task[new_inds] 51 | y = labels[new_inds] 52 | 53 | from sklearn.cross_validation import StratifiedShuffleSplit 54 | folder = StratifiedShuffleSplit(y, n_iter=1, test_size=0.50) 55 | new_devs, inds_val = iter(folder).next() 56 | X_dev, X_val = X_task[new_devs], X_task[inds_val] 57 | y_dev, y_val = y[new_devs], y[inds_val] 58 | 59 | del X_task 60 | 61 | ############################################################################## 62 | # class defintion 63 | ############################################################################## 64 | 65 | class AutoEncoder(BaseEstimator): 66 | def __init__(self, n_hidden, gain1, learning_rate, max_epochs=500): 67 | self.n_hidden = n_hidden 68 | self.gain1 = gain1 69 | self.max_epochs = max_epochs 70 | self.learning_rate = np.float32(learning_rate) 71 | 72 | # def rectify(X): 73 | # return T.maximum(0., X) 74 | 75 | from theano.tensor.shared_randomstreams import RandomStreams 76 | 77 | def RMSprop(self, cost, params, lr=0.001, rho=0.9, epsilon=1e-6): 78 | grads = T.grad(cost=cost, wrt=params) 79 | updates = [] 80 | for p, g in zip(params, grads): 81 | acc = theano.shared(p.get_value() * 0.) 82 | acc_new = rho * acc + (1 - rho) * g ** 2 83 | gradient_scaling = T.sqrt(acc_new + epsilon) 84 | g = g / gradient_scaling 85 | updates.append((acc, acc_new)) 86 | updates.append((p, p - lr * g)) 87 | return updates 88 | 89 | def get_param_pool(self): 90 | cur_params = ( 91 | self.W0s, self.W1s, self.bW0s, self.bW1s 92 | ) 93 | return cur_params 94 | 95 | 96 | def fit(self, X_rest): 97 | DEBUG_FLAG = True 98 | 99 | # self.max_epochs = 333 100 | self.batch_size = 100 101 | n_input = X_rest.shape[1] # sklearn-like structure 102 | n_output = n_input 103 | rng = np.random.RandomState(42) 104 | self.params_from_last_iters = [] 105 | 106 | self.input_restdata = T.matrix(dtype='float32', name='input_restdata') 107 | 108 | index = T.iscalar(name='index') 109 | 110 | # prepare data for theano computation 111 | if DEBUG_FLAG: 112 | self.dbg_ae_cost_ = list() 113 | self.dbg_ae_nonimprovesteps = list() 114 | X_rest_s = theano.shared(value=np.float32(X_rest), name='X_rest_s') 115 | ae_train_samples = len(X_rest) 116 | 117 | # W -> unsupervised / auto-encoder 118 | 119 | # computational graph: auto-encoder 120 | W0_vals = rng.randn(n_input, self.n_hidden).astype(np.float32) * self.gain1 121 | 122 | self.W0s = theano.shared(W0_vals) 123 | self.W1s = self.W0s.T # tied 124 | bW0_vals = np.zeros(self.n_hidden).astype(np.float32) 125 | self.bW0s = theano.shared(value=bW0_vals, name='bW0') 126 | bW1_vals = np.zeros(n_output).astype(np.float32) 127 | self.bW1s = theano.shared(value=bW1_vals, name='bW1') 128 | 129 | givens_ae = { 130 | self.input_restdata: X_rest_s[ 131 | index * self.batch_size:(index + 1) * self.batch_size] 132 | } 133 | 134 | encoding = (self.input_restdata.dot(self.W0s) + self.bW0s).dot(self.W1s) + self.bW1s 135 | 136 | self.ae_loss = T.sum((self.input_restdata - encoding) ** 2, axis=1) 137 | 138 | self.ae_cost = ( 139 | T.mean(self.ae_loss) / n_input 140 | ) 141 | 142 | # combined loss for AE and LR 143 | ae_params = [self.W0s, self.bW0s, self.bW1s] 144 | ae_updates = self.RMSprop( 145 | cost=self.ae_cost, 146 | params=ae_params, 147 | lr=self.learning_rate) 148 | f_train_ae = theano.function( 149 | [index], 150 | [self.ae_cost], 151 | givens=givens_ae, 152 | updates=ae_updates, allow_input_downcast=True) 153 | 154 | # optimization loop 155 | start_time = time.time() 156 | ae_last_cost = np.inf 157 | no_improve_steps = 0 158 | acc_train, acc_val = 0., 0. 159 | for i_epoch in range(self.max_epochs): 160 | if i_epoch == 1: 161 | epoch_dur = time.time() - start_time 162 | total_mins = (epoch_dur * self.max_epochs) / 60 163 | hs, mins = divmod(total_mins, 60) 164 | print("Max estimated duration: %i hours and %i minutes" % (hs, mins)) 165 | 166 | # AE 167 | # if i_epoch % 2 == 0: # every second time 168 | #if False: 169 | # auto-encoder 170 | ae_n_batches = ae_train_samples // self.batch_size 171 | # for i in range(lr_n_batches): 172 | # for i in range(max(ae_n_batches, lr_n_batches)): 173 | # if i < ae_n_batches: 174 | # ae_cur_cost = float(f_train_ae(i)[0]) 175 | # ae_cur_cost = 0 176 | # if i < lr_n_batches: 177 | # lr_cur_cost = float(f_train_lr(i)[0]) 178 | # for i in range(lr_n_batches): 179 | for i in range(ae_n_batches): 180 | # lr_cur_cost = f_train_lr(i)[0] 181 | # ae_cur_cost = lr_cur_cost 182 | ae_cur_cost = f_train_ae(i)[0] 183 | 184 | # evaluate epoch cost 185 | if ae_last_cost - ae_cur_cost < 0.1: 186 | no_improve_steps += 1 187 | else: 188 | ae_last_cost = ae_cur_cost 189 | no_improve_steps = 0 190 | 191 | print('E:%i, ae_cost:%.4f, adsteps:%i' % ( 192 | i_epoch + 1, ae_cur_cost, no_improve_steps)) 193 | 194 | if (i_epoch % 10 == 0): 195 | self.dbg_ae_cost_.append(ae_cur_cost) 196 | self.dbg_ae_nonimprovesteps.append(no_improve_steps) 197 | 198 | # save paramters from last 100 iterations 199 | if i_epoch > -1: # (self.max_epochs - 100): 200 | print('Param pool!') 201 | param_pool = self.get_param_pool() 202 | self.params_from_last_iters.append(param_pool) 203 | 204 | total_mins = (time.time() - start_time) / 60 205 | hs, mins = divmod(total_mins, 60) 206 | print("Final duration: %i hours and %i minutes" % (hs, mins)) 207 | 208 | return self 209 | 210 | def transform(self, newdata): 211 | compr_matrix = self.W0s.get_value().T # currently best compression 212 | transformed_data = np.dot(compr_matrix, newdata.T).T 213 | return transformed_data 214 | 215 | 216 | class SEncoder(BaseEstimator): 217 | def __init__(self, gain1, learning_rate, max_epochs=100, 218 | l1=0.1, l2=0.1): 219 | """ 220 | Parameters 221 | ---------- 222 | lambda : float 223 | Mediates between AE and LR. lambda==1 equates with LR only. 224 | """ 225 | self.gain1 = gain1 226 | self.max_epochs = max_epochs 227 | self.learning_rate = np.float32(learning_rate) 228 | self.penalty_l1 = np.float32(l1) 229 | self.penalty_l2 = np.float32(l2) 230 | 231 | # def rectify(X): 232 | # return T.maximum(0., X) 233 | 234 | from theano.tensor.shared_randomstreams import RandomStreams 235 | 236 | def RMSprop(self, cost, params, lr=0.001, rho=0.9, epsilon=1e-6): 237 | grads = T.grad(cost=cost, wrt=params) 238 | updates = [] 239 | for p, g in zip(params, grads): 240 | acc = theano.shared(p.get_value() * 0.) 241 | acc_new = rho * acc + (1 - rho) * g ** 2 242 | gradient_scaling = T.sqrt(acc_new + epsilon) 243 | g = g / gradient_scaling 244 | updates.append((acc, acc_new)) 245 | updates.append((p, p - lr * g)) 246 | return updates 247 | 248 | def get_param_pool(self): 249 | cur_params = ( 250 | self.V0s, self.bV0 251 | ) 252 | return cur_params 253 | 254 | def fit(self, X_task, y): 255 | DEBUG_FLAG = True 256 | 257 | # self.max_epochs = 333 258 | self.batch_size = 100 259 | rng = np.random.RandomState(42) 260 | self.input_taskdata = T.matrix(dtype='float32', name='input_taskdata') 261 | self.input_restdata = T.matrix(dtype='float32', name='input_restdata') 262 | self.params_from_last_iters = [] 263 | n_input = X_task.shape[1] 264 | 265 | index = T.iscalar(name='index') 266 | 267 | # prepare data for theano computation 268 | if not DEBUG_FLAG: 269 | X_train_s = theano.shared( 270 | value=np.float32(X_task), name='X_train_s') 271 | y_train_s = theano.shared( 272 | value=np.int32(y), name='y_train_s') 273 | lr_train_samples = len(X_task) 274 | else: 275 | from sklearn.cross_validation import StratifiedShuffleSplit 276 | folder = StratifiedShuffleSplit(y, n_iter=1, test_size=0.20) 277 | new_trains, inds_val = iter(folder).next() 278 | X_train, X_val = X_task[new_trains], X_task[inds_val] 279 | y_train, y_val = y[new_trains], y[inds_val] 280 | 281 | X_train_s = theano.shared(value=np.float32(X_train), 282 | name='X_train_s', borrow=False) 283 | y_train_s = theano.shared(value=np.int32(y_train), 284 | name='y_train_s', borrow=False) 285 | # X_val_s = theano.shared(value=np.float32(X_val), 286 | # name='X_train_s', borrow=False) 287 | # y_val_s = theano.shared(value=np.int32(y_val), 288 | # name='y_cal_s', borrow=False) 289 | lr_train_samples = len(X_train) 290 | self.dbg_epochs_ = list() 291 | self.dbg_acc_train_ = list() 292 | self.dbg_acc_val_ = list() 293 | self.dbg_ae_cost_ = list() 294 | self.dbg_lr_cost_ = list() 295 | self.dbg_ae_nonimprovesteps = list() 296 | self.dbg_acc_other_ds_ = list() 297 | self.dbg_prfs_ = list() 298 | self.dbg_prfs_other_ds_ = list() 299 | 300 | # computation graph: logistic regression 301 | clf_n_output = 18 # number of labels 302 | my_y = T.ivector(name='y') 303 | 304 | bV0_vals = np.zeros(clf_n_output).astype(np.float32) 305 | self.bV0 = theano.shared(value=bV0_vals, name='bV0') 306 | 307 | V0_vals = rng.randn(n_input, clf_n_output).astype(np.float32) * self.gain1 308 | self.V0s = theano.shared(V0_vals) 309 | 310 | self.p_y_given_x = T.nnet.softmax( 311 | T.dot(self.input_taskdata, self.V0s) + self.bV0 312 | ) 313 | self.lr_cost = -T.mean(T.log(self.p_y_given_x)[T.arange(my_y.shape[0]), my_y]) 314 | self.lr_cost = ( 315 | self.lr_cost + 316 | T.mean(abs(self.V0s)) * self.penalty_l1 + 317 | T.mean(abs(self.bV0)) * self.penalty_l1 + 318 | 319 | T.mean((self.V0s ** np.float32(2))) * self.penalty_l2 + 320 | T.mean((self.bV0 ** np.float32(2))) * self.penalty_l2 321 | ) 322 | self.y_pred = T.argmax(self.p_y_given_x, axis=1) 323 | 324 | givens_lr = { 325 | self.input_taskdata: X_train_s[index * self.batch_size:(index + 1) * self.batch_size], 326 | my_y: y_train_s[index * self.batch_size:(index + 1) * self.batch_size] 327 | } 328 | 329 | params = [self.V0s, self.bV0] 330 | updates = self.RMSprop(cost=self.lr_cost, params=params, 331 | lr=self.learning_rate) 332 | 333 | f_train_lr = theano.function( 334 | [index], 335 | [self.lr_cost], 336 | givens=givens_lr, 337 | updates=updates) 338 | 339 | # optimization loop 340 | start_time = time.time() 341 | lr_last_cost = np.inf 342 | ae_cur_cost = np.inf 343 | no_improve_steps = 0 344 | acc_train, acc_val = 0., 0. 345 | for i_epoch in range(self.max_epochs): 346 | if i_epoch == 1: 347 | epoch_dur = time.time() - start_time 348 | total_mins = (epoch_dur * self.max_epochs) / 60 349 | hs, mins = divmod(total_mins, 60) 350 | print("Max estimated duration: %i hours and %i minutes" % (hs, mins)) 351 | 352 | lr_n_batches = lr_train_samples // self.batch_size 353 | for i in range(lr_n_batches): 354 | lr_cur_cost = f_train_lr(i)[0] 355 | 356 | # evaluate epoch cost 357 | if lr_last_cost - lr_cur_cost < 0.1: 358 | no_improve_steps += 1 359 | else: 360 | lr_last_cost = lr_cur_cost 361 | no_improve_steps = 0 362 | 363 | # logistic 364 | lr_last_cost = lr_cur_cost 365 | acc_train = self.score(X_train, y_train) 366 | acc_val, prfs_val = self.score(X_val, y_val, return_prfs=True) 367 | 368 | print('E:%i, ae_cost:%.4f, lr_cost:%.4f, train_score:%.2f, vald_score:%.2f, ae_badsteps:%i' % ( 369 | i_epoch + 1, ae_cur_cost, lr_cur_cost, acc_train, acc_val, no_improve_steps)) 370 | 371 | if (i_epoch % 10 == 0): 372 | self.dbg_ae_cost_.append(ae_cur_cost) 373 | self.dbg_lr_cost_.append(lr_cur_cost) 374 | 375 | self.dbg_epochs_.append(i_epoch + 1) 376 | self.dbg_ae_nonimprovesteps.append(no_improve_steps) 377 | self.dbg_acc_train_.append(acc_train) 378 | self.dbg_acc_val_.append(acc_val) 379 | self.dbg_prfs_.append(prfs_val) 380 | 381 | # if i_epoch > (self.max_epochs - 100): 382 | param_pool = self.get_param_pool() 383 | self.params_from_last_iters.append(param_pool) 384 | 385 | total_mins = (time.time() - start_time) / 60 386 | hs, mins = divmod(total_mins, 60) 387 | print("Final duration: %i hours and %i minutes" % (hs, mins)) 388 | 389 | return self 390 | 391 | def predict(self, X): 392 | X_test_s = theano.shared(value=np.float32(X), name='X_test_s', borrow=True) 393 | 394 | givens_te = { 395 | self.input_taskdata: X_test_s 396 | } 397 | 398 | f_test = theano.function( 399 | [], 400 | [self.y_pred], 401 | givens=givens_te) 402 | predictions = f_test() 403 | del X_test_s 404 | del givens_te 405 | return predictions[0] 406 | 407 | def score(self, X, y, return_prfs=False): 408 | pred_y = self.predict(X) 409 | acc = np.mean(pred_y == y) 410 | prfs = precision_recall_fscore_support(pred_y, y) 411 | if return_prfs: 412 | return acc, prfs 413 | else: 414 | return acc 415 | 416 | ############################################################################## 417 | # data processing 418 | ############################################################################## 419 | 420 | n_comps = [5, 20, 50, 100] 421 | for n_comp in n_comps: 422 | print('#' * 80) 423 | print('#components: %i' % n_comp) 424 | 425 | # 2-step approach 1: ICA + LR 426 | # print('Compressing...') 427 | # from sklearn.decomposition import FastICA 428 | # compressor = FastICA(n_components=n_comp, whiten=True) 429 | # compressor.fit(X_dev) 430 | # half2compr = compressor.transform(X_val) 431 | # 432 | # print('Classifiying...') 433 | # l1 = 0.1 434 | # l2 = 0.1 435 | # my_title = r'LR: L1=%.1f L2=%.1f res=3mm' % ( 436 | # l1, l2 437 | # ) 438 | # print(my_title) 439 | # estimator = SEncoder( 440 | # gain1=0.004, # empirically determined by CV 441 | # learning_rate = np.float32(0.00001), # empirically determined by CV, 442 | # max_epochs=500, l1=l1, l2=l2) 443 | # 444 | # estimator.fit(half2compr, y_val) 445 | # 446 | # acc_ICA = estimator.dbg_acc_val_ 447 | # print(acc_ICA) 448 | # print('ICA: %.4f' % acc_ICA[-1]) 449 | # 450 | # outpath = op.join(WRITE_DIR, 'ICA-LR_ncomp=%i' % n_comp) 451 | # np.save(outpath + '_acc', acc_ICA) 452 | # joblib.dump(estimator, outpath + '_est', compress=9) 453 | 454 | # 2-step approach: SPCA + LR 455 | # print('Compressing...') 456 | # from sklearn.decomposition import SparsePCA 457 | # compressor = SparsePCA(n_components=n_comp, alpha=1.0, # big sparsity 458 | # n_jobs=1, verbose=0, tol=0.1) 459 | # compressor.fit(X_dev) 460 | # half2compr = compressor.transform(X_val) 461 | # 462 | # print('Classifiying...') 463 | # l1 = 0.1 464 | # l2 = 0.1 465 | # my_title = r'LR: L1=%.1f L2=%.1f res=3mm' % ( 466 | # l1, l2 467 | # ) 468 | # print(my_title) 469 | # estimator = SEncoder( 470 | # gain1=0.004, # empirically determined by CV 471 | # learning_rate = np.float32(0.00001), # empirically determined by CV, 472 | # max_epochs=500, l1=l1, l2=l2) 473 | # 474 | # estimator.fit(half2compr, y_val) 475 | # 476 | # acc_SPCA = estimator.dbg_acc_val_ 477 | # print(acc_SPCA) 478 | # print('SPCA: %.4f' % acc_SPCA[-1]) 479 | # 480 | # outpath = op.join(WRITE_DIR, 'SPCA-LR_ncomp=%i' % n_comp) 481 | # np.save(outpath + '_acc', acc_SPCA) 482 | # joblib.dump(estimator, outpath + '_est', compress=9) 483 | 484 | # 2-step approach 3: AE + LR 485 | # print('Compressing by autoencoder...') 486 | # compressor = AutoEncoder( 487 | # n_hidden=n_comp, 488 | # gain1=0.004, # empirically determined by CV 489 | # learning_rate = np.float32(0.00001), # empirically determined by CV, 490 | # max_epochs=500) 491 | # compressor.fit(X_dev) 492 | # 493 | # half2compr = compressor.transform(X_val) 494 | # 495 | # print('Classifiying...') 496 | # l1 = 0.1 497 | # l2 = 0.1 498 | # my_title = r'LR: L1=%.1f L2=%.1f res=3mm' % ( 499 | # l1, l2 500 | # ) 501 | # print(my_title) 502 | # estimator = SEncoder( 503 | # gain1=0.004, # empirically determined by CV 504 | # learning_rate = np.float32(0.00001), # empirically determined by CV, 505 | # max_epochs=500, l1=l1, l2=l2) 506 | # 507 | # estimator.fit(half2compr, y_val) 508 | # 509 | # acc_AE = estimator.dbg_acc_val_ 510 | # print(acc_AE) 511 | # print('AE: %.4f' % acc_AE[-1]) 512 | # 513 | # outpath = op.join(WRITE_DIR, 'AE-LR_ncomp=%i' % n_comp) 514 | # np.save(outpath + '_acc', acc_AE) 515 | # joblib.dump(estimator, outpath + '_est', compress=9) 516 | 517 | # 2-step approach 4: PCA + LR 518 | from sklearn.decomposition import PCA 519 | print('Compressing by whitened PCA...') 520 | compressor = PCA(n_components=n_comp, whiten=True) 521 | compressor.fit(X_dev) 522 | 523 | half2compr = compressor.transform(X_val) 524 | 525 | print('Classifiying...') 526 | l1 = 0.1 527 | l2 = 0.1 528 | my_title = r'LR: L1=%.1f L2=%.1f res=3mm' % ( 529 | l1, l2 530 | ) 531 | print(my_title) 532 | estimator = SEncoder( 533 | gain1=0.004, # empirically determined by CV 534 | learning_rate = np.float32(0.00001), # empirically determined by CV, 535 | max_epochs=500, l1=l1, l2=l2) 536 | 537 | estimator.fit(half2compr, y_val) 538 | 539 | acc_PCA = estimator.dbg_acc_val_ 540 | print(acc_PCA) 541 | print('wPCA: %.4f' % acc_PCA[-1]) 542 | 543 | outpath = op.join(WRITE_DIR, 'nwPCA-LR_ncomp=%i' % n_comp) 544 | np.save(outpath + '_acc', acc_PCA) 545 | joblib.dump(estimator, outpath + '_est', compress=9) 546 | 547 | -------------------------------------------------------------------------------- /nips3mm_vanilla.py: -------------------------------------------------------------------------------- 1 | """ 2 | HCP: Semi-supervised network decomposition by low-rank logistic regression 3 | """ 4 | 5 | print __doc__ 6 | 7 | import os 8 | import os.path as op 9 | import numpy as np 10 | import glob 11 | from scipy.linalg import norm 12 | import nibabel as nib 13 | from sklearn.grid_search import RandomizedSearchCV 14 | from sklearn.base import BaseEstimator 15 | from sklearn.preprocessing import StandardScaler 16 | from nilearn.input_data import NiftiMasker 17 | from sklearn.metrics import precision_recall_fscore_support 18 | from sklearn.metrics import confusion_matrix 19 | import theano 20 | import theano.tensor as T 21 | from matplotlib import pylab as plt 22 | print('Running THEANO on %s' % theano.config.device) 23 | from nilearn.image import concat_imgs 24 | import joblib 25 | import time 26 | 27 | RES_NAME = 'nips3mm_vanilla' 28 | WRITE_DIR = op.join(os.getcwd(), RES_NAME) 29 | if not op.exists(WRITE_DIR): 30 | os.mkdir(WRITE_DIR) 31 | 32 | ############################################################################## 33 | # load+preprocess data 34 | ############################################################################## 35 | 36 | mask_img = 'grey10_icbm_3mm_bin.nii.gz' 37 | nifti_masker = NiftiMasker(mask_img=mask_img, smoothing_fwhm=False, 38 | standardize=False) 39 | nifti_masker.fit() 40 | mask_nvox = nifti_masker.mask_img_.get_data().sum() 41 | 42 | print('Loading data...') 43 | 44 | # ARCHI task 45 | X_task, labels = joblib.load('preload_HT_3mm') 46 | 47 | labels = np.int32(labels) 48 | 49 | # contrasts are IN ORDER -> shuffle! 50 | new_inds = np.arange(0, X_task.shape[0]) 51 | np.random.shuffle(new_inds) 52 | X_task = X_task[new_inds] 53 | labels = labels[new_inds] 54 | # subs = subs[new_inds] 55 | 56 | X_task = StandardScaler().fit_transform(X_task) 57 | 58 | # ARCHI task 59 | AT_niis, AT_labels, AT_subs = joblib.load('preload_AT_3mm') 60 | AT_X = nifti_masker.transform(AT_niis) 61 | AT_X = StandardScaler().fit_transform(AT_X) 62 | print('done :)') 63 | 64 | ############################################################################## 65 | # define computation graph 66 | ############################################################################## 67 | 68 | class SSEncoder(BaseEstimator): 69 | def __init__(self, gain1, learning_rate, max_epochs=100, 70 | l1=0.1, l2=0.1): 71 | """ 72 | Parameters 73 | ---------- 74 | lambda : float 75 | Mediates between AE and LR. lambda==1 equates with LR only. 76 | """ 77 | self.gain1 = gain1 78 | self.max_epochs = max_epochs 79 | self.learning_rate = np.float32(learning_rate) 80 | self.penalty_l1 = np.float32(l1) 81 | self.penalty_l2 = np.float32(l2) 82 | 83 | # def rectify(X): 84 | # return T.maximum(0., X) 85 | 86 | from theano.tensor.shared_randomstreams import RandomStreams 87 | 88 | def RMSprop(self, cost, params, lr=0.001, rho=0.9, epsilon=1e-6): 89 | grads = T.grad(cost=cost, wrt=params) 90 | updates = [] 91 | for p, g in zip(params, grads): 92 | acc = theano.shared(p.get_value() * 0.) 93 | acc_new = rho * acc + (1 - rho) * g ** 2 94 | gradient_scaling = T.sqrt(acc_new + epsilon) 95 | g = g / gradient_scaling 96 | updates.append((acc, acc_new)) 97 | updates.append((p, p - lr * g)) 98 | return updates 99 | 100 | def get_param_pool(self): 101 | cur_params = ( 102 | self.V0s, self.bV0 103 | ) 104 | return cur_params 105 | 106 | def fit(self, X_task, y): 107 | DEBUG_FLAG = True 108 | 109 | # self.max_epochs = 333 110 | self.batch_size = 100 111 | rng = np.random.RandomState(42) 112 | self.input_taskdata = T.matrix(dtype='float32', name='input_taskdata') 113 | self.input_restdata = T.matrix(dtype='float32', name='input_restdata') 114 | self.params_from_last_iters = [] 115 | n_input = X_task.shape[1] 116 | 117 | index = T.iscalar(name='index') 118 | 119 | # prepare data for theano computation 120 | if not DEBUG_FLAG: 121 | X_train_s = theano.shared( 122 | value=np.float32(X_task), name='X_train_s') 123 | y_train_s = theano.shared( 124 | value=np.int32(y), name='y_train_s') 125 | lr_train_samples = len(X_task) 126 | else: 127 | from sklearn.cross_validation import StratifiedShuffleSplit 128 | folder = StratifiedShuffleSplit(y, n_iter=1, test_size=0.20) 129 | new_trains, inds_val = iter(folder).next() 130 | X_train, X_val = X_task[new_trains], X_task[inds_val] 131 | y_train, y_val = y[new_trains], y[inds_val] 132 | 133 | X_train_s = theano.shared(value=np.float32(X_train), 134 | name='X_train_s', borrow=False) 135 | y_train_s = theano.shared(value=np.int32(y_train), 136 | name='y_train_s', borrow=False) 137 | # X_val_s = theano.shared(value=np.float32(X_val), 138 | # name='X_train_s', borrow=False) 139 | # y_val_s = theano.shared(value=np.int32(y_val), 140 | # name='y_cal_s', borrow=False) 141 | lr_train_samples = len(X_train) 142 | self.dbg_epochs_ = list() 143 | self.dbg_acc_train_ = list() 144 | self.dbg_acc_val_ = list() 145 | self.dbg_ae_cost_ = list() 146 | self.dbg_lr_cost_ = list() 147 | self.dbg_ae_nonimprovesteps = list() 148 | self.dbg_acc_other_ds_ = list() 149 | self.dbg_prfs_ = list() 150 | self.dbg_prfs_other_ds_ = list() 151 | 152 | # computation graph: logistic regression 153 | clf_n_output = 18 # number of labels 154 | my_y = T.ivector(name='y') 155 | 156 | bV0_vals = np.zeros(clf_n_output).astype(np.float32) 157 | self.bV0 = theano.shared(value=bV0_vals, name='bV0') 158 | 159 | V0_vals = rng.randn(n_input, clf_n_output).astype(np.float32) * self.gain1 160 | self.V0s = theano.shared(V0_vals) 161 | 162 | self.p_y_given_x = T.nnet.softmax( 163 | T.dot(self.input_taskdata, self.V0s) + self.bV0 164 | ) 165 | self.lr_cost = -T.mean(T.log(self.p_y_given_x)[T.arange(my_y.shape[0]), my_y]) 166 | self.lr_cost = ( 167 | self.lr_cost + 168 | T.mean(abs(self.V0s)) * self.penalty_l1 + 169 | T.mean(abs(self.bV0)) * self.penalty_l1 + 170 | 171 | T.mean((self.V0s ** np.float32(2))) * self.penalty_l2 + 172 | T.mean((self.bV0 ** np.float32(2))) * self.penalty_l2 173 | ) 174 | self.y_pred = T.argmax(self.p_y_given_x, axis=1) 175 | 176 | givens_lr = { 177 | self.input_taskdata: X_train_s[index * self.batch_size:(index + 1) * self.batch_size], 178 | my_y: y_train_s[index * self.batch_size:(index + 1) * self.batch_size] 179 | } 180 | 181 | params = [self.V0s, self.bV0] 182 | updates = self.RMSprop(cost=self.lr_cost, params=params, 183 | lr=self.learning_rate) 184 | 185 | f_train_lr = theano.function( 186 | [index], 187 | [self.lr_cost], 188 | givens=givens_lr, 189 | updates=updates) 190 | 191 | # optimization loop 192 | start_time = time.time() 193 | lr_last_cost = np.inf 194 | ae_cur_cost = np.inf 195 | no_improve_steps = 0 196 | acc_train, acc_val = 0., 0. 197 | for i_epoch in range(self.max_epochs): 198 | if i_epoch == 1: 199 | epoch_dur = time.time() - start_time 200 | total_mins = (epoch_dur * self.max_epochs) / 60 201 | hs, mins = divmod(total_mins, 60) 202 | print("Max estimated duration: %i hours and %i minutes" % (hs, mins)) 203 | 204 | lr_n_batches = lr_train_samples // self.batch_size 205 | for i in range(lr_n_batches): 206 | lr_cur_cost = f_train_lr(i)[0] 207 | 208 | # evaluate epoch cost 209 | if lr_last_cost - lr_cur_cost < 0.1: 210 | no_improve_steps += 1 211 | else: 212 | lr_last_cost = lr_cur_cost 213 | no_improve_steps = 0 214 | 215 | # logistic 216 | lr_last_cost = lr_cur_cost 217 | acc_train = self.score(X_train, y_train) 218 | acc_val, prfs_val = self.score(X_val, y_val, return_prfs=True) 219 | 220 | print('E:%i, ae_cost:%.4f, lr_cost:%.4f, train_score:%.2f, vald_score:%.2f, ae_badsteps:%i' % ( 221 | i_epoch + 1, ae_cur_cost, lr_cur_cost, acc_train, acc_val, no_improve_steps)) 222 | 223 | if (i_epoch % 10 == 0): 224 | self.dbg_ae_cost_.append(ae_cur_cost) 225 | self.dbg_lr_cost_.append(lr_cur_cost) 226 | 227 | self.dbg_epochs_.append(i_epoch + 1) 228 | self.dbg_ae_nonimprovesteps.append(no_improve_steps) 229 | self.dbg_acc_train_.append(acc_train) 230 | self.dbg_acc_val_.append(acc_val) 231 | self.dbg_prfs_.append(prfs_val) 232 | 233 | # if i_epoch > (self.max_epochs - 100): 234 | param_pool = self.get_param_pool() 235 | self.params_from_last_iters.append(param_pool) 236 | 237 | total_mins = (time.time() - start_time) / 60 238 | hs, mins = divmod(total_mins, 60) 239 | print("Final duration: %i hours and %i minutes" % (hs, mins)) 240 | 241 | return self 242 | 243 | def predict(self, X): 244 | X_test_s = theano.shared(value=np.float32(X), name='X_test_s', borrow=True) 245 | 246 | givens_te = { 247 | self.input_taskdata: X_test_s 248 | } 249 | 250 | f_test = theano.function( 251 | [], 252 | [self.y_pred], 253 | givens=givens_te) 254 | predictions = f_test() 255 | del X_test_s 256 | del givens_te 257 | return predictions[0] 258 | 259 | def score(self, X, y, return_prfs=False): 260 | pred_y = self.predict(X) 261 | acc = np.mean(pred_y == y) 262 | prfs = precision_recall_fscore_support(pred_y, y) 263 | if return_prfs: 264 | return acc, prfs 265 | else: 266 | return acc 267 | 268 | 269 | ############################################################################## 270 | # plot figures 271 | ############################################################################## 272 | 273 | def dump_comps(masker, compressor, components, threshold=2, fwhm=None, 274 | perc=None): 275 | from scipy.stats import zscore 276 | from nilearn.plotting import plot_stat_map 277 | from nilearn.image import smooth_img 278 | from scipy.stats import scoreatpercentile 279 | 280 | if isinstance(compressor, basestring): 281 | comp_name = compressor 282 | else: 283 | comp_name = compressor.__str__().split('(')[0] 284 | 285 | for i_c, comp in enumerate(components): 286 | path_mask = op.join(WRITE_DIR, '%s_%i-%i' % (comp_name, 287 | n_comp, i_c + 1)) 288 | nii_raw = masker.inverse_transform(comp) 289 | nii_raw.to_filename(path_mask + '.nii.gz') 290 | 291 | comp_z = zscore(comp) 292 | 293 | if perc is not None: 294 | cur_thresh = scoreatpercentile(np.abs(comp_z), per=perc) 295 | path_mask += '_perc%i' % perc 296 | print('Applying percentile %.2f (threshold: %.2f)' % (perc, cur_thresh)) 297 | else: 298 | cur_thresh = threshold 299 | path_mask += '_thr%.2f' % cur_thresh 300 | print('Applying threshold: %.2f' % cur_thresh) 301 | 302 | nii_z = masker.inverse_transform(comp_z) 303 | gz_path = path_mask + '_zmap.nii.gz' 304 | nii_z.to_filename(gz_path) 305 | plot_stat_map(gz_path, bg_img='colin.nii', threshold=cur_thresh, 306 | cut_coords=(0, -2, 0), draw_cross=False, 307 | output_file=path_mask + 'zmap.png') 308 | 309 | # optional: do smoothing 310 | if fwhm is not None: 311 | nii_z_fwhm = smooth_img(nii_z, fwhm=fwhm) 312 | plot_stat_map(nii_z_fwhm, bg_img='colin.nii', threshold=cur_thresh, 313 | cut_coords=(0, -2, 0), draw_cross=False, 314 | output_file=path_mask + 315 | ('zmap_%imm.png' % fwhm)) 316 | l1 = 0.1 317 | l2 = 0.1 318 | my_title = r'LR: L1=%.1f L2=%.1f res=3mm' % ( 319 | l1, l2 320 | ) 321 | print(my_title) 322 | estimator = SSEncoder( 323 | gain1=0.004, # empirically determined by CV 324 | learning_rate = np.float32(0.00001), # empirically determined by CV, 325 | max_epochs=500, l1=l1, l2=l2) 326 | 327 | estimator.fit(X_task, labels) 328 | 329 | # my_title = r'Low-rank LR: n_comp=%i L1=%.1f L2=%.1f res=10mm pca20RS' % ( 330 | # n_comp, l1, l2 331 | # ) 332 | # FONTS = 12 333 | # plt.figure(facecolor='white', figsize=(8, 6)) 334 | # plt.plot(np.log(estimator.dbg_ae_cost_), label='cost autoencoder') 335 | # plt.plot(estimator.dbg_lr_cost_, label='cost logistic') 336 | # plt.plot(estimator.dbg_acc_train_, label='score training set') 337 | # plt.plot(estimator.dbg_acc_val_, label='score validation set') 338 | # plt.plot(estimator.dbg_acc_other_ds_, label='other-datset acc') 339 | # plt.legend(loc='best', fontsize=12) 340 | # plt.xlabel('epoch', fontsize=FONTS) 341 | # plt.ylabel('misc', fontsize=FONTS) 342 | # plt.yticks(np.arange(12), np.arange(12)) 343 | # plt.grid(True) 344 | # plt.title(my_title) 345 | # plt.show() 346 | 347 | fname = my_title.replace(' ', '_').replace('+', '').replace(':', '').replace('__', '_').replace('%', '') 348 | cur_path = op.join(WRITE_DIR, fname) 349 | joblib.dump(estimator, cur_path) 350 | # plt.savefig(cur_path + '_SUMMARY.png', dpi=200) 351 | 352 | V0_mat = estimator.V0s.get_value().T 353 | np.save(cur_path + 'V0comps', V0_mat) 354 | 355 | # dump data also as numpy array 356 | np.save(cur_path + 'dbg_epochs_', np.array(estimator.dbg_epochs_)) 357 | np.save(cur_path + 'dbg_acc_train_', np.array(estimator.dbg_acc_train_)) 358 | np.save(cur_path + 'dbg_acc_val_', np.array(estimator.dbg_acc_val_)) 359 | np.save(cur_path + 'dbg_lr_cost_', np.array(estimator.dbg_lr_cost_)) 360 | np.save(cur_path + 'dbg_prfs_', np.array(estimator.dbg_prfs_)) 361 | 362 | 363 | dump_comps(nifti_masker, fname, comps, threshold=0.25) 364 | 365 | import re 366 | pkgs = glob.glob(RES_NAME + '/*dbg_epochs*.npy') 367 | dbg_epochs_ = np.load(pkgs[0]) 368 | d = { 369 | 'training accuracy': '/*dbg_acc_train*.npy', 370 | 'accuracy val': '/*dbg_acc_val_*.npy', 371 | 'loss lr': '/*dbg_lr_cost_*.npy' 372 | } 373 | for k, v in d.iteritems(): 374 | pkgs = glob.glob(RES_NAME + v) 375 | p = pkgs[0] 376 | cur_values = np.load(p) 377 | 378 | cur_label = '' 379 | if not '_AE' in p: 380 | cur_label += 'LR only!' 381 | elif 'subRS' in p: 382 | cur_label += 'RSnormal' 383 | elif 'spca20RS' in p: 384 | cur_label += 'RSspca20' 385 | elif 'pca20RS' in p: 386 | cur_label += 'RSpca20' 387 | cur_label += '/' 388 | cur_label += 'separate decomp.' if 'decomp_separate' in p else 'joint decomp.' 389 | cur_label += '' if '_AE' in p else '/LR only!' 390 | plt.figure() 391 | plt.plot( 392 | dbg_epochs_, 393 | cur_values, 394 | label=cur_label) 395 | plt.title('LR L1=0.1 L2=0.1 res=3mm') 396 | plt.yticks(np.linspace(0., 1., 11)) 397 | plt.ylabel(k) 398 | plt.xlabel('epochs') 399 | plt.ylim(0., 1.05) 400 | plt.grid(True) 401 | plt.show() 402 | plt.savefig(op.join(WRITE_DIR, k.replace(' ', '_') + '.png')) 403 | 404 | 405 | # print class weights 406 | pkgs = ['nips3mm_vanilla/V0comps.npy'] 407 | comps = np.load(pkgs[0]) 408 | new_fname = 'comps_perc75' 409 | dump_comps(nifti_masker, new_fname, comps, perc=75, fwhm=None) --------------------------------------------------------------------------------